lib/hypothesis/src/report.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const pretty = @import("pretty");
3 const conjecture = @import("conjecture.zig");
4 const engine = @import("engine.zig");
5
6 const ChoiceNode = conjecture.ChoiceNode;
7 const TestResult = engine.TestResult;
8
9 pub fn printPropertyFailure(result: *const TestResult) void {
10 var arena = std.heap.ArenaAllocator.init(result.allocator);
11 defer arena.deinit();
12 var report = propertyFailureReport(arena.allocator(), result) catch return;
13 defer report.deinit();
14 pretty.diagnostic.writeStderr(&report, .{ .width = 100 });
15 }
16
17 pub fn writePropertyFailure(writer: *std.Io.Writer, result: *const TestResult) !void {
18 var arena = std.heap.ArenaAllocator.init(result.allocator);
19 defer arena.deinit();
20 var report = try propertyFailureReport(arena.allocator(), result);
21 defer report.deinit();
22 try report.write(writer, .{ .width = 100 });
23 try writer.writeByte('\n');
24 }
25
26 fn propertyFailureReport(
27 allocator: std.mem.Allocator,
28 result: *const TestResult,
29 ) !pretty.diagnostic.Report {
30 var report = try pretty.diagnostic.Report.init(allocator, "Property failed");
31 errdefer report.deinit();
32 if (result.failing_error) |err| {
33 try report.field("Error", "{s}", .{@errorName(err)});
34 }
35 try report.field("Seed", "{}", .{result.seed});
36 try report.section("Settings");
37 try report.line(".seed = {},", .{result.seed});
38 if (result.database_path) |path| {
39 try report.line(".database_path = \"{s}\",", .{path});
40 } else {
41 try report.line(".database_path = null,", .{});
42 }
43 if (result.database_namespace) |namespace| {
44 try report.line(".database_namespace = \"{s}\",", .{namespace});
45 } else {
46 try report.line(".database_namespace = null,", .{});
47 }
48 try report.line(".max_examples = {},", .{result.max_examples});
49 try report.line(".max_replays = {},", .{result.max_replays});
50 try report.line(".max_choices = {},", .{result.max_choices});
51 try report.line(".max_input_bytes = {},", .{result.max_input_bytes});
52 try report.line(".max_shrinks = {},", .{result.max_shrinks});
53 try report.line(".target_examples = {},", .{result.target_examples});
54 try report.line(".per_example_leak_check = {},", .{result.per_example_leak_check});
55 try report.field(
56 "Replay corpus",
57 "{} executed, {} entries scanned, {} rejected, saturated={}",
58 .{
59 result.replayed_examples,
60 result.database_entries_scanned,
61 result.database_failures_rejected,
62 result.replay_budget_saturated,
63 },
64 );
65 try addReplayHints(&report, result);
66 if (result.failing_choices) |choices| {
67 try addChoices(&report, "Minimal counterexample", choices);
68 }
69 if (result.failing_byte_blocks) |blocks| {
70 try addByteBlocks(&report, blocks);
71 }
72 return report;
73 }
74
75 pub fn printStatefulFailure(result: *const TestResult) void {
76 var arena = std.heap.ArenaAllocator.init(result.allocator);
77 defer arena.deinit();
78 var report = statefulFailureReport(arena.allocator(), result) catch return;
79 defer report.deinit();
80 pretty.diagnostic.writeStderr(&report, .{ .width = 100 });
81 }
82
83 pub fn writeStatefulFailure(writer: *std.Io.Writer, result: *const TestResult) !void {
84 var arena = std.heap.ArenaAllocator.init(result.allocator);
85 defer arena.deinit();
86 var report = try statefulFailureReport(arena.allocator(), result);
87 defer report.deinit();
88 try report.write(writer, .{ .width = 100 });
89 try writer.writeByte('\n');
90 }
91
92 fn statefulFailureReport(
93 allocator: std.mem.Allocator,
94 result: *const TestResult,
95 ) !pretty.diagnostic.Report {
96 var report = try pretty.diagnostic.Report.init(
97 allocator,
98 "Stateful test failed",
99 );
100 errdefer report.deinit();
101 try report.field("Seed", "{}", .{result.seed});
102 if (result.failing_choices) |choices| {
103 try addChoices(&report, "Minimal command sequence", choices);
104 }
105 return report;
106 }
107
108 fn addChoices(
109 report: *pretty.diagnostic.Report,
110 label: []const u8,
111 choices: []const ChoiceNode,
112 ) !void {
113 try report.section(label);
114 try report.line("{} choices", .{choices.len});
115 for (choices, 0..) |node, index| {
116 switch (node.kind) {
117 .integer => try report.line("[{d}] integer: {d}", .{ index, node.value }),
118 .boolean => try report.line("[{d}] boolean: {}", .{ index, node.value != 0 }),
119 .float => try report.line(
120 "[{d}] float: {d}",
121 .{ index, @as(f64, @bitCast(node.value)) },
122 ),
123 .bytes => try report.line("[{d}] bytes: len={d}", .{ index, node.value }),
124 }
125 }
126 }
127
128 fn addReplayHints(
129 report: *pretty.diagnostic.Report,
130 result: *const TestResult,
131 ) !void {
132 if (result.database_path != null) {
133 try report.line(
134 "Replay: rerun the same test; database replay runs before" ++
135 " random generation.",
136 .{},
137 );
138 }
139 if (result.database_namespace) |namespace| {
140 try report.line(
141 "Filter: pass -- --test-filter \"{s}\" to the owning zig build step.",
142 .{namespace},
143 );
144 }
145 }
146
147 fn addByteBlocks(report: *pretty.diagnostic.Report, blocks: []const u8) !void {
148 try report.section("Minimal byte block");
149 try report.line("{d} bytes", .{blocks.len});
150 var storage: [3 * 64 + 32]u8 = undefined;
151 var text = std.Io.Writer.fixed(&storage);
152 const limit = @min(blocks.len, 64);
153 try text.writeAll("hex:");
154 for (blocks[0..limit]) |byte| {
155 try text.print(" {x:0>2}", .{byte});
156 }
157 if (blocks.len > limit) {
158 try text.print(" ... +{d} bytes", .{blocks.len - limit});
159 }
160 try report.line("{s}", .{text.buffered()});
161 }
162
163 test "property failure report is buffered writer friendly" {
164 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
165 defer out.deinit();
166
167 var choices = [_]ChoiceNode{.{
168 .kind = .integer,
169 .value = 7,
170 .min = 0,
171 .max = 10,
172 .shrink_towards = 0,
173 }};
174 const result = TestResult{
175 .passed = false,
176 .valid_examples = 1,
177 .invalid_examples = 0,
178 .replayed_examples = 2,
179 .database_entries_scanned = 3,
180 .database_failures_rejected = 1,
181 .replay_budget_saturated = false,
182 .failing_choices = choices[0..],
183 .failing_byte_blocks = &.{ 0x61, 0x62, 0xff },
184 .seed = 42,
185 .failing_error = error.PropertyFailed,
186 .database_path = "zig-out/hypothesis-failures/example",
187 .database_namespace = "example",
188 .max_examples = 100,
189 .max_replays = 100,
190 .max_choices = 4096,
191 .max_input_bytes = 1024 * 1024,
192 .max_shrinks = 5000,
193 .target_examples = 100,
194 .per_example_leak_check = false,
195 .allocator = std.testing.allocator,
196 };
197
198 try writePropertyFailure(&out.writer, &result);
199 const rendered = out.written();
200 try std.testing.expect(std.mem.indexOf(u8, rendered, "Property failed") != null);
201 try std.testing.expect(std.mem.indexOf(u8, rendered, "integer: 7") != null);
202 try std.testing.expect(std.mem.indexOf(
203 u8,
204 rendered,
205 "Filter: pass -- --test-filter \"example\"",
206 ) != null);
207 try std.testing.expect(std.mem.indexOf(u8, rendered, "hex: 61 62 ff") != null);
208 }