lib/hypothesis/src/testing.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const Allocator = std.mem.Allocator;
3 const conjecture = @import("conjecture.zig");
4 const ConjectureData = conjecture.ConjectureData;
5 const DrawError = conjecture.DrawError;
6 const strategy_mod = @import("strategy.zig");
7 const engine_mod = @import("engine.zig");
8 const report_mod = @import("report.zig");
9 const Settings = engine_mod.Settings;
10 const TestResult = engine_mod.TestResult;
11 const SeedCase = engine_mod.SeedCase;
12
13 pub const FuzzInputOptions = std.testing.FuzzInputOptions;
14
15 const default_max_input_size = 4096;
16
17 fn ForAllTestType(comptime Property: type, comptime strats: anytype) type {
18 return struct {
19 fn testFn(data: *ConjectureData, _: Allocator) anyerror!void {
20 const Args = std.meta.ArgsTuple(@TypeOf(Property.property));
21 var args: Args = undefined;
22
23 inline for (@typeInfo(Args).@"struct".field_types, 0..) |Arg, idx| {
24 args[idx] = try drawFromComptime(
25 Arg,
26 strats[idx],
27 data,
28 );
29 }
30
31 const result = @call(.auto, Property.property, args);
32 if (!result) {
33 return error.PropertyFailed;
34 }
35 }
36 };
37 }
38
39 fn FuzzContextType(comptime Context: type) type {
40 return struct {
41 user_context: Context,
42 fuzz_fn: *const fn (context: Context, input: []const u8) anyerror!void,
43 max_input_size: usize,
44 };
45 }
46
47 fn FuzzTestType(comptime Context: type) type {
48 return struct {
49 fn testFn(
50 data: *ConjectureData,
51 _: Allocator,
52 context_ptr: *anyopaque,
53 ) anyerror!void {
54 const ctx: *FuzzContextType(Context) = @ptrCast(@alignCast(context_ptr));
55 const input = try data.drawBytes(0, ctx.max_input_size);
56 return ctx.fuzz_fn(ctx.user_context, input);
57 }
58 };
59 }
60
61 pub fn check(comptime Property: type, settings: Settings) !void {
62 const allocator = std.testing.allocator;
63
64 const run_settings = applyNamespace(Property, settings, null);
65 var result = try engine_mod.run(allocator, &Property.property, run_settings);
66 defer result.deinit();
67
68 if (!result.passed) {
69 if (run_settings.report_failure) {
70 printFailure(&result);
71 }
72 return error.PropertyFailed;
73 }
74 }
75
76 pub fn checkNamed(comptime Property: type, name: []const u8, settings: Settings) !void {
77 const allocator = std.testing.allocator;
78
79 const run_settings = applyNamespace(Property, settings, name);
80 var result = try engine_mod.run(allocator, &Property.property, run_settings);
81 defer result.deinit();
82
83 if (!result.passed) {
84 if (run_settings.report_failure) {
85 printFailure(&result);
86 }
87 return error.PropertyFailed;
88 }
89 }
90
91 pub fn forAll(
92 comptime Property: type,
93 comptime strats: anytype,
94 settings: Settings,
95 ) !void {
96 const allocator = std.testing.allocator;
97
98 const run_settings = applyNamespace(Property, settings, null);
99 var result = try engine_mod.run(
100 allocator,
101 &ForAllTestType(Property, strats).testFn,
102 run_settings,
103 );
104 defer result.deinit();
105
106 if (!result.passed) {
107 if (run_settings.report_failure) {
108 printFailure(&result);
109 }
110 return error.PropertyFailed;
111 }
112 }
113
114 pub fn forAllNamed(
115 comptime Property: type,
116 comptime strats: anytype,
117 name: []const u8,
118 settings: Settings,
119 ) !void {
120 const allocator = std.testing.allocator;
121
122 const run_settings = applyNamespace(Property, settings, name);
123 var result = try engine_mod.run(
124 allocator,
125 &ForAllTestType(Property, strats).testFn,
126 run_settings,
127 );
128 defer result.deinit();
129
130 if (!result.passed) {
131 if (run_settings.report_failure) {
132 printFailure(&result);
133 }
134 return error.PropertyFailed;
135 }
136 }
137
138 pub fn fuzz(
139 context: anytype,
140 comptime fuzzFn: fn (context: @TypeOf(context), input: []const u8) anyerror!void,
141 options: FuzzInputOptions,
142 ) anyerror!void {
143 return fuzzWithSettings(context, fuzzFn, options, .{});
144 }
145
146 pub fn fuzzWithSettings(
147 context: anytype,
148 comptime fuzzFn: fn (context: @TypeOf(context), input: []const u8) anyerror!void,
149 options: FuzzInputOptions,
150 settings: Settings,
151 ) anyerror!void {
152 const allocator = std.testing.allocator;
153
154 var seed_cases: []SeedCase = &.{};
155 var seed_nodes: []conjecture.ChoiceNode = &.{};
156 defer {
157 if (seed_cases.len > 0) allocator.free(seed_cases);
158 if (seed_nodes.len > 0) allocator.free(seed_nodes);
159 }
160
161 if (options.corpus.len > 0) {
162 const corpus_count = @min(options.corpus.len, settings.max_replays);
163 if (corpus_count > 0) {
164 seed_cases = try allocator.alloc(SeedCase, corpus_count);
165 seed_nodes = try allocator.alloc(conjecture.ChoiceNode, corpus_count);
166
167 for (options.corpus[0..corpus_count], 0..) |input, idx| {
168 const max_len: usize = if (input.len > default_max_input_size)
169 input.len
170 else
171 default_max_input_size;
172 seed_nodes[idx] = .{
173 .kind = .integer,
174 .value = @intCast(input.len),
175 .min = 0,
176 .max = @intCast(max_len),
177 .shrink_towards = 0,
178 };
179 seed_cases[idx] = .{
180 .choices = seed_nodes[idx .. idx + 1],
181 .byte_blocks = input,
182 };
183 }
184 }
185 }
186
187 var wrapper_ctx = FuzzContextType(@TypeOf(context)){
188 .user_context = context,
189 .fuzz_fn = fuzzFn,
190 .max_input_size = default_max_input_size,
191 };
192
193 var result = try engine_mod.runWithContextSeeded(
194 allocator,
195 &FuzzTestType(@TypeOf(context)).testFn,
196 @ptrCast(&wrapper_ctx),
197 settings,
198 seed_cases,
199 );
200 defer result.deinit();
201
202 if (!result.passed) {
203 if (settings.report_failure) {
204 printFailure(&result);
205 }
206 return error.PropertyFailed;
207 }
208 }
209
210 fn drawFromComptime(
211 comptime T: type,
212 comptime strat: anytype,
213 data: *ConjectureData,
214 ) DrawError!T {
215 const S = @TypeOf(strat);
216 if (@hasField(S, "min") and @hasField(S, "max") and @hasField(S, "shrink_towards")) {
217 const raw = try data.drawInteger(
218 strategy_mod.intToU64(T, strat.min),
219 strategy_mod.intToU64(T, strat.max),
220 strategy_mod.intToU64(T, strat.shrink_towards),
221 );
222 return strategy_mod.u64ToInt(T, raw);
223 }
224 if (T == bool) {
225 return data.drawBoolean();
226 }
227 @compileError("Unsupported strategy type for forAll. Use check() with ConjectureData instead.");
228 }
229
230 fn applyNamespace(
231 comptime Property: type,
232 settings: Settings,
233 explicit: ?[]const u8,
234 ) Settings {
235 var run_settings = settings;
236 if (explicit) |name| {
237 run_settings.database_namespace = name;
238 return run_settings;
239 }
240
241 if (run_settings.database_path != null and run_settings.database_namespace == null) {
242 run_settings.database_namespace = @typeName(Property);
243 }
244 return run_settings;
245 }
246
247 pub fn printFailure(result: *const TestResult) void {
248 report_mod.printPropertyFailure(result);
249 }
250
251 const DirectConjectureProperty = struct {
252 pub fn property(data: *ConjectureData, _: Allocator) !void {
253 const a = try data.drawInteger(0, 1000, 0);
254 const b = try data.drawInteger(0, 1000, 0);
255 if (a +% b != b +% a) return error.PropertyFailed;
256 }
257 };
258
259 const CommutativeAdditionProperty = struct {
260 pub fn property(a: i32, b: i32) bool {
261 return a +% b == b +% a;
262 }
263 };
264
265 const BooleanIdentityProperty = struct {
266 pub fn property(b: bool) bool {
267 return b == b;
268 }
269 };
270
271 const CustomMaxExamplesProperty = struct {
272 pub fn property(a: u8) bool {
273 _ = a;
274 return true;
275 }
276 };
277
278 const CorpusFailureContext = struct {
279 target: u8,
280 };
281
282 const CorpusFailureFuzz = struct {
283 pub fn fuzzFn(ctx: CorpusFailureContext, input: []const u8) anyerror!void {
284 if (input.len > 0 and input[0] == ctx.target) {
285 return error.PropertyFailed;
286 }
287 }
288 };
289
290 const PassingFuzz = struct {
291 pub fn fuzzFn(_: void, _: []const u8) anyerror!void {}
292 };
293
294 test "check: direct conjecture property" {
295 try check(DirectConjectureProperty, .{ .seed = 42 });
296 }
297
298 test "forAll: commutative addition" {
299 try forAll(CommutativeAdditionProperty, .{
300 strategy_mod.integers(i32, -1000, 1000),
301 strategy_mod.integers(i32, -1000, 1000),
302 }, .{ .seed = 42 });
303 }
304
305 test "forAll: boolean identity" {
306 try forAll(BooleanIdentityProperty, .{
307 strategy_mod.booleans(),
308 }, .{ .seed = 42 });
309 }
310
311 test "forAll: custom max_examples" {
312 try forAll(CustomMaxExamplesProperty, .{
313 strategy_mod.integers(u8, 0, 255),
314 }, .{ .max_examples = 50, .seed = 42 });
315 }
316
317 test "fuzz: corpus failure reports property failed" {
318 const corpus = [_][]const u8{
319 &.{0x00},
320 &.{ 0xAB, 0xCD },
321 };
322
323 try std.testing.expectError(
324 error.PropertyFailed,
325 fuzzWithSettings(
326 CorpusFailureContext{ .target = 0xAB },
327 CorpusFailureFuzz.fuzzFn,
328 .{ .corpus = corpus[0..] },
329 .{ .report_failure = false },
330 ),
331 );
332 }
333
334 test "fuzz: no failures passes" {
335 const corpus = [_][]const u8{
336 "alpha",
337 "beta",
338 };
339
340 try fuzz({}, PassingFuzz.fuzzFn, .{ .corpus = corpus[0..] });
341 }
342
343 test "fuzz: corpus admission stops at the replay budget" {
344 const corpus = [_][]const u8{
345 &.{0x00},
346 &.{0x01},
347 &.{0xAB},
348 };
349 try fuzzWithSettings(
350 CorpusFailureContext{ .target = 0xAB },
351 CorpusFailureFuzz.fuzzFn,
352 .{ .corpus = &corpus },
353 .{
354 .max_examples = 0,
355 .max_replays = 2,
356 .target_examples = 0,
357 .report_failure = false,
358 },
359 );
360 }