lib/hypothesis/src/engine.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const builtin = @import("builtin");
3 const Allocator = std.mem.Allocator;
4 const conjecture = @import("conjecture.zig");
5 const ConjectureData = conjecture.ConjectureData;
6 const ChoiceNode = conjecture.ChoiceNode;
7 const Status = conjecture.Status;
8 const DrawError = conjecture.DrawError;
9 const shrinker_mod = @import("shrinker.zig");
10 const database = @import("database.zig");
11
12 pub const TestFn = *const fn (data: *ConjectureData, allocator: Allocator) anyerror!void;
13 pub const TestFnWithContext = *const fn (
14 data: *ConjectureData,
15 allocator: Allocator,
16 context: *anyopaque,
17 ) anyerror!void;
18
19 pub const SeedCase = struct {
20 choices: []const ChoiceNode,
21 byte_blocks: ?[]const u8,
22 };
23
24 pub const Settings = struct {
25 max_examples: usize = 100,
26 max_replays: usize = 100,
27 max_choices: usize = 4096,
28 max_input_bytes: usize = conjecture.default_max_input_bytes,
29 max_shrinks: usize = 5000,
30 target_examples: usize = 100,
31 seed: ?u64 = null,
32 database_path: ?[]const u8 = null,
33 database_namespace: ?[]const u8 = null,
34 shrinking: bool = true,
35 report_failure: bool = true,
36 per_example_leak_check: bool = false,
37
38 pub fn quick() Settings {
39 return .{
40 .max_examples = 25,
41 .max_replays = 25,
42 .max_choices = 2048,
43 .max_input_bytes = 256 * 1024,
44 .max_shrinks = 1000,
45 .target_examples = 25,
46 };
47 }
48
49 pub fn dev() Settings {
50 return .{};
51 }
52
53 pub fn ci() Settings {
54 return .{
55 .max_examples = 1000,
56 .max_replays = 1000,
57 .max_choices = 8192,
58 .max_input_bytes = 4 * 1024 * 1024,
59 .max_shrinks = 20_000,
60 .target_examples = 1000,
61 };
62 }
63
64 pub fn withSeed(self: Settings, seed: ?u64) Settings {
65 var out = self;
66 out.seed = seed;
67 return out;
68 }
69
70 pub fn withDatabase(self: Settings, path: ?[]const u8) Settings {
71 var out = self;
72 out.database_path = path;
73 return out;
74 }
75
76 pub fn withNamespace(self: Settings, namespace: ?[]const u8) Settings {
77 var out = self;
78 out.database_namespace = namespace;
79 return out;
80 }
81
82 pub fn withSeedFromEnv(self: Settings) Settings {
83 if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) return self;
84 const threaded = std.Options.debug_threaded_io orelse return self;
85 const text = std.process.Environ.getPosix(
86 threaded.environ.process_environ,
87 seed_env_name,
88 ) orelse return self;
89 return self.withSeedText(text);
90 }
91
92 pub fn withSeedText(self: Settings, text: []const u8) Settings {
93 var out = self;
94 if (std.mem.eql(u8, text, "random")) {
95 out.seed = null;
96 return out;
97 }
98 out.seed = std.fmt.parseUnsigned(u64, text, 0) catch
99 @panic(seed_env_name ++ " must be an unsigned integer or \"random\"");
100 return out;
101 }
102 };
103
104 pub const seed_env_name = "TINY_HYPOTHESIS_SEED";
105
106 pub const TestResult = struct {
107 passed: bool,
108 valid_examples: usize,
109 invalid_examples: usize,
110 replayed_examples: usize,
111 database_entries_scanned: usize,
112 database_failures_rejected: usize,
113 replay_budget_saturated: bool,
114 failing_choices: ?[]const ChoiceNode,
115 failing_byte_blocks: ?[]const u8,
116 seed: u64,
117 failing_error: ?anyerror,
118 database_path: ?[]const u8,
119 database_namespace: ?[]const u8,
120 max_examples: usize,
121 max_replays: usize,
122 max_choices: usize,
123 max_input_bytes: usize,
124 max_shrinks: usize,
125 target_examples: usize,
126 per_example_leak_check: bool,
127
128 allocator: Allocator,
129
130 pub fn initFailureReplay(self: *const TestResult, allocator: Allocator) ?ConjectureData {
131 const choices = self.failing_choices orelse return null;
132 var replay = ConjectureData.initReplay(
133 allocator,
134 choices,
135 self.failing_byte_blocks,
136 );
137 replay.max_choices = self.max_choices;
138 replay.max_input_bytes = self.max_input_bytes;
139 return replay;
140 }
141
142 pub fn deinit(self: *TestResult) void {
143 if (self.failing_choices) |fc| self.allocator.free(fc);
144 if (self.failing_byte_blocks) |fbb| self.allocator.free(fbb);
145 }
146 };
147
148 const DirectRunContext = struct {
149 test_fn: TestFn,
150 };
151
152 const DirectRunThunk = struct {
153 fn call(
154 data: *ConjectureData,
155 ctx_allocator: Allocator,
156 context: *anyopaque,
157 ) anyerror!void {
158 const ctx: *const DirectRunContext = @ptrCast(@alignCast(context));
159 return ctx.test_fn(data, ctx_allocator);
160 }
161 };
162
163 pub fn run(allocator: Allocator, test_fn: TestFn, settings: Settings) !TestResult {
164 var ctx = DirectRunContext{ .test_fn = test_fn };
165 return runWithContext(allocator, DirectRunThunk.call, &ctx, settings);
166 }
167
168 pub fn runWithContext(
169 allocator: Allocator,
170 test_fn: TestFnWithContext,
171 context: *anyopaque,
172 settings: Settings,
173 ) !TestResult {
174 return runWithContextSeeded(allocator, test_fn, context, settings, &.{});
175 }
176
177 pub fn runWithContextSeeded(
178 allocator: Allocator,
179 test_fn: TestFnWithContext,
180 context: *anyopaque,
181 settings: Settings,
182 seed_cases: []const SeedCase,
183 ) !TestResult {
184 const seed = settings.seed orelse seedU64(0);
185
186 var valid_examples: usize = 0;
187 var invalid_examples: usize = 0;
188 var replayed_examples: usize = 0;
189 var database_entries_scanned: usize = 0;
190 var database_failures_rejected: usize = 0;
191 var replay_budget_saturated = seed_cases.len > settings.max_replays;
192 var failing_choices: ?[]ChoiceNode = null;
193 var failing_byte_blocks: ?[]u8 = null;
194 var failing_spans: ?[]conjecture.Span = null;
195 var failing_error: ?anyerror = null;
196 var target_choices: ?[]ChoiceNode = null;
197 var target_byte_blocks: ?[]u8 = null;
198 var target_score: f64 = -std.math.inf(f64);
199 defer {
200 if (target_choices) |choices| allocator.free(choices);
201 if (target_byte_blocks) |byte_blocks| allocator.free(byte_blocks);
202 }
203
204 var example_runner = ReusableExampleRunner.init(allocator, test_fn, context, settings);
205 defer example_runner.deinit();
206
207 if (seed_cases.len > 0) {
208 const seed_count = @min(seed_cases.len, settings.max_replays);
209 for (seed_cases[0..seed_count]) |seed_case| {
210 replayed_examples += 1;
211 assertReplayBudget(replayed_examples, settings.max_replays);
212 var outcome = try executeExample(
213 allocator,
214 &example_runner,
215 test_fn,
216 context,
217 settings,
218 .{ .replay = .{
219 .choices = seed_case.choices,
220 .byte_blocks = seed_case.byte_blocks,
221 } },
222 true,
223 );
224 defer outcome.deinit();
225
226 if (outcome.status == .interesting) {
227 failing_error = outcome.err;
228 adoptFailure(&outcome, &failing_choices, &failing_byte_blocks, &failing_spans);
229 break;
230 } else {
231 considerTargetOutcome(
232 allocator,
233 &outcome,
234 &target_choices,
235 &target_byte_blocks,
236 &target_score,
237 );
238 }
239 }
240 }
241
242 if (failing_choices == null) {
243 if (settings.database_path) |db_path| {
244 const remaining_replays = settings.max_replays - replayed_examples;
245 var saved = try database.ReplayCursor.init(
246 allocator,
247 .{
248 .db_path = db_path,
249 .namespace = settings.database_namespace,
250 .max_entries = remaining_replays,
251 .max_choices = settings.max_choices,
252 .max_byte_blocks = settings.max_input_bytes,
253 },
254 );
255 defer saved.deinit(allocator);
256 saved.activate();
257 while (try saved.next()) |entry| {
258 replayed_examples += 1;
259 assertReplayBudget(replayed_examples, settings.max_replays);
260 var outcome = try executeExample(
261 allocator,
262 &example_runner,
263 test_fn,
264 context,
265 settings,
266 .{ .replay = .{
267 .choices = entry.choices,
268 .byte_blocks = entry.byte_blocks,
269 } },
270 true,
271 );
272 defer outcome.deinit();
273
274 if (outcome.status == .interesting) {
275 failing_error = outcome.err;
276 adoptFailure(&outcome, &failing_choices, &failing_byte_blocks, &failing_spans);
277 break;
278 } else {
279 considerTargetOutcome(
280 allocator,
281 &outcome,
282 &target_choices,
283 &target_byte_blocks,
284 &target_score,
285 );
286 }
287 }
288 const database_status = saved.status();
289 database_entries_scanned = database_status.entries_scanned;
290 database_failures_rejected = database_status.failures_rejected;
291 replay_budget_saturated = replay_budget_saturated or
292 database_status.scan_budget_saturated;
293 }
294 }
295
296 replay_budget_saturated = replay_budget_saturated or
297 (settings.max_replays > 0 and replayed_examples == settings.max_replays);
298
299 if (failing_choices == null) {
300 var prng_seed = seed;
301 var examples_run: usize = 0;
302
303 while (examples_run < settings.max_examples) : (examples_run += 1) {
304 var outcome = try executeExample(
305 allocator,
306 &example_runner,
307 test_fn,
308 context,
309 settings,
310 .{ .generate = prng_seed },
311 true,
312 );
313 defer outcome.deinit();
314 prng_seed +%= 1;
315
316 switch (outcome.status) {
317 .valid => {
318 valid_examples += 1;
319 considerTargetOutcome(
320 allocator,
321 &outcome,
322 &target_choices,
323 &target_byte_blocks,
324 &target_score,
325 );
326 },
327 .invalid => invalid_examples += 1,
328 .interesting => {
329 failing_error = outcome.err;
330 adoptFailure(&outcome, &failing_choices, &failing_byte_blocks, &failing_spans);
331 break;
332 },
333 .overrun => {
334 invalid_examples += 1;
335 },
336 }
337 }
338 }
339
340 if (failing_choices == null and target_choices != null and settings.target_examples > 0) {
341 try runTargetPhase(
342 allocator,
343 &example_runner,
344 test_fn,
345 context,
346 settings,
347 &target_choices,
348 &target_byte_blocks,
349 &target_score,
350 &valid_examples,
351 &invalid_examples,
352 &failing_choices,
353 &failing_byte_blocks,
354 &failing_spans,
355 &failing_error,
356 );
357 }
358
359 if (failing_choices != null and settings.shrinking) {
360 const replay_ctx = ReplayContext{
361 .test_fn = test_fn,
362 .context = context,
363 .allocator = allocator,
364 .max_choices = settings.max_choices,
365 .max_input_bytes = settings.max_input_bytes,
366 .per_example_leak_check = settings.per_example_leak_check,
367 .runner = &example_runner,
368 };
369 var ctx = replay_ctx;
370
371 var result = try shrinker_mod.shrink(
372 allocator,
373 failing_choices.?,
374 failing_spans orelse &.{},
375 failing_byte_blocks,
376 &replayForShrink,
377 @ptrCast(&ctx),
378 settings.max_shrinks,
379 );
380 _ = &result;
381
382 allocator.free(failing_choices.?);
383 failing_choices = result.choices;
384 if (failing_byte_blocks) |fbb| allocator.free(fbb);
385 failing_byte_blocks = result.byte_blocks;
386 if (failing_spans) |fs| allocator.free(fs);
387 failing_spans = result.spans;
388 }
389
390 if (failing_choices != null) {
391 if (settings.database_path) |db_path| {
392 database.saveFailure(
393 allocator,
394 db_path,
395 settings.database_namespace,
396 failing_choices.?,
397 failing_byte_blocks,
398 ) catch {};
399 }
400 }
401
402 if (failing_spans) |fs| allocator.free(fs);
403
404 return .{
405 .passed = failing_choices == null,
406 .valid_examples = valid_examples,
407 .invalid_examples = invalid_examples,
408 .replayed_examples = replayed_examples,
409 .database_entries_scanned = database_entries_scanned,
410 .database_failures_rejected = database_failures_rejected,
411 .replay_budget_saturated = replay_budget_saturated,
412 .failing_choices = failing_choices,
413 .failing_byte_blocks = failing_byte_blocks,
414 .seed = seed,
415 .failing_error = failing_error,
416 .database_path = settings.database_path,
417 .database_namespace = settings.database_namespace,
418 .max_examples = settings.max_examples,
419 .max_replays = settings.max_replays,
420 .max_choices = settings.max_choices,
421 .max_input_bytes = settings.max_input_bytes,
422 .max_shrinks = settings.max_shrinks,
423 .target_examples = settings.target_examples,
424 .per_example_leak_check = settings.per_example_leak_check,
425 .allocator = allocator,
426 };
427 }
428
429 fn seedU64(fallback: u64) u64 {
430 var seed: u64 = undefined;
431 std.Io.Threaded.global_single_threaded.io().randomSecure(
432 std.mem.asBytes(&seed),
433 ) catch return fallback;
434 return seed;
435 }
436
437 fn assertReplayBudget(actual: usize, maximum: usize) void {
438 std.debug.assert(actual <= maximum);
439 }
440
441 const leak_failure = error.PerExampleLeak;
442 const ExampleDebugAllocator = std.heap.DebugAllocator(.{
443 .enable_memory_limit = true,
444 .safety = false,
445 });
446
447 const ExampleInput = union(enum) {
448 generate: u64,
449 replay: SeedCase,
450 };
451
452 const ExampleOutcome = struct {
453 status: Status,
454 choices: ?[]ChoiceNode = null,
455 byte_blocks: ?[]u8 = null,
456 spans: ?[]conjecture.Span = null,
457 targets: ?[]conjecture.TargetObservation = null,
458 err: ?anyerror = null,
459 allocator: Allocator,
460
461 fn deinit(self: *ExampleOutcome) void {
462 self.clearCaptured();
463 self.* = undefined;
464 }
465
466 fn clearCaptured(self: *ExampleOutcome) void {
467 if (self.choices) |choices| self.allocator.free(choices);
468 if (self.byte_blocks) |byte_blocks| self.allocator.free(byte_blocks);
469 if (self.spans) |spans| self.allocator.free(spans);
470 if (self.targets) |targets| freeTargetObservations(self.allocator, targets);
471 self.choices = null;
472 self.byte_blocks = null;
473 self.spans = null;
474 self.targets = null;
475 }
476
477 fn hasTargets(self: *const ExampleOutcome) bool {
478 if (self.targets) |targets| return targets.len > 0;
479 return false;
480 }
481 };
482
483 const ReusableExampleRunner = struct {
484 allocator: Allocator,
485 test_fn: TestFnWithContext,
486 context: *anyopaque,
487 settings: Settings,
488 data: ConjectureData,
489
490 fn init(
491 allocator: Allocator,
492 test_fn: TestFnWithContext,
493 context: *anyopaque,
494 settings: Settings,
495 ) ReusableExampleRunner {
496 var data = ConjectureData.init(allocator, 0);
497 data.max_choices = settings.max_choices;
498 data.max_input_bytes = settings.max_input_bytes;
499 return .{
500 .allocator = allocator,
501 .test_fn = test_fn,
502 .context = context,
503 .settings = settings,
504 .data = data,
505 };
506 }
507
508 fn deinit(self: *ReusableExampleRunner) void {
509 self.data.deinit();
510 }
511
512 fn execute(
513 self: *ReusableExampleRunner,
514 input: ExampleInput,
515 capture_interesting: bool,
516 ) !ExampleOutcome {
517 switch (input) {
518 .generate => |seed| self.data.reset(seed),
519 .replay => |seed_case| self.data.resetReplay(
520 seed_case.choices,
521 seed_case.byte_blocks,
522 ),
523 }
524 self.data.max_choices = self.settings.max_choices;
525 self.data.max_input_bytes = self.settings.max_input_bytes;
526
527 var failing_error: ?anyerror = null;
528 self.test_fn(&self.data, self.allocator, self.context) catch |err| {
529 markInterestingUnlessOverrun(&self.data, err, &failing_error);
530 };
531
532 return try outcomeFromDataAlloc(
533 self.allocator,
534 &self.data,
535 failing_error,
536 self.data.status,
537 capture_interesting and
538 (self.data.status == .interesting or self.data.targets.items.len > 0),
539 );
540 }
541 };
542
543 fn executeExample(
544 allocator: Allocator,
545 runner: *ReusableExampleRunner,
546 test_fn: TestFnWithContext,
547 context: *anyopaque,
548 settings: Settings,
549 input: ExampleInput,
550 capture_interesting: bool,
551 ) !ExampleOutcome {
552 if (!settings.per_example_leak_check) {
553 return runner.execute(input, capture_interesting);
554 }
555
556 return try executeLeakCheckedExampleAlloc(
557 allocator,
558 test_fn,
559 context,
560 settings,
561 input,
562 capture_interesting,
563 );
564 }
565
566 fn executeLeakCheckedExampleAlloc(
567 allocator: Allocator,
568 test_fn: TestFnWithContext,
569 context: *anyopaque,
570 settings: Settings,
571 input: ExampleInput,
572 capture_interesting: bool,
573 ) !ExampleOutcome {
574 var backing_arena = std.heap.ArenaAllocator.init(allocator);
575 defer backing_arena.deinit();
576
577 var debug_allocator: ExampleDebugAllocator = .{
578 .backing_allocator = backing_arena.allocator(),
579 };
580 const example_allocator = debug_allocator.allocator();
581
582 var data = initExampleData(example_allocator, input);
583 var data_deinited = false;
584 errdefer if (!data_deinited) data.deinit();
585 var debug_deinited = false;
586 errdefer if (!debug_deinited) {
587 _ = debug_allocator.deinit();
588 };
589 data.max_choices = settings.max_choices;
590 data.max_input_bytes = settings.max_input_bytes;
591
592 var failing_error: ?anyerror = null;
593 test_fn(&data, example_allocator, context) catch |err| {
594 markInterestingUnlessOverrun(&data, err, &failing_error);
595 };
596
597 var outcome = try outcomeFromDataAlloc(
598 allocator,
599 &data,
600 failing_error,
601 data.status,
602 capture_interesting,
603 );
604 errdefer outcome.deinit();
605 data.deinit();
606 data_deinited = true;
607
608 const leaked = debug_allocator.total_requested_bytes != 0;
609 _ = debug_allocator.deinit();
610 debug_deinited = true;
611 if (leaked) {
612 if (outcome.status != .interesting) {
613 outcome.status = .interesting;
614 outcome.err = leak_failure;
615 }
616 } else if (outcome.status != .interesting and !outcome.hasTargets()) {
617 outcome.clearCaptured();
618 }
619 return outcome;
620 }
621
622 fn initExampleData(allocator: Allocator, input: ExampleInput) ConjectureData {
623 return switch (input) {
624 .generate => |seed| ConjectureData.init(allocator, seed),
625 .replay => |seed_case| ConjectureData.initReplay(
626 allocator,
627 seed_case.choices,
628 seed_case.byte_blocks,
629 ),
630 };
631 }
632
633 fn outcomeFromDataAlloc(
634 allocator: Allocator,
635 data: *const ConjectureData,
636 failing_error: ?anyerror,
637 status: Status,
638 capture: bool,
639 ) !ExampleOutcome {
640 var outcome = ExampleOutcome{
641 .status = status,
642 .err = failing_error,
643 .allocator = allocator,
644 };
645 errdefer outcome.deinit();
646
647 if (capture) {
648 try captureDataAlloc(allocator, data, &outcome);
649 }
650 return outcome;
651 }
652
653 fn captureDataAlloc(
654 allocator: Allocator,
655 data: *const ConjectureData,
656 outcome: *ExampleOutcome,
657 ) !void {
658 outcome.choices = try allocator.alloc(ChoiceNode, data.choices.items.len);
659 @memcpy(outcome.choices.?, data.choices.items);
660
661 if (data.byte_blocks.items.len > 0) {
662 outcome.byte_blocks = try allocator.alloc(u8, data.byte_blocks.items.len);
663 @memcpy(outcome.byte_blocks.?, data.byte_blocks.items);
664 }
665
666 outcome.spans = try allocator.alloc(conjecture.Span, data.spans.items.len);
667 @memcpy(outcome.spans.?, data.spans.items);
668
669 if (data.targets.items.len > 0) {
670 outcome.targets = try allocator.alloc(conjecture.TargetObservation, data.targets.items.len);
671 var filled: usize = 0;
672 errdefer {
673 for (outcome.targets.?[0..filled]) |target_observation| {
674 allocator.free(target_observation.label);
675 }
676 allocator.free(outcome.targets.?);
677 outcome.targets = null;
678 }
679 for (data.targets.items, 0..) |target_observation, idx| {
680 const label = try allocator.dupe(u8, target_observation.label);
681 outcome.targets.?[idx] = .{
682 .label = label,
683 .value = target_observation.value,
684 };
685 filled += 1;
686 }
687 }
688 }
689
690 fn adoptFailure(
691 outcome: *ExampleOutcome,
692 failing_choices: *?[]ChoiceNode,
693 failing_byte_blocks: *?[]u8,
694 failing_spans: *?[]conjecture.Span,
695 ) void {
696 failing_choices.* = outcome.choices;
697 failing_byte_blocks.* = outcome.byte_blocks;
698 failing_spans.* = outcome.spans;
699 outcome.choices = null;
700 outcome.byte_blocks = null;
701 outcome.spans = null;
702 }
703
704 fn freeTargetObservations(allocator: Allocator, targets: []conjecture.TargetObservation) void {
705 for (targets) |target_observation| {
706 allocator.free(target_observation.label);
707 }
708 allocator.free(targets);
709 }
710
711 fn considerTargetOutcome(
712 allocator: Allocator,
713 outcome: *ExampleOutcome,
714 target_choices: *?[]ChoiceNode,
715 target_byte_blocks: *?[]u8,
716 target_score: *f64,
717 ) void {
718 if (outcome.status != .valid) return;
719 const score = outcomeTargetScore(outcome) orelse return;
720 if (outcome.choices == null) return;
721 if (target_choices.* != null and score <= target_score.*) return;
722
723 if (target_choices.*) |choices| allocator.free(choices);
724 if (target_byte_blocks.*) |byte_blocks| allocator.free(byte_blocks);
725
726 target_choices.* = outcome.choices;
727 target_byte_blocks.* = outcome.byte_blocks;
728 target_score.* = score;
729 outcome.choices = null;
730 outcome.byte_blocks = null;
731 }
732
733 fn outcomeTargetScore(outcome: *const ExampleOutcome) ?f64 {
734 const targets = outcome.targets orelse return null;
735 if (targets.len == 0) return null;
736 var score: f64 = 0.0;
737 for (targets) |target_observation| {
738 score += target_observation.value;
739 }
740 return score;
741 }
742
743 fn runTargetPhase(
744 allocator: Allocator,
745 runner: *ReusableExampleRunner,
746 test_fn: TestFnWithContext,
747 context: *anyopaque,
748 settings: Settings,
749 target_choices: *?[]ChoiceNode,
750 target_byte_blocks: *?[]u8,
751 target_score: *f64,
752 valid_examples: *usize,
753 invalid_examples: *usize,
754 failing_choices: *?[]ChoiceNode,
755 failing_byte_blocks: *?[]u8,
756 failing_spans: *?[]conjecture.Span,
757 failing_error: *?anyerror,
758 ) !void {
759 var attempts: usize = 0;
760 while (attempts < settings.target_examples and
761 failing_choices.* == null) : (attempts += 1)
762 {
763 const base_choices = target_choices.* orelse return;
764 const candidate = (try mutateTargetCandidate(
765 allocator,
766 base_choices,
767 attempts,
768 )) orelse continue;
769 defer allocator.free(candidate);
770
771 var outcome = try executeExample(
772 allocator,
773 runner,
774 test_fn,
775 context,
776 settings,
777 .{ .replay = .{
778 .choices = candidate,
779 .byte_blocks = target_byte_blocks.*,
780 } },
781 true,
782 );
783 defer outcome.deinit();
784
785 switch (outcome.status) {
786 .valid => {
787 valid_examples.* += 1;
788 considerTargetOutcome(
789 allocator,
790 &outcome,
791 target_choices,
792 target_byte_blocks,
793 target_score,
794 );
795 },
796 .invalid, .overrun => invalid_examples.* += 1,
797 .interesting => {
798 failing_error.* = outcome.err;
799 adoptFailure(&outcome, failing_choices, failing_byte_blocks, failing_spans);
800 return;
801 },
802 }
803 }
804 }
805
806 fn mutateTargetCandidate(
807 allocator: Allocator,
808 choices: []const ChoiceNode,
809 attempt: usize,
810 ) !?[]ChoiceNode {
811 if (choices.len == 0) return null;
812 const index = attempt % choices.len;
813 const mode = (attempt / choices.len) % 6;
814 const node = mutateChoiceForTarget(choices[index], mode) orelse return null;
815 if (node.value == choices[index].value) return null;
816
817 const candidate = try allocator.alloc(ChoiceNode, choices.len);
818 @memcpy(candidate, choices);
819 candidate[index] = node;
820 return candidate;
821 }
822
823 fn mutateChoiceForTarget(node: ChoiceNode, mode: usize) ?ChoiceNode {
824 if (node.was_forced) return null;
825 var out = node;
826 switch (node.kind) {
827 .integer, .boolean => {
828 out.value = targetIntegerCandidate(node, mode) orelse return null;
829 },
830 .float => {
831 out.value = targetFloatCandidate(node, mode) orelse return null;
832 },
833 .bytes => return null,
834 }
835 return out;
836 }
837
838 fn targetIntegerCandidate(node: ChoiceNode, mode: usize) ?u64 {
839 return switch (mode) {
840 0 => node.max,
841 1 => if (node.max > node.value)
842 node.value + @max(@as(u64, 1), (node.max - node.value) / 2)
843 else
844 null,
845 2 => node.min,
846 3 => node.shrink_towards,
847 4 => if (node.value < node.max) node.value + 1 else null,
848 5 => if (node.value > node.min) node.value - 1 else null,
849 else => null,
850 };
851 }
852
853 fn targetFloatCandidate(node: ChoiceNode, mode: usize) ?u64 {
854 const min: f64 = @bitCast(node.min);
855 const max: f64 = @bitCast(node.max);
856 const current: f64 = @bitCast(node.value);
857 if (!std.math.isFinite(current)) return null;
858
859 const candidate = switch (mode) {
860 0 => max,
861 1 => min,
862 2 => 0.0,
863 3 => 1.0,
864 4 => -1.0,
865 5 => if (std.math.isFinite(max)) current + (max - current) * 0.5 else return null,
866 else => return null,
867 };
868 if (!std.math.isFinite(candidate)) return null;
869 if (candidate < min or candidate > max) return null;
870 if (candidate == current) return null;
871 return @bitCast(candidate);
872 }
873
874 const ReplayContext = struct {
875 test_fn: TestFnWithContext,
876 context: *anyopaque,
877 allocator: Allocator,
878 max_choices: usize,
879 max_input_bytes: usize,
880 per_example_leak_check: bool,
881 runner: *ReusableExampleRunner,
882 };
883
884 fn replayForShrink(
885 choices: []const ChoiceNode,
886 byte_blocks: ?[]const u8,
887 context: *anyopaque,
888 ) Status {
889 const ctx: *const ReplayContext = @ptrCast(@alignCast(context));
890 var outcome = executeExample(
891 ctx.allocator,
892 ctx.runner,
893 ctx.test_fn,
894 ctx.context,
895 .{
896 .max_choices = ctx.max_choices,
897 .max_input_bytes = ctx.max_input_bytes,
898 .per_example_leak_check = ctx.per_example_leak_check,
899 },
900 .{ .replay = .{
901 .choices = choices,
902 .byte_blocks = byte_blocks,
903 } },
904 false,
905 ) catch return .overrun;
906 defer outcome.deinit();
907 return outcome.status;
908 }
909
910 fn markInterestingUnlessOverrun(
911 data: *ConjectureData,
912 err: anyerror,
913 failing_error: ?*?anyerror,
914 ) void {
915 if (data.status == .overrun) return;
916 data.markInteresting();
917 if (failing_error) |ptr| ptr.* = err;
918 }
919
920 const AlwaysPassingProperty = struct {
921 fn prop(data: *ConjectureData, _: Allocator) !void {
922 _ = try data.drawInteger(0, 100, 0);
923 }
924 };
925
926 const AlwaysFailingProperty = struct {
927 fn prop(_: *ConjectureData, _: Allocator) !void {
928 return error.AlwaysFails;
929 }
930 };
931
932 const PassingNoopProperty = struct {
933 fn prop(_: *ConjectureData, _: Allocator) !void {}
934 };
935
936 const ShrinkThresholdProperty = struct {
937 fn prop(data: *ConjectureData, _: Allocator) !void {
938 const x = try data.drawInteger(0, 1000, 0);
939 if (x > 100) return error.TooLarge;
940 }
941 };
942
943 const TargetMaximumProperty = struct {
944 fn prop(data: *ConjectureData, _: Allocator, _: *anyopaque) !void {
945 const x = try data.drawInteger(0, 1000, 0);
946 try data.target(x, "x");
947 if (x == 1000) return error.TargetReached;
948 }
949 };
950
951 const DuplicateTargetProperty = struct {
952 fn prop(data: *ConjectureData, _: Allocator) !void {
953 _ = try data.drawInteger(0, 10, 0);
954 try data.target(1, "score");
955 try data.target(2, "score");
956 }
957 };
958
959 const LeakOnlyProperty = struct {
960 fn prop(data: *ConjectureData, allocator: Allocator, _: *anyopaque) !void {
961 const x = try data.drawInteger(0, 10, 0);
962 if (x > 0) {
963 _ = try allocator.alloc(u8, 1);
964 }
965 }
966 };
967
968 const SeedFailureContext = struct {
969 target: u8,
970 max_size: usize,
971 };
972
973 const SeedFailureProperty = struct {
974 fn testFn(
975 data: *ConjectureData,
976 _: Allocator,
977 context_ptr: *anyopaque,
978 ) anyerror!void {
979 const ctx: *SeedFailureContext = @ptrCast(@alignCast(context_ptr));
980 const input = try data.drawBytes(0, ctx.max_size);
981 if (input.len > 0 and input[0] == ctx.target) {
982 return error.PropertyFailed;
983 }
984 }
985 };
986
987 const DatabaseFailureProperty = struct {
988 fn prop(_: *ConjectureData, _: Allocator) !void {
989 return error.PropertyFailed;
990 }
991 };
992
993 const DatabasePassingProperty = struct {
994 fn prop(data: *ConjectureData, _: Allocator) !void {
995 _ = try data.drawInteger(0, 10, 0);
996 }
997 };
998
999 const ReplayFailureProperty = struct {
1000 fn prop(data: *ConjectureData, _: Allocator) !void {
1001 _ = try data.drawInteger(0, 10, 0);
1002 return error.PropertyFailed;
1003 }
1004 };
1005
1006 const ReplayOverrunProperty = struct {
1007 fn prop(data: *ConjectureData, _: Allocator) !void {
1008 _ = try data.drawInteger(0, 10, 0);
1009 _ = try data.drawInteger(0, 10, 0);
1010 }
1011 };
1012
1013 const ReplayCountContext = struct {
1014 calls: usize = 0,
1015 };
1016
1017 const ReplayCountProperty = struct {
1018 fn prop(data: *ConjectureData, _: Allocator, context_ptr: *anyopaque) !void {
1019 const context: *ReplayCountContext = @ptrCast(@alignCast(context_ptr));
1020 context.calls += 1;
1021 _ = try data.drawInteger(0, 10, 0);
1022 }
1023 };
1024
1025 test "always-passing property passes" {
1026 const allocator = std.testing.allocator;
1027
1028 var result = try run(allocator, &AlwaysPassingProperty.prop, .{
1029 .max_examples = 10,
1030 .seed = 42,
1031 });
1032 defer result.deinit();
1033
1034 try std.testing.expect(result.passed);
1035 try std.testing.expectEqual(10, result.valid_examples);
1036 }
1037
1038 test "settings presets scale property budgets" {
1039 const quick = Settings.quick();
1040 const dev = Settings.dev();
1041 const ci = Settings.ci();
1042
1043 try std.testing.expect(quick.max_examples < dev.max_examples);
1044 try std.testing.expect(dev.max_examples < ci.max_examples);
1045 try std.testing.expect(quick.max_replays < dev.max_replays);
1046 try std.testing.expect(dev.max_replays < ci.max_replays);
1047 try std.testing.expect(quick.max_input_bytes < dev.max_input_bytes);
1048 try std.testing.expect(dev.max_input_bytes < ci.max_input_bytes);
1049 try std.testing.expect(quick.max_shrinks < ci.max_shrinks);
1050 try std.testing.expect(!dev.per_example_leak_check);
1051 }
1052
1053 test "result carries replay settings" {
1054 const allocator = std.testing.allocator;
1055 var tmp = std.testing.tmpDir(.{});
1056 defer tmp.cleanup();
1057 const tmp_path = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);
1058 defer allocator.free(tmp_path);
1059
1060 var result = try run(allocator, &AlwaysFailingProperty.prop, Settings.quick()
1061 .withSeed(99)
1062 .withDatabase(tmp_path)
1063 .withNamespace("engine-result-replay-settings"));
1064 defer result.deinit();
1065
1066 try std.testing.expect(!result.passed);
1067 try std.testing.expectEqual(@as(u64, 99), result.seed);
1068 try std.testing.expectEqualStrings(tmp_path, result.database_path.?);
1069 try std.testing.expectEqualStrings(
1070 "engine-result-replay-settings",
1071 result.database_namespace.?,
1072 );
1073 try std.testing.expectEqual(Settings.quick().max_examples, result.max_examples);
1074 try std.testing.expectEqual(Settings.quick().max_replays, result.max_replays);
1075 try std.testing.expectEqual(Settings.quick().max_input_bytes, result.max_input_bytes);
1076 }
1077
1078 test "explicit seeds stop at the replay budget" {
1079 var node = ChoiceNode{ .kind = .integer, .value = 1, .min = 0, .max = 10 };
1080 const seed = SeedCase{ .choices = (&node)[0..1], .byte_blocks = null };
1081 const seeds = [_]SeedCase{ seed, seed, seed };
1082 var context = ReplayCountContext{};
1083 var result = try runWithContextSeeded(
1084 std.testing.allocator,
1085 &ReplayCountProperty.prop,
1086 @ptrCast(&context),
1087 .{
1088 .max_examples = 0,
1089 .max_replays = 2,
1090 .target_examples = 0,
1091 .shrinking = false,
1092 .seed = 1,
1093 },
1094 &seeds,
1095 );
1096 defer result.deinit();
1097 try std.testing.expect(result.passed);
1098 try std.testing.expectEqual(@as(usize, 2), context.calls);
1099 try std.testing.expectEqual(@as(usize, 2), result.replayed_examples);
1100 try std.testing.expect(result.replay_budget_saturated);
1101 }
1102
1103 test "explicit seeds and database scans share the replay budget" {
1104 const allocator = std.testing.allocator;
1105 var tmp = std.testing.tmpDir(.{});
1106 defer tmp.cleanup();
1107 const tmp_path = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);
1108 defer allocator.free(tmp_path);
1109 for (0..3) |value| {
1110 try database.saveFailure(
1111 allocator,
1112 tmp_path,
1113 "shared-budget",
1114 &.{.{ .kind = .integer, .value = value, .min = 0, .max = 10 }},
1115 null,
1116 );
1117 }
1118 var seed_node = ChoiceNode{ .kind = .integer, .value = 9, .min = 0, .max = 10 };
1119 const seeds = [_]SeedCase{.{
1120 .choices = (&seed_node)[0..1],
1121 .byte_blocks = null,
1122 }};
1123 var context = ReplayCountContext{};
1124 var result = try runWithContextSeeded(
1125 allocator,
1126 &ReplayCountProperty.prop,
1127 @ptrCast(&context),
1128 .{
1129 .max_examples = 0,
1130 .max_replays = 2,
1131 .max_choices = 1,
1132 .max_input_bytes = 0,
1133 .target_examples = 0,
1134 .shrinking = false,
1135 .seed = 1,
1136 .database_path = tmp_path,
1137 .database_namespace = "shared-budget",
1138 },
1139 &seeds,
1140 );
1141 defer result.deinit();
1142 try std.testing.expect(result.passed);
1143 try std.testing.expectEqual(@as(usize, 2), context.calls);
1144 try std.testing.expectEqual(@as(usize, 2), result.replayed_examples);
1145 try std.testing.expectEqual(@as(usize, 1), result.database_entries_scanned);
1146 try std.testing.expect(result.replay_budget_saturated);
1147 }
1148
1149 test "always-failing property finds failure" {
1150 const allocator = std.testing.allocator;
1151
1152 var result = try run(allocator, &AlwaysFailingProperty.prop, .{
1153 .max_examples = 10,
1154 .seed = 42,
1155 });
1156 defer result.deinit();
1157
1158 try std.testing.expect(!result.passed);
1159 }
1160
1161 test "passing result has no failure replay" {
1162 const allocator = std.testing.allocator;
1163
1164 var result = try run(allocator, &PassingNoopProperty.prop, .{
1165 .max_examples = 1,
1166 .target_examples = 0,
1167 .seed = 42,
1168 });
1169 defer result.deinit();
1170
1171 try std.testing.expect(result.passed);
1172 try std.testing.expect(result.initFailureReplay(allocator) == null);
1173 }
1174
1175 test "failure replay restores minimized choices bytes and choice bound" {
1176 const allocator = std.testing.allocator;
1177
1178 var minimized_choices = [_]ChoiceNode{
1179 .{
1180 .kind = .integer,
1181 .value = 7,
1182 .min = 0,
1183 .max = 10,
1184 .shrink_towards = 0,
1185 },
1186 .{
1187 .kind = .integer,
1188 .value = 3,
1189 .min = 1,
1190 .max = 4,
1191 .shrink_towards = 1,
1192 },
1193 };
1194 const minimized_bytes = [_]u8{ 0x5a, 0x1c, 0xe7 };
1195 const result = TestResult{
1196 .passed = false,
1197 .valid_examples = 4,
1198 .invalid_examples = 1,
1199 .replayed_examples = 2,
1200 .database_entries_scanned = 3,
1201 .database_failures_rejected = 1,
1202 .replay_budget_saturated = false,
1203 .failing_choices = minimized_choices[0..],
1204 .failing_byte_blocks = minimized_bytes[0..],
1205 .seed = 42,
1206 .failing_error = error.PropertyFailed,
1207 .database_path = null,
1208 .database_namespace = null,
1209 .max_examples = 25,
1210 .max_replays = 25,
1211 .max_choices = minimized_choices.len,
1212 .max_input_bytes = minimized_bytes.len,
1213 .max_shrinks = 1000,
1214 .target_examples = 25,
1215 .per_example_leak_check = false,
1216 .allocator = allocator,
1217 };
1218
1219 var replay = result.initFailureReplay(allocator).?;
1220 defer replay.deinit();
1221
1222 try std.testing.expectEqual(minimized_choices.len, replay.max_choices);
1223 try std.testing.expectEqual(@as(u64, 7), try replay.drawInteger(0, 10, 0));
1224 try std.testing.expectEqualSlices(u8, &minimized_bytes, try replay.drawBytes(1, 4));
1225 try std.testing.expectEqualSlices(ChoiceNode, &minimized_choices, replay.choices.items);
1226 try std.testing.expectError(DrawError.Overrun, replay.drawBoolean());
1227 }
1228
1229 test "shrinks x > 100 to 101" {
1230 const allocator = std.testing.allocator;
1231
1232 var result = try run(allocator, &ShrinkThresholdProperty.prop, .{
1233 .max_examples = 200,
1234 .seed = 42,
1235 .shrinking = true,
1236 });
1237 defer result.deinit();
1238
1239 try std.testing.expect(!result.passed);
1240 if (result.failing_choices) |fc| {
1241 try std.testing.expect(fc.len > 0);
1242 try std.testing.expectEqual(101, fc[0].value);
1243 }
1244 }
1245
1246 test "target phase mutates valid seeds toward higher scores" {
1247 const allocator = std.testing.allocator;
1248
1249 var node = ChoiceNode{
1250 .kind = .integer,
1251 .value = 1,
1252 .min = 0,
1253 .max = 1000,
1254 .shrink_towards = 0,
1255 };
1256 const seed_cases = [_]SeedCase{
1257 .{
1258 .choices = (&node)[0..1],
1259 .byte_blocks = null,
1260 },
1261 };
1262 var unused_context: u8 = 0;
1263
1264 var result = try runWithContextSeeded(
1265 allocator,
1266 &TargetMaximumProperty.prop,
1267 @ptrCast(&unused_context),
1268 .{
1269 .max_examples = 0,
1270 .target_examples = 4,
1271 .shrinking = false,
1272 .seed = 1,
1273 },
1274 seed_cases[0..],
1275 );
1276 defer result.deinit();
1277
1278 try std.testing.expect(!result.passed);
1279 try std.testing.expectEqual(error.TargetReached, result.failing_error.?);
1280 try std.testing.expectEqual(@as(u64, 1000), result.failing_choices.?[0].value);
1281 }
1282
1283 test "duplicate target labels fail the property" {
1284 const allocator = std.testing.allocator;
1285
1286 var result = try run(allocator, &DuplicateTargetProperty.prop, .{
1287 .max_examples = 1,
1288 .target_examples = 0,
1289 .shrinking = false,
1290 .seed = 1,
1291 });
1292 defer result.deinit();
1293
1294 try std.testing.expect(!result.passed);
1295 try std.testing.expectEqual(
1296 conjecture.TargetError.DuplicateTargetLabel,
1297 result.failing_error.?,
1298 );
1299 }
1300
1301 test "per-example leak check shrinks leak-only failures" {
1302 const allocator = std.testing.allocator;
1303
1304 var node = ChoiceNode{
1305 .kind = .integer,
1306 .value = 7,
1307 .min = 0,
1308 .max = 10,
1309 .shrink_towards = 0,
1310 };
1311 const seed_cases = [_]SeedCase{
1312 .{
1313 .choices = (&node)[0..1],
1314 .byte_blocks = null,
1315 },
1316 };
1317 var unused_context: u8 = 0;
1318
1319 var result = try runWithContextSeeded(
1320 allocator,
1321 &LeakOnlyProperty.prop,
1322 @ptrCast(&unused_context),
1323 .{
1324 .max_examples = 0,
1325 .max_shrinks = 100,
1326 .seed = 1,
1327 .per_example_leak_check = true,
1328 .report_failure = false,
1329 },
1330 seed_cases[0..],
1331 );
1332 defer result.deinit();
1333
1334 try std.testing.expect(!result.passed);
1335 try std.testing.expectEqual(leak_failure, result.failing_error.?);
1336 try std.testing.expect(result.failing_choices != null);
1337 try std.testing.expectEqual(@as(u64, 1), result.failing_choices.?[0].value);
1338 }
1339
1340 test "runWithContextSeeded: seed case triggers failure with max_examples = 0" {
1341 const allocator = std.testing.allocator;
1342
1343 var ctx = SeedFailureContext{ .target = 0xAC, .max_size = 8 };
1344 var node = ChoiceNode{
1345 .kind = .integer,
1346 .value = 1,
1347 .min = 0,
1348 .max = 8,
1349 .shrink_towards = 0,
1350 };
1351 const seed_cases = [_]SeedCase{
1352 .{
1353 .choices = (&node)[0..1],
1354 .byte_blocks = &.{0xAC},
1355 },
1356 };
1357
1358 var result = try runWithContextSeeded(
1359 allocator,
1360 &SeedFailureProperty.testFn,
1361 @ptrCast(&ctx),
1362 .{ .max_examples = 0, .seed = 1 },
1363 seed_cases[0..],
1364 );
1365 defer result.deinit();
1366
1367 try std.testing.expect(!result.passed);
1368 try std.testing.expect(result.failing_byte_blocks != null);
1369 try std.testing.expect(result.failing_choices != null);
1370 }
1371
1372 test "database namespaces isolate failures" {
1373 const allocator = std.testing.allocator;
1374
1375 var tmp = std.testing.tmpDir(.{});
1376 defer tmp.cleanup();
1377
1378 const tmp_path = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);
1379 defer allocator.free(tmp_path);
1380
1381 var fail_result = try run(allocator, &DatabaseFailureProperty.prop, .{
1382 .max_examples = 1,
1383 .seed = 1,
1384 .shrinking = false,
1385 .database_path = tmp_path,
1386 .database_namespace = "prop-a",
1387 });
1388 defer fail_result.deinit();
1389 try std.testing.expect(!fail_result.passed);
1390
1391 var pass_result = try run(allocator, &DatabasePassingProperty.prop, .{
1392 .max_examples = 1,
1393 .seed = 2,
1394 .shrinking = false,
1395 .database_path = tmp_path,
1396 .database_namespace = "prop-b",
1397 });
1398 defer pass_result.deinit();
1399 try std.testing.expect(pass_result.passed);
1400 }
1401
1402 test "replay overrun does not fail the property" {
1403 const allocator = std.testing.allocator;
1404
1405 var tmp = std.testing.tmpDir(.{});
1406 defer tmp.cleanup();
1407
1408 const tmp_path = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);
1409 defer allocator.free(tmp_path);
1410
1411 var fail_result = try run(allocator, &ReplayFailureProperty.prop, .{
1412 .max_examples = 1,
1413 .seed = 3,
1414 .shrinking = false,
1415 .database_path = tmp_path,
1416 .database_namespace = "shared",
1417 });
1418 defer fail_result.deinit();
1419 try std.testing.expect(!fail_result.passed);
1420
1421 var replay_result = try run(allocator, &ReplayOverrunProperty.prop, .{
1422 .max_examples = 0,
1423 .seed = 4,
1424 .shrinking = false,
1425 .database_path = tmp_path,
1426 .database_namespace = "shared",
1427 });
1428 defer replay_result.deinit();
1429 try std.testing.expect(replay_result.passed);
1430 }
1431
1432 test "seed text selects fixed, hex, and fresh-entropy seeds" {
1433 const base = Settings.quick().withSeed(7);
1434 try std.testing.expectEqual(@as(?u64, 12345), base.withSeedText("12345").seed);
1435 try std.testing.expectEqual(@as(?u64, 0xabc), base.withSeedText("0xabc").seed);
1436 try std.testing.expectEqual(@as(?u64, null), base.withSeedText("random").seed);
1437 }