tiny.hypothesis.report

1const std = @import("std");2const pretty = @import("pretty");3const conjecture = @import("conjecture.zig");4const engine = @import("engine.zig");56const ChoiceNode = conjecture.ChoiceNode;7const TestResult = engine.TestResult;89pub fn printPropertyFailure(result: *const TestResult) void {10    var arena = std.heap.ArenaAllocator.init(result.allocator);11    defer arena.deinit();12    var report = propertyFailureReport(arena.allocator(), result) catch return;13    defer report.deinit();14    pretty.diagnostic.writeStderr(&report, .{ .width = 100 });15}1617pub fn writePropertyFailure(writer: *std.Io.Writer, result: *const TestResult) !void {18    var arena = std.heap.ArenaAllocator.init(result.allocator);19    defer arena.deinit();20    var report = try propertyFailureReport(arena.allocator(), result);21    defer report.deinit();22    try report.write(writer, .{ .width = 100 });23    try writer.writeByte('\n');24}2526fn propertyFailureReport(27    allocator: std.mem.Allocator,28    result: *const TestResult,29) !pretty.diagnostic.Report {30    var report = try pretty.diagnostic.Report.init(allocator, "Property failed");31    errdefer report.deinit();32    if (result.failing_error) |err| {33        try report.field("Error", "{s}", .{@errorName(err)});34    }35    try report.field("Seed", "{}", .{result.seed});36    try report.section("Settings");37    try report.line(".seed = {},", .{result.seed});38    if (result.database_path) |path| {39        try report.line(".database_path = \"{s}\",", .{path});40    } else {41        try report.line(".database_path = null,", .{});42    }43    if (result.database_namespace) |namespace| {44        try report.line(".database_namespace = \"{s}\",", .{namespace});45    } else {46        try report.line(".database_namespace = null,", .{});47    }48    try report.line(".max_examples = {},", .{result.max_examples});49    try report.line(".max_replays = {},", .{result.max_replays});50    try report.line(".max_choices = {},", .{result.max_choices});51    try report.line(".max_input_bytes = {},", .{result.max_input_bytes});52    try report.line(".max_shrinks = {},", .{result.max_shrinks});53    try report.line(".target_examples = {},", .{result.target_examples});54    try report.line(".per_example_leak_check = {},", .{result.per_example_leak_check});55    try report.field(56        "Replay corpus",57        "{} executed, {} entries scanned, {} rejected, saturated={}",58        .{59            result.replayed_examples,60            result.database_entries_scanned,61            result.database_failures_rejected,62            result.replay_budget_saturated,63        },64    );65    try addReplayHints(&report, result);66    if (result.failing_choices) |choices| {67        try addChoices(&report, "Minimal counterexample", choices);68    }69    if (result.failing_byte_blocks) |blocks| {70        try addByteBlocks(&report, blocks);71    }72    return report;73}7475pub fn printStatefulFailure(result: *const TestResult) void {76    var arena = std.heap.ArenaAllocator.init(result.allocator);77    defer arena.deinit();78    var report = statefulFailureReport(arena.allocator(), result) catch return;79    defer report.deinit();80    pretty.diagnostic.writeStderr(&report, .{ .width = 100 });81}8283pub fn writeStatefulFailure(writer: *std.Io.Writer, result: *const TestResult) !void {84    var arena = std.heap.ArenaAllocator.init(result.allocator);85    defer arena.deinit();86    var report = try statefulFailureReport(arena.allocator(), result);87    defer report.deinit();88    try report.write(writer, .{ .width = 100 });89    try writer.writeByte('\n');90}9192fn statefulFailureReport(93    allocator: std.mem.Allocator,94    result: *const TestResult,95) !pretty.diagnostic.Report {96    var report = try pretty.diagnostic.Report.init(97        allocator,98        "Stateful test failed",99    );100    errdefer report.deinit();101    try report.field("Seed", "{}", .{result.seed});102    if (result.failing_choices) |choices| {103        try addChoices(&report, "Minimal command sequence", choices);104    }105    return report;106}107108fn addChoices(109    report: *pretty.diagnostic.Report,110    label: []const u8,111    choices: []const ChoiceNode,112) !void {113    try report.section(label);114    try report.line("{} choices", .{choices.len});115    for (choices, 0..) |node, index| {116        switch (node.kind) {117            .integer => try report.line("[{d}] integer: {d}", .{ index, node.value }),118            .boolean => try report.line("[{d}] boolean: {}", .{ index, node.value != 0 }),119            .float => try report.line(120                "[{d}] float: {d}",121                .{ index, @as(f64, @bitCast(node.value)) },122            ),123            .bytes => try report.line("[{d}] bytes: len={d}", .{ index, node.value }),124        }125    }126}127128fn addReplayHints(129    report: *pretty.diagnostic.Report,130    result: *const TestResult,131) !void {132    if (result.database_path != null) {133        try report.line(134            "Replay: rerun the same test; database replay runs before" ++135                " random generation.",136            .{},137        );138    }139    if (result.database_namespace) |namespace| {140        try report.line(141            "Filter: pass -- --test-filter \"{s}\" to the owning zig build step.",142            .{namespace},143        );144    }145}146147fn addByteBlocks(report: *pretty.diagnostic.Report, blocks: []const u8) !void {148    try report.section("Minimal byte block");149    try report.line("{d} bytes", .{blocks.len});150    var storage: [3 * 64 + 32]u8 = undefined;151    var text = std.Io.Writer.fixed(&storage);152    const limit = @min(blocks.len, 64);153    try text.writeAll("hex:");154    for (blocks[0..limit]) |byte| {155        try text.print(" {x:0>2}", .{byte});156    }157    if (blocks.len > limit) {158        try text.print(" ... +{d} bytes", .{blocks.len - limit});159    }160    try report.line("{s}", .{text.buffered()});161}162163test "property failure report is buffered writer friendly" {164    var out = std.Io.Writer.Allocating.init(std.testing.allocator);165    defer out.deinit();166167    var choices = [_]ChoiceNode{.{168        .kind = .integer,169        .value = 7,170        .min = 0,171        .max = 10,172        .shrink_towards = 0,173    }};174    const result = TestResult{175        .passed = false,176        .valid_examples = 1,177        .invalid_examples = 0,178        .replayed_examples = 2,179        .database_entries_scanned = 3,180        .database_failures_rejected = 1,181        .replay_budget_saturated = false,182        .failing_choices = choices[0..],183        .failing_byte_blocks = &.{ 0x61, 0x62, 0xff },184        .seed = 42,185        .failing_error = error.PropertyFailed,186        .database_path = "zig-out/hypothesis-failures/example",187        .database_namespace = "example",188        .max_examples = 100,189        .max_replays = 100,190        .max_choices = 4096,191        .max_input_bytes = 1024 * 1024,192        .max_shrinks = 5000,193        .target_examples = 100,194        .per_example_leak_check = false,195        .allocator = std.testing.allocator,196    };197198    try writePropertyFailure(&out.writer, &result);199    const rendered = out.written();200    try std.testing.expect(std.mem.indexOf(u8, rendered, "Property failed") != null);201    try std.testing.expect(std.mem.indexOf(u8, rendered, "integer: 7") != null);202    try std.testing.expect(std.mem.indexOf(203        u8,204        rendered,205        "Filter: pass -- --test-filter \"example\"",206    ) != null);207    try std.testing.expect(std.mem.indexOf(u8, rendered, "hex: 61 62 ff") != null);208}

1pub const report = @import("report.zig");