lib/bench/src/evidence.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

 1 const std = @import("std");
 2 const pretty_json = @import("pretty").json;
 3 const stabilizer = @import("stabilizer");
 4 
 5 pub fn writeBenchmarkEvidence(object: pretty_json.Object, config: ?stabilizer.Config) !void {
 6     try object.field("schema", "tiny.profile-evidence/v1");
 7     try object.field("primary_user", "agent");
 8     try object.field("runner", "lib.bench");
 9     try object.field("decision_state", "distribution_measurement");
10     const bias = try object.object("bias_repetition");
11     try bias.field("build", "single_observed");
12     try bias.field("process", "in_process_samples");
13     try bias.field("sample_sequence", "measured_acquisition_order");
14     try bias.field("sample_timestamps", "not_recorded");
15     try bias.field("heap", layoutState(config, .heap));
16     try bias.field("heap_pointer_validation", pointerValidationState(config));
17     try bias.field("stack", layoutState(config, .stack));
18     try bias.field("code", layoutState(config, .code));
19     try bias.field("environment", "single_observed");
20     try bias.end();
21     const statistics = try object.object("statistics");
22     try writeStrings(
23         try statistics.array("primary"),
24         &.{ "median_ns", "p75_ns", "p99_ns", "sample_ns" },
25     );
26     try writeStrings(
27         try statistics.array("secondary"),
28         &.{ "mean_ns", "stddev_ns", "min_ns", "max_ns" },
29     );
30     const intervals = try statistics.object("confidence_intervals");
31     try intervals.field("field", "confidence_intervals");
32     try intervals.field("method", "percentile_bootstrap");
33     try intervals.field(
34         "default_iterations",
35         @import("stats/root.zig").default_bootstrap_iterations,
36     );
37     try intervals.field("default_confidence", 0.95);
38     try intervals.end();
39     try statistics.end();
40     try writeStrings(try object.array("claim_limits"), &.{
41         "single-process benchmark samples do not prove cross-build speedups",
42         "code layout is not randomized by the default bench configuration",
43         "mean and standard deviation are secondary for skewed timing distributions",
44         "automatic eval calibration is discarded preconditioning, not an independent sample",
45         "automatic eval selection requires two consecutive target-covering calibration batches",
46         "fixed eval batches may remain timer-bound; inspect clock and sample_total_ns",
47         "sample order exposes within-process shifts without attributing their cause",
48     });
49     try object.end();
50 }
51 
52 const LayoutAxis = enum {
53     heap,
54     stack,
55     code,
56 };
57 
58 fn layoutState(config: ?stabilizer.Config, axis: LayoutAxis) []const u8 {
59     const actual = config orelse return "not_recorded";
60     return switch (axis) {
61         .heap => if (actual.heap.enabled) "per_sample" else "disabled",
62         .stack => if (actual.stack.enabled) "per_sample" else "disabled",
63         .code => if (actual.code.enabled) "configured_not_verified" else "not_randomized",
64     };
65 }
66 
67 fn pointerValidationState(config: ?stabilizer.Config) []const u8 {
68     const actual = config orelse return "not_recorded";
69     return if (actual.heap.pointer_validation) "enabled" else "disabled";
70 }
71 
72 fn writeStrings(array: pretty_json.Array, values: []const []const u8) !void {
73     for (values) |value| try array.element(value);
74     try array.end();
75 }
76 
77 test "benchmark evidence records code layout limit" {
78     var out = std.Io.Writer.Allocating.init(std.testing.allocator);
79     defer out.deinit();
80     var stream = pretty_json.Writer.init(&out.writer, .minified);
81     try writeBenchmarkEvidence(try stream.object(), .{
82         .heap = .{ .pointer_validation = false, .shuffle_slots = 5 },
83         .code = .{ .enabled = false },
84     });
85     const json = out.written();
86     try std.testing.expect(std.mem.indexOf(u8, json, "\"code\":\"not_randomized\"") != null);
87     try std.testing.expect(std.mem.indexOf(u8, json, "\"heap_pointer_validation\":\"disabled\"") != null);
88     try std.testing.expect(std.mem.indexOf(u8, json, "\"primary_user\":\"agent\"") != null);
89     try std.testing.expect(
90         std.mem.indexOf(u8, json, "\"sample_sequence\":\"measured_acquisition_order\"") != null,
91     );
92     try std.testing.expect(
93         std.mem.indexOf(u8, json, "\"sample_timestamps\":\"not_recorded\"") != null,
94     );
95     try std.testing.expect(std.mem.indexOf(u8, json, "\"confidence_intervals\":{\"field\":\"confidence_intervals\"") != null);
96 }