lib/bench/src/event.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const pretty_json = @import("pretty").json;
3 const stabilizer = @import("stabilizer");
4
5 const evidence = @import("evidence.zig");
6 const instrumentation = @import("instrumentation.zig");
7 const metadata_mod = @import("metadata.zig");
8 const stats = @import("stats/root.zig");
9 const timing = @import("timing.zig");
10
11 pub const RunStart = struct {
12 output: []const u8,
13 coz_output: ?[]const u8 = null,
14 coz_summary: ?[]const u8 = null,
15 metadata: metadata_mod.BuildMetadata,
16 warmup: u32,
17 samples: u32,
18 evals: u32,
19 kind: ?[]const u8 = null,
20 load_samples: u32 = 0,
21 phase_samples: u32 = 0,
22 filter: ?[]const u8 = null,
23 stabilizer: bool = false,
24 stabilizer_config: ?stabilizer.Config = null,
25 profiling_session: ?instrumentation.ProfilingReport = null,
26 };
27
28 pub const RunEnd = struct {
29 files: ?u32 = null,
30 benches: u32,
31 errors: u32,
32 };
33
34 const SampledResultFixture = struct {
35 warmup: u32 = 1,
36 samples: u32 = 2,
37 evals: u32 = 3,
38 sample_ns: []const u64 = &.{ 20, 10 },
39 sample_total_ns: []const u64 = &.{ 60, 30 },
40 min_ns: u64 = 10,
41 max_ns: u64 = 20,
42 mean_ns: f64 = 15.0,
43 median_ns: u64 = 20,
44 p75_ns: u64 = 20,
45 p95_ns: u64 = 20,
46 p99_ns: u64 = 20,
47 total_ns: u64 = 30,
48 confidence_intervals: ?stats.BootstrapConfidenceIntervals = .{
49 .confidence = 0.95,
50 .iterations = 100,
51 .seed = 7,
52 .mean_ns = .{ .low_ns = 12.0, .high_ns = 18.0 },
53 .median_ns = .{ .low_ns = 10.0, .high_ns = 20.0 },
54 .p75_ns = .{ .low_ns = 20.0, .high_ns = 20.0 },
55 .p95_ns = .{ .low_ns = 20.0, .high_ns = 20.0 },
56 .p99_ns = .{ .low_ns = 20.0, .high_ns = 20.0 },
57 },
58 };
59
60 pub fn writeRunStart(writer: *std.Io.Writer, event: RunStart) !void {
61 var stream = pretty_json.Writer.init(writer, .minified);
62 const object = try stream.object();
63 try object.field("event", "run_start");
64 try object.field("protocol", "bench.run/v1");
65 if (event.kind) |kind| try object.field("kind", kind);
66 try object.field("timestamp_ns", timing.realNowNs());
67 try object.field("output", event.output);
68 if (event.coz_output) |path| try object.field("coz_output", path);
69 if (event.coz_summary) |path| try object.field("coz_summary", path);
70 try metadata_mod.writeBuildMetadata(try object.object("metadata"), event.metadata);
71 try object.field("warmup", event.warmup);
72 try object.field("samples", event.samples);
73 try object.field("evals", event.evals);
74 if (event.load_samples > 0) try object.field("load_samples", event.load_samples);
75 if (event.phase_samples > 0) try object.field("phase_samples", event.phase_samples);
76 if (event.filter) |filter| try object.field("filter", filter);
77 try object.field("stabilizer", event.stabilizer);
78 if (event.stabilizer_config) |layout_config| {
79 try metadata_mod.writeStabilizerConfig(
80 try object.object("stabilizer_config"),
81 layout_config,
82 );
83 }
84 try evidence.writeBenchmarkEvidence(
85 try object.object("agent_evidence"),
86 event.stabilizer_config,
87 );
88 if (event.profiling_session) |report| {
89 try instrumentation.writeProfilingReport(
90 try object.object("profiling_session"),
91 report,
92 );
93 }
94 try object.endLine();
95 }
96
97 pub fn writeSampleEvent(
98 writer: *std.Io.Writer,
99 id: []const u8,
100 index: u32,
101 evals: u32,
102 total_ns: u64,
103 ns: u64,
104 ) !void {
105 var stream = pretty_json.Writer.init(writer, .minified);
106 const object = try stream.object();
107 try object.field("event", "sample");
108 try object.field("id", id);
109 try object.field("sample_index", index);
110 try object.field("evals", evals);
111 try object.field("total_ns", total_ns);
112 try object.field("ns", ns);
113 try object.endLine();
114 }
115
116 pub fn writeRunEnd(writer: *std.Io.Writer, event: RunEnd) !void {
117 var stream = pretty_json.Writer.init(writer, .minified);
118 const object = try stream.object();
119 try object.field("event", "run_end");
120 if (event.files) |files| try object.field("files", files);
121 try object.field("benches", event.benches);
122 try object.field("errors", event.errors);
123 try object.endLine();
124 }
125
126 pub fn writeSampledResultFields(
127 comptime Row: type,
128 object: pretty_json.Object,
129 result: Row,
130 ) !void {
131 if (result.sample_ns.len != result.sample_total_ns.len) {
132 return error.InvalidSamplePairing;
133 }
134 try object.field("warmup", result.warmup);
135 try object.field("samples", result.samples);
136 try object.field("evals", result.evals);
137 try writeU64Array(try object.array("sample_ns"), result.sample_ns);
138 try writeU64Array(try object.array("sample_total_ns"), result.sample_total_ns);
139 try writeSampleSequence(try object.object("sample_sequence"));
140 try object.field("min_ns", result.min_ns);
141 try object.field("max_ns", result.max_ns);
142 try object.field("mean_ns", result.mean_ns);
143 try object.field("median_ns", result.median_ns);
144 try object.field("p75_ns", result.p75_ns);
145 try object.field("p95_ns", result.p95_ns);
146 try object.field("p99_ns", result.p99_ns);
147 try object.field("total_ns", result.total_ns);
148 if (@hasField(Row, "confidence_intervals")) {
149 if (result.confidence_intervals) |intervals| {
150 try writeBootstrapConfidenceIntervals(
151 try object.object("confidence_intervals"),
152 intervals,
153 );
154 } else {
155 try object.field("confidence_intervals", null);
156 }
157 }
158 }
159
160 fn writeSampleSequence(object: pretty_json.Object) !void {
161 try object.field("order", "measured_acquisition_order");
162 try object.field("index_origin", 0);
163 try object.field("pairing", "same_index");
164 const paired = try object.array("paired_fields");
165 try paired.element("sample_ns");
166 try paired.element("sample_total_ns");
167 try paired.end();
168 try object.field("timestamps", "not_recorded");
169 try object.field(
170 "claim_limit",
171 "sequence exposes within-process shifts but does not attribute their cause",
172 );
173 try object.end();
174 }
175
176 pub fn writeBootstrapConfidenceIntervals(
177 object: pretty_json.Object,
178 intervals: stats.BootstrapConfidenceIntervals,
179 ) !void {
180 try object.field("method", "percentile_bootstrap");
181 try object.field("confidence", intervals.confidence);
182 try object.field("iterations", intervals.iterations);
183 try object.field("seed", intervals.seed);
184 try writeConfidenceInterval(try object.object("mean_ns"), intervals.mean_ns);
185 try writeConfidenceInterval(try object.object("median_ns"), intervals.median_ns);
186 try writeConfidenceInterval(try object.object("p75_ns"), intervals.p75_ns);
187 try writeConfidenceInterval(try object.object("p95_ns"), intervals.p95_ns);
188 try writeConfidenceInterval(try object.object("p99_ns"), intervals.p99_ns);
189 try object.end();
190 }
191
192 fn writeConfidenceInterval(object: pretty_json.Object, interval: stats.ConfidenceInterval) !void {
193 try object.field("low_ns", interval.low_ns);
194 try object.field("high_ns", interval.high_ns);
195 try object.end();
196 }
197
198 fn writeU64Array(array: pretty_json.Array, values: []const u64) !void {
199 for (values) |item| try array.element(item);
200 try array.end();
201 }
202
203 test "run events keep benchmark report fields" {
204 const allocator = std.testing.allocator;
205 var out = std.Io.Writer.Allocating.init(allocator);
206 defer out.deinit();
207
208 try writeRunStart(&out.writer, .{
209 .output = "out.jsonl",
210 .coz_output = "out.coz.jsonl",
211 .metadata = .{
212 .git_sha = "abc123",
213 .git_dirty = false,
214 .optimize = "ReleaseFast",
215 .target = "native",
216 .zig_version = "0.16.0",
217 .host_os = "linux",
218 .host_arch = "x86_64",
219 },
220 .warmup = 1,
221 .samples = 2,
222 .evals = 3,
223 .kind = "load",
224 .load_samples = 4,
225 .filter = "answer",
226 .stabilizer = true,
227 .stabilizer_config = .{
228 .seed = 77,
229 .heap = .{ .shuffle_slots = 5 },
230 .code = .{ .enabled = false },
231 },
232 .profiling_session = .{
233 .capabilities = &.{
234 .{ .name = .coz, .state = .enabled, .requested = true, .artifact = "out.coz.jsonl" },
235 .{ .name = .perf, .state = .disabled, .reason = "no perf counter artifact configured" },
236 },
237 .artifacts = &.{
238 .{ .name = "coz_output", .path = "out.coz.jsonl" },
239 .{ .name = "coz_summary", .path = "out.coz.analysis.json" },
240 },
241 },
242 });
243 try writeSampleEvent(&out.writer, "bench_answer", 1, 3, 99, 33);
244 try writeRunEnd(&out.writer, .{ .files = 1, .benches = 2, .errors = 0 });
245
246 const jsonl = out.written();
247 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"event\":\"run_start\"") != null);
248 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"protocol\":\"bench.run/v1\"") != null);
249 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"timestamp_ns\":") != null);
250 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"git_sha\":\"abc123\"") != null);
251 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"host_os\":\"linux\"") != null);
252 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"host_arch\":\"x86_64\"") != null);
253 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"load_samples\":4") != null);
254 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"filter\":\"answer\"") != null);
255 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"stabilizer_config\":{\"seed\":77") != null);
256 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"agent_evidence\":") != null);
257 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"code\":\"not_randomized\"") != null);
258 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"profiling_session\":") != null);
259 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"name\":\"perf\"") != null);
260 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"coz_summary\":\"out.coz.analysis.json\"") != null);
261 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"event\":\"sample\"") != null);
262 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"id\":\"bench_answer\"") != null);
263 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"event\":\"run_end\"") != null);
264 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"errors\":0") != null);
265 }
266
267 test "sampled result fields write raw samples and summaries" {
268 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
269 defer out.deinit();
270 var stream = pretty_json.Writer.init(&out.writer, .minified);
271 const object = try stream.object();
272 try writeSampledResultFields(SampledResultFixture, object, .{});
273 try object.end();
274
275 const json = out.written();
276 try std.testing.expect(std.mem.indexOf(u8, json, "\"sample_ns\":[20,10]") != null);
277 try std.testing.expect(std.mem.indexOf(u8, json, "\"sample_total_ns\":[60,30]") != null);
278 try std.testing.expect(
279 std.mem.indexOf(u8, json, "\"order\":\"measured_acquisition_order\"") != null,
280 );
281 try std.testing.expect(std.mem.indexOf(u8, json, "\"pairing\":\"same_index\"") != null);
282 try std.testing.expect(std.mem.indexOf(u8, json, "\"timestamps\":\"not_recorded\"") != null);
283 try std.testing.expect(std.mem.indexOf(u8, json, "\"median_ns\":20") != null);
284 try std.testing.expect(std.mem.indexOf(u8, json, "\"confidence_intervals\":{\"method\":\"percentile_bootstrap\"") != null);
285 try std.testing.expect(std.mem.indexOf(u8, json, "\"seed\":7") != null);
286 try std.testing.expect(std.mem.indexOf(u8, json, "\"low_ns\":") != null);
287 try std.testing.expect(std.mem.indexOf(u8, json, "\"total_ns\":30") != null);
288 }
289
290 test "sampled result fields reject unpaired arrays" {
291 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
292 defer out.deinit();
293 var stream = pretty_json.Writer.init(&out.writer, .minified);
294 const object = try stream.object();
295 try std.testing.expectError(
296 error.InvalidSamplePairing,
297 writeSampledResultFields(SampledResultFixture, object, .{
298 .sample_total_ns = &.{60},
299 }),
300 );
301 }