lib/bench/src/suite.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const coz = @import("coz");
3 const memtrace = @import("memtrace");
4 const pretty = @import("pretty");
5 const pretty_json = pretty.json;
6 const stabilizer = @import("stabilizer");
7 const sys = @import("sys");
8 const tracy = @import("tracy");
9
10 const allocation = @import("allocation.zig");
11 const evidence = @import("evidence.zig");
12 const event = @import("event.zig");
13 const instrumentation = @import("instrumentation.zig");
14 const metadata_mod = @import("metadata.zig");
15 const output = @import("output.zig");
16 const stats = @import("stats/root.zig");
17 const timing = @import("timing.zig");
18
19 const Allocator = std.mem.Allocator;
20 const CountingAllocator = allocation.CountingAllocator;
21 const callWithStackPad = instrumentation.callWithStackPad;
22 const phaseAt = instrumentation.phaseAt;
23 const nowNs = timing.nowNs;
24 const saturatingI64 = timing.saturatingI64;
25 const jsonl_output_env = "BENCH_JSONL";
26 const coz_output_env = "BENCH_COZ_JSONL";
27 const coz_analysis_env = "BENCH_COZ_ANALYSIS_JSON";
28 const coz_experiments_env = "BENCH_COZ_EXPERIMENTS";
29 const tracy_output_env = "BENCH_TRACY_JSONL";
30 const tracy_summary_env = "BENCH_TRACY_SUMMARY_JSONL";
31 const min_time_env = "BENCH_MIN_TIME_NS";
32 const filter_env = "BENCH_FILTER";
33 const layout_env = "BENCH_LAYOUT";
34 const scoped_allocations_env = "TINY_PROFILE_ALLOCATIONS_PATH";
35 const metric_schema = "tiny.profiling.metric/v1";
36 const stretched_iteration_limit: u32 = 1_000_000;
37 const sample_batch_window_ns: u64 = 10 * std.time.ns_per_ms;
38 const zero_average_batch_size: u32 = 1000;
39 const automatic_eval_limit: u32 = 1_000_000;
40 const automatic_sample_time_limit_ns: u64 = 10 * std.time.ns_per_ms;
41 const max_calibration_steps: u32 = 21;
42 const max_calibration_batches: u32 = max_calibration_steps * 2;
43
44 pub const AutomaticEvaluation = struct {
45 min_sample_time_ns: u64 = 100 * std.time.ns_per_us,
46 max_sample_time_ns: u64 = automatic_sample_time_limit_ns,
47 clock_resolution_multiple: u32 = 1000,
48 max_evals: u32 = automatic_eval_limit,
49 };
50
51 pub const Evaluation = union(enum) {
52 automatic: AutomaticEvaluation,
53 fixed: u32,
54 };
55
56 pub const AutomaticSelection = struct {
57 policy: AutomaticEvaluation,
58 evals: u32,
59 clock_target_ns: u64,
60 target_ns: u64,
61 target_capped: bool,
62 calibration_batches: u32,
63 calibration_confirmations: u32,
64 calibration_evals: u64,
65 calibration_ns: u64,
66 calibration_last_ns: u64,
67 calibration_target_reached: bool,
68 samples_below_target: u32 = 0,
69 };
70
71 pub const EvaluationSelection = union(enum) {
72 automatic: AutomaticSelection,
73 fixed: u32,
74
75 pub fn evalCount(self: EvaluationSelection) u32 {
76 return switch (self) {
77 .automatic => |selection| selection.evals,
78 .fixed => |evals| evals,
79 };
80 }
81 };
82
83 pub const AllocationAttribution = enum {
84 none,
85 sample_call,
86 prepare_owner,
87 };
88
89 pub const Config = struct {
90 suite_name: []const u8 = "bench",
91
92 min_time_ns: u64 = 5_000_000,
93
94 max_iterations: u32 = 10_000,
95
96 min_iterations: u32 = 1,
97
98 evaluation: Evaluation = .{ .automatic = .{} },
99
100 warmup_iterations: u32 = 0,
101
102 prepare: ?PrepareFn = null,
103
104 teardown: ?TeardownFn = null,
105
106 allocation_attribution: AllocationAttribution = .none,
107
108 layout_randomization: ?stabilizer.Config = .{
109 .seed = stabilizer.default_seed,
110 .heap = .{
111 .pointer_validation = false,
112 .shuffle_slots = 16,
113 .max_shuffled_size = 16 * 1024,
114 },
115 .code = .{ .enabled = false },
116 },
117 };
118
119 pub const RunOptions = struct {
120 write_stdout: bool = true,
121 write_env_jsonl: bool = true,
122 };
123
124 pub const Result = struct {
125 name: []const u8,
126 iterations: u32,
127 evals: u32,
128 evaluation: EvaluationSelection,
129 clock_probe: timing.ClockProbe,
130 samples: []const u64,
131 sample_total_ns: []const u64,
132 min_ns: u64,
133 max_ns: u64,
134 mean_ns: f64,
135 median_ns: u64,
136 stddev_ns: f64,
137 p75_ns: u64,
138 p95_ns: u64,
139 p99_ns: u64,
140 total_ns: u64,
141 warmup_iterations: u32,
142 confidence_intervals: ?stats.BootstrapConfidenceIntervals = null,
143 allocation_attribution: AllocationAttribution,
144 alloc_count: ?u64 = null,
145 free_count: ?u64 = null,
146 alloc_bytes: ?u64 = null,
147 alloc_count_per_eval: ?u64 = null,
148 free_count_per_eval: ?u64 = null,
149 alloc_bytes_per_eval: ?u64 = null,
150 layout_randomization: ?stabilizer.Config = null,
151 };
152
153 pub const BenchFn = *const fn (Allocator) void;
154 pub const PrepareFn = *const fn (Allocator) void;
155 pub const TeardownFn = *const fn () void;
156
157 pub const JsonlOptions = struct {
158 suite_name: []const u8,
159 output_path: []const u8 = "",
160 metadata: ?metadata_mod.BuildMetadata = null,
161 };
162
163 const Definition = struct {
164 name: []const u8,
165 func: BenchFn,
166 config: Config,
167 };
168
169 pub const Suite = struct {
170 alloc: Allocator,
171 default_config: Config,
172 filter: ?[]const u8 = null,
173 clock_probe: ?timing.ClockProbe = null,
174 benchmarks: std.ArrayListUnmanaged(Definition) = .empty,
175 results: std.ArrayListUnmanaged(Result) = .empty,
176
177 pub fn init(alloc: Allocator, config: Config) Suite {
178 return .{ .alloc = alloc, .default_config = config };
179 }
180
181 pub fn deinit(self: *Suite) void {
182 for (self.results.items) |result| {
183 self.alloc.free(result.samples);
184 self.alloc.free(result.sample_total_ns);
185 }
186 self.results.deinit(self.alloc);
187 self.benchmarks.deinit(self.alloc);
188 }
189
190 pub fn add(self: *Suite, name: []const u8, func: BenchFn, override: anytype) !void {
191 try self.benchmarks.append(self.alloc, .{
192 .name = name,
193 .func = func,
194 .config = applyConfigOverride(self.default_config, override),
195 });
196 }
197
198 pub fn setFilter(self: *Suite, filter: ?[]const u8) void {
199 self.filter = filter;
200 }
201
202 pub fn run(self: *Suite) !void {
203 try self.runWithOptions(.{});
204 }
205
206 pub fn runWithOptions(self: *Suite, options: RunOptions) !void {
207 const clock_probe = try timing.probeClock();
208 self.clock_probe = clock_probe;
209 var coz_env = try CozEnv.read(self.alloc);
210 defer coz_env.deinit(self.alloc);
211 var tracy_env = try TracyEnv.read(self.alloc);
212 defer tracy_env.deinit(self.alloc);
213 var control_env = try ControlEnv.read(self.alloc);
214 defer control_env.deinit(self.alloc);
215 var coz_session: instrumentation.CozSession = .{};
216 try startCozSession(self.alloc, &coz_session, coz_env);
217 defer coz_session.deinit();
218 var tracy_session: instrumentation.ProfilingSession = .{};
219 try startTracySession(self.alloc, &tracy_session, tracy_env, self.default_config.suite_name);
220 defer tracy_session.deinit();
221 var memtrace_session: MemtraceSession = .{};
222 try memtrace_session.start(self.alloc);
223 defer memtrace_session.deinit(self.alloc);
224 var executed: usize = 0;
225
226 tracy.setThreadName("tiny-bench");
227 tracy.appInfo("tiny benchmark harness");
228 {
229 const suite_phase = phaseAt("bench.suite.run", @src());
230 defer suite_phase.end();
231
232 if (options.write_stdout) try output.writeHeader();
233
234 for (self.benchmarks.items) |def| {
235 if (!matchesFilter(def.name, self.filter)) continue;
236 if (!matchesFilter(def.name, control_env.filter)) continue;
237 const result = try runOne(
238 self.alloc,
239 memtrace_session.sampleAllocator(),
240 memtrace_session.tracerPtr(),
241 controlledDefinition(def, control_env),
242 clock_probe,
243 );
244 executed += 1;
245 try self.results.append(self.alloc, result);
246 if (options.write_stdout) try output.writeResult(result);
247 }
248
249 if (options.write_stdout) try output.writeStdoutDoc(pretty.hardline);
250 }
251 if (options.write_env_jsonl) try self.writeEnvJsonlFile();
252 try finishInstrumentation(
253 self.alloc,
254 executed,
255 &memtrace_session,
256 &coz_session,
257 coz_env,
258 &tracy_session,
259 tracy_env,
260 );
261 }
262
263 pub fn runSilent(self: *Suite) ![]const Result {
264 const clock_probe = try timing.probeClock();
265 self.clock_probe = clock_probe;
266 var coz_env = try CozEnv.read(self.alloc);
267 defer coz_env.deinit(self.alloc);
268 var tracy_env = try TracyEnv.read(self.alloc);
269 defer tracy_env.deinit(self.alloc);
270 var control_env = try ControlEnv.read(self.alloc);
271 defer control_env.deinit(self.alloc);
272 var coz_session: instrumentation.CozSession = .{};
273 try startCozSession(self.alloc, &coz_session, coz_env);
274 defer coz_session.deinit();
275 var tracy_session: instrumentation.ProfilingSession = .{};
276 try startTracySession(self.alloc, &tracy_session, tracy_env, self.default_config.suite_name);
277 defer tracy_session.deinit();
278 var memtrace_session: MemtraceSession = .{};
279 try memtrace_session.start(self.alloc);
280 defer memtrace_session.deinit(self.alloc);
281 var executed: usize = 0;
282
283 tracy.setThreadName("tiny-bench");
284 tracy.appInfo("tiny benchmark harness");
285 {
286 const suite_phase = phaseAt("bench.suite.run_silent", @src());
287 defer suite_phase.end();
288
289 for (self.benchmarks.items) |def| {
290 if (!matchesFilter(def.name, self.filter)) continue;
291 if (!matchesFilter(def.name, control_env.filter)) continue;
292 const result = try runOne(
293 self.alloc,
294 memtrace_session.sampleAllocator(),
295 memtrace_session.tracerPtr(),
296 controlledDefinition(def, control_env),
297 clock_probe,
298 );
299 executed += 1;
300 try self.results.append(self.alloc, result);
301 }
302 }
303 try finishInstrumentation(
304 self.alloc,
305 executed,
306 &memtrace_session,
307 &coz_session,
308 coz_env,
309 &tracy_session,
310 tracy_env,
311 );
312 return self.results.items;
313 }
314
315 pub fn writeJsonl(self: *const Suite, writer: *std.Io.Writer, options: JsonlOptions) !void {
316 try writeJsonlStart(self, writer, options);
317 for (self.results.items) |result| try writeJsonlResult(writer, options, result);
318 try event.writeRunEnd(writer, .{
319 .benches = @intCast(self.results.items.len),
320 .errors = 0,
321 });
322 }
323
324 pub fn writeJsonlFile(self: *const Suite, path: []const u8, options: JsonlOptions) !void {
325 try ensureParent(path);
326 var file = try sys.fs.createFile(path, .{ .truncate = true });
327 defer file.close(sys.fs.debugIo());
328
329 var buffer: [8192]u8 = undefined;
330 var writer = file.writer(sys.fs.debugIo(), &buffer);
331 try self.writeJsonl(&writer.interface, withOutputPath(options, path));
332 try writer.interface.flush();
333 }
334
335 pub fn appendJsonlFile(self: *const Suite, path: []const u8, options: JsonlOptions) !void {
336 try ensureParent(path);
337 var out = std.Io.Writer.Allocating.init(self.alloc);
338 defer out.deinit();
339 try self.writeJsonl(&out.writer, withOutputPath(options, path));
340 try sys.fs.appendFile(path, &.{out.written()});
341 }
342
343 fn writeEnvJsonlFile(self: *const Suite) !void {
344 const path = try sys.env.getOwned(self.alloc, jsonl_output_env);
345 defer if (path) |actual| self.alloc.free(actual);
346 if (path) |actual| try self.appendJsonlFile(actual, .{ .suite_name = self.default_config.suite_name });
347 }
348 };
349
350 const CozEnv = struct {
351 output_path: ?[]u8 = null,
352 analysis_path: ?[]u8 = null,
353 experiments: bool = false,
354
355 fn read(allocator: Allocator) !CozEnv {
356 const experiments_value = try sys.env.getOwned(allocator, coz_experiments_env);
357 defer if (experiments_value) |value| allocator.free(value);
358 return .{
359 .output_path = try sys.env.getOwned(allocator, coz_output_env),
360 .analysis_path = try sys.env.getOwned(allocator, coz_analysis_env),
361 .experiments = experimentsEnabled(experiments_value),
362 };
363 }
364
365 fn deinit(self: *CozEnv, allocator: Allocator) void {
366 if (self.output_path) |path| allocator.free(path);
367 if (self.analysis_path) |path| allocator.free(path);
368 self.* = .{};
369 }
370 };
371
372 const TracyEnv = struct {
373 output_path: ?[]u8 = null,
374 summary_path: ?[]u8 = null,
375
376 fn read(allocator: Allocator) !TracyEnv {
377 const output_path = try sys.env.getOwned(allocator, tracy_output_env);
378 errdefer if (output_path) |path| allocator.free(path);
379 const summary_path = try sys.env.getOwned(allocator, tracy_summary_env);
380 errdefer if (summary_path) |path| allocator.free(path);
381 if (output_path == null and summary_path != null) return error.InvalidTracyEnvironment;
382 return .{ .output_path = output_path, .summary_path = summary_path };
383 }
384
385 fn deinit(self: *TracyEnv, allocator: Allocator) void {
386 if (self.output_path) |path| allocator.free(path);
387 if (self.summary_path) |path| allocator.free(path);
388 self.* = .{};
389 }
390 };
391
392 fn experimentsEnabled(value: ?[]const u8) bool {
393 const actual = value orelse return false;
394 if (actual.len == 0) return false;
395 return !std.mem.eql(u8, actual, "0");
396 }
397
398 const ControlEnv = struct {
399 min_time_ns: ?u64 = null,
400 filter: ?[]u8 = null,
401 layout: LayoutControl = .inherit,
402
403 fn read(allocator: Allocator) !ControlEnv {
404 const min_time_value = try sys.env.getOwned(allocator, min_time_env);
405 defer if (min_time_value) |value| allocator.free(value);
406 const layout_value = try sys.env.getOwned(allocator, layout_env);
407 defer if (layout_value) |value| allocator.free(value);
408 const min_time_ns = try parseMinTime(min_time_value);
409 const layout = try parseLayoutControl(layout_value);
410 return .{
411 .min_time_ns = min_time_ns,
412 .filter = try sys.env.getOwned(allocator, filter_env),
413 .layout = layout,
414 };
415 }
416
417 fn deinit(self: *ControlEnv, allocator: Allocator) void {
418 if (self.filter) |value| allocator.free(value);
419 self.* = .{};
420 }
421 };
422
423 const LayoutControl = enum {
424 inherit,
425 disabled,
426 };
427
428 fn parseMinTime(value: ?[]const u8) !?u64 {
429 const actual = value orelse return null;
430 if (actual.len == 0) return null;
431 return std.fmt.parseUnsigned(u64, actual, 10) catch error.InvalidBenchMinTime;
432 }
433
434 fn parseLayoutControl(value: ?[]const u8) !LayoutControl {
435 const actual = value orelse return .inherit;
436 if (actual.len == 0) return .inherit;
437 if (std.mem.eql(u8, actual, "off")) return .disabled;
438 return error.InvalidBenchLayout;
439 }
440
441 fn controlledDefinition(def: Definition, control: ControlEnv) Definition {
442 var controlled = stretchedDefinition(def, control.min_time_ns);
443 if (control.layout == .disabled) controlled.config.layout_randomization = null;
444 return controlled;
445 }
446
447 fn stretchedDefinition(def: Definition, min_time_ns: ?u64) Definition {
448 const stretch = min_time_ns orelse return def;
449 var stretched = def;
450 stretched.config.min_time_ns = @max(def.config.min_time_ns, stretch);
451 stretched.config.max_iterations = @max(def.config.max_iterations, stretched_iteration_limit);
452 return stretched;
453 }
454
455 fn sampleBackingAllocator() Allocator {
456 return sys.allocator.benchmarkAllocator();
457 }
458
459 const MemtraceSession = struct {
460 output_path: ?[]u8 = null,
461 tracer: ?memtrace.Tracer = null,
462 traced: ?memtrace.TracingAllocator = null,
463
464 fn start(self: *MemtraceSession, control_allocator: Allocator) !void {
465 self.output_path = try sys.env.getOwned(control_allocator, scoped_allocations_env);
466 errdefer if (self.output_path) |path| control_allocator.free(path);
467 if (self.output_path == null) return;
468 self.tracer = try memtrace.Tracer.init(control_allocator, .{
469 .record_events = true,
470 .allocation_attribution = .stack,
471 });
472 if (self.tracer) |*tracer| {
473 self.traced = try tracer.tracedAllocatorWithOptions(sampleBackingAllocator(), .{
474 .name = "bench.sample",
475 .retention = .retains_freed_memory,
476 });
477 }
478 }
479
480 fn sampleAllocator(self: *MemtraceSession) Allocator {
481 if (self.traced) |*traced| return traced.allocator();
482 return sampleBackingAllocator();
483 }
484
485 fn tracerPtr(self: *MemtraceSession) ?*memtrace.Tracer {
486 if (self.tracer) |*tracer| return tracer;
487 return null;
488 }
489
490 fn finish(self: *MemtraceSession) !void {
491 const path = self.output_path orelse return;
492 const tracer = self.tracerPtr() orelse return;
493 try tracer.writeEventsBundlePath(path);
494 }
495
496 fn deinit(self: *MemtraceSession, control_allocator: Allocator) void {
497 if (self.tracer) |*tracer| tracer.deinit();
498 if (self.output_path) |path| control_allocator.free(path);
499 self.* = .{};
500 }
501 };
502
503 fn startCozSession(allocator: Allocator, session: *instrumentation.CozSession, env: CozEnv) !void {
504 if (env.output_path) |path| {
505 try session.startWithOutputPathAndOptions(allocator, path, .{
506 .resolve_sample_locations = true,
507 .start_experiment_worker = env.experiments,
508 });
509 } else {
510 try session.start(allocator);
511 }
512 }
513
514 fn finishCozSession(allocator: Allocator, session: *instrumentation.CozSession, env: CozEnv) !void {
515 try session.finish();
516 const output_path = env.output_path orelse return;
517 const analysis_path = env.analysis_path orelse return;
518 try writeCozAnalysisFile(allocator, output_path, analysis_path);
519 }
520
521 fn startTracySession(
522 allocator: Allocator,
523 session: *instrumentation.ProfilingSession,
524 env: TracyEnv,
525 name: []const u8,
526 ) !void {
527 const output_path = env.output_path orelse return;
528 try session.start(allocator, .{
529 .name = name,
530 .tracy_output = output_path,
531 .tracy_summary = env.summary_path,
532 .require_tracy = true,
533 });
534 tracy.plotConfig("bench.calibration.evals", .{ .unit = .count, .step = true });
535 tracy.plotConfig("bench.sample.ns", .{ .unit = .nanoseconds });
536 tracy.plotConfig("bench.definition.median_ns", .{ .unit = .nanoseconds });
537 }
538
539 fn finishTracySession(session: *instrumentation.ProfilingSession, env: TracyEnv) !void {
540 if (env.output_path == null) return;
541 try session.finish();
542 }
543
544 fn finishInstrumentation(
545 allocator: Allocator,
546 executed: usize,
547 memtrace_session: *MemtraceSession,
548 coz_session: *instrumentation.CozSession,
549 coz_env: CozEnv,
550 tracy_session: *instrumentation.ProfilingSession,
551 tracy_env: TracyEnv,
552 ) !void {
553 var first_error: ?anyerror = null;
554 if (executed > 0) memtrace_session.finish() catch |err| rememberError(&first_error, err);
555 finishCozSession(allocator, coz_session, coz_env) catch |err| rememberError(&first_error, err);
556 finishTracySession(tracy_session, tracy_env) catch |err| rememberError(&first_error, err);
557 if (first_error) |err| return err;
558 }
559
560 fn rememberError(first_error: *?anyerror, err: anyerror) void {
561 if (first_error.* == null) first_error.* = err;
562 }
563
564 fn writeCozAnalysisFile(allocator: Allocator, output_path: []const u8, analysis_path: []const u8) !void {
565 var summary = try coz.analysis.summarizeFileAlloc(allocator, output_path, coz.analysis.default_max_profile_bytes, .{});
566 defer summary.deinit(allocator);
567 try ensureParent(analysis_path);
568 var file = try sys.fs.createFile(analysis_path, .{ .truncate = true });
569 defer file.close(sys.fs.debugIo());
570 var buffer: [8192]u8 = undefined;
571 var writer = file.writer(sys.fs.debugIo(), &buffer);
572 try summary.writeJson(&writer.interface);
573 try writer.interface.flush();
574 }
575
576 fn matchesFilter(name: []const u8, filter: ?[]const u8) bool {
577 const needle = filter orelse return true;
578 return needle.len == 0 or std.mem.indexOf(u8, name, needle) != null;
579 }
580
581 pub fn time(func: BenchFn, alloc: Allocator) u64 {
582 const t0 = nowNs();
583 func(alloc);
584 const t1 = nowNs();
585 return @intCast(@max(t1 - t0, 0));
586 }
587
588 fn applyConfigOverride(default_config: Config, override: anytype) Config {
589 const Override = @TypeOf(override);
590 var config = default_config;
591 if (@hasField(Override, "suite_name")) config.suite_name = override.suite_name;
592 if (@hasField(Override, "min_time_ns")) config.min_time_ns = override.min_time_ns;
593 if (@hasField(Override, "max_iterations")) config.max_iterations = override.max_iterations;
594 if (@hasField(Override, "min_iterations")) config.min_iterations = override.min_iterations;
595 if (@hasField(Override, "evaluation")) config.evaluation = override.evaluation;
596 if (@hasField(Override, "warmup_iterations")) {
597 config.warmup_iterations = override.warmup_iterations;
598 }
599 if (@hasField(Override, "prepare")) config.prepare = override.prepare;
600 if (@hasField(Override, "teardown")) config.teardown = override.teardown;
601 if (@hasField(Override, "allocation_attribution")) {
602 config.allocation_attribution = override.allocation_attribution;
603 }
604 if (@hasField(Override, "layout_randomization")) {
605 config.layout_randomization = override.layout_randomization;
606 }
607 return config;
608 }
609
610 const DefinitionContext = struct {
611 name: []const u8,
612 runtime: *?stabilizer.Runtime,
613 func: BenchFn,
614 alloc: Allocator,
615 prepare: ?PrepareFn,
616 prepare_alloc: Allocator,
617 trace: ?*memtrace.Tracer,
618 };
619
620 const CollectedSamples = struct {
621 samples: std.ArrayListUnmanaged(u64) = .empty,
622 totals: std.ArrayListUnmanaged(u64) = .empty,
623 total_ns: u64 = 0,
624 iterations: u32 = 0,
625
626 fn deinit(self: *CollectedSamples, allocator: Allocator) void {
627 self.samples.deinit(allocator);
628 self.totals.deinit(allocator);
629 self.* = .{};
630 }
631 };
632
633 const MeasuredSample = struct {
634 per_eval_ns: u64,
635 total_ns: u64,
636 };
637
638 const CalibrationAction = union(enum) {
639 confirm,
640 reached,
641 exhausted,
642 grow: u32,
643 };
644
645 fn runOne(
646 control_alloc: Allocator,
647 sample_backing_alloc: Allocator,
648 trace: ?*memtrace.Tracer,
649 def: Definition,
650 clock_probe: timing.ClockProbe,
651 ) !Result {
652 const definition_phase = phaseAt("bench.definition", @src());
653 definition_phase.setName(def.name);
654 defer definition_phase.end();
655 var definition_trace = try enterTrace(trace, def.name);
656 defer definition_trace.exit();
657
658 const cfg = def.config;
659 try validatePreparedEvaluation(cfg);
660 var layout_runtime: ?stabilizer.Runtime = if (cfg.layout_randomization) |layout_config|
661 try stabilizer.Runtime.init(sample_backing_alloc, layout_config)
662 else
663 null;
664 defer if (layout_runtime) |*runtime| runtime.deinit();
665 var teardown_pending = cfg.teardown != null;
666 defer if (teardown_pending) cfg.teardown.?();
667 const layout_allocator = if (layout_runtime) |*runtime|
668 runtime.randomizedAllocator()
669 else
670 sample_backing_alloc;
671 var allocation_tracker = CountingAllocator.init(layout_allocator);
672 const tracked_allocator = allocation_tracker.allocator();
673 const bench_allocator = switch (cfg.allocation_attribution) {
674 .none, .prepare_owner => layout_allocator,
675 .sample_call => tracked_allocator,
676 };
677 const prepare_allocator = switch (cfg.allocation_attribution) {
678 .none, .sample_call => layout_allocator,
679 .prepare_owner => tracked_allocator,
680 };
681 const context = DefinitionContext{
682 .name = def.name,
683 .runtime = &layout_runtime,
684 .func = def.func,
685 .alloc = bench_allocator,
686 .prepare = cfg.prepare,
687 .prepare_alloc = prepare_allocator,
688 .trace = trace,
689 };
690 const selection = try selectEvaluation(context, cfg.evaluation, clock_probe);
691 try runWarmups(context, cfg.warmup_iterations, selection.evalCount());
692 if (cfg.allocation_attribution == .prepare_owner and cfg.warmup_iterations > 0) {
693 cfg.teardown.?();
694 }
695 if (cfg.allocation_attribution != .none) allocation_tracker.reset();
696 var collected = try collectSamples(control_alloc, context, cfg, selection.evalCount());
697 defer collected.deinit(control_alloc);
698 if (cfg.allocation_attribution == .prepare_owner) {
699 cfg.teardown.?();
700 teardown_pending = false;
701 }
702 return buildResult(
703 control_alloc,
704 def,
705 &collected,
706 selection,
707 clock_probe,
708 allocation_tracker.counts,
709 );
710 }
711
712 fn selectEvaluation(
713 context: DefinitionContext,
714 evaluation: Evaluation,
715 clock_probe: timing.ClockProbe,
716 ) !EvaluationSelection {
717 return switch (evaluation) {
718 .fixed => |evals| if (evals == 0) error.InvalidEvalCount else .{ .fixed = evals },
719 .automatic => |policy| calibrateEvaluation(context, policy, clock_probe),
720 };
721 }
722
723 fn validatePreparedEvaluation(config: Config) !void {
724 if (config.allocation_attribution == .prepare_owner and
725 (config.prepare == null or config.teardown == null))
726 {
727 return error.InvalidPrepareOwnerAttribution;
728 }
729 if (config.prepare == null) return;
730 switch (config.evaluation) {
731 .fixed => |evals| if (evals != 1) return error.InvalidPreparedEvaluation,
732 .automatic => return error.InvalidPreparedEvaluation,
733 }
734 }
735
736 fn calibrateEvaluation(
737 context: DefinitionContext,
738 policy: AutomaticEvaluation,
739 clock_probe: timing.ClockProbe,
740 ) !EvaluationSelection {
741 try validateAutomaticEvaluation(policy);
742 const clock_target = std.math.mul(
743 u64,
744 clock_probe.minimum_delta_ns,
745 policy.clock_resolution_multiple,
746 ) catch std.math.maxInt(u64);
747 const target = @min(@max(policy.min_sample_time_ns, clock_target), policy.max_sample_time_ns);
748 var selection = AutomaticSelection{
749 .policy = policy,
750 .evals = 1,
751 .clock_target_ns = clock_target,
752 .target_ns = target,
753 .target_capped = clock_target > policy.max_sample_time_ns,
754 .calibration_batches = 0,
755 .calibration_confirmations = 0,
756 .calibration_evals = 0,
757 .calibration_ns = 0,
758 .calibration_last_ns = 0,
759 .calibration_target_reached = false,
760 };
761 try runCalibration(context, &selection);
762 return .{ .automatic = selection };
763 }
764
765 fn validateAutomaticEvaluation(policy: AutomaticEvaluation) !void {
766 if (policy.min_sample_time_ns == 0) return error.InvalidEvaluationPolicy;
767 if (policy.max_sample_time_ns < policy.min_sample_time_ns) {
768 return error.InvalidEvaluationPolicy;
769 }
770 if (policy.max_sample_time_ns > automatic_sample_time_limit_ns) {
771 return error.InvalidEvaluationPolicy;
772 }
773 if (policy.clock_resolution_multiple == 0) return error.InvalidEvaluationPolicy;
774 if (policy.max_evals == 0 or policy.max_evals > automatic_eval_limit) {
775 return error.InvalidEvaluationPolicy;
776 }
777 }
778
779 fn runCalibration(context: DefinitionContext, selection: *AutomaticSelection) !void {
780 std.debug.assert(selection.evals > 0);
781 std.debug.assert(selection.evals <= selection.policy.max_evals);
782 const calibration_phase = phaseAt("bench.calibration", @src());
783 calibration_phase.setName(context.name);
784 defer calibration_phase.end();
785 var calibration_trace = try enterTrace(context.trace, "calibration");
786 defer calibration_trace.exit();
787
788 var steps: u32 = 0;
789 while (steps < max_calibration_steps) : (steps += 1) {
790 const first = try runCalibrationBatch(context, selection);
791 var action = calibrationAction(selection.*, first, null);
792 if (action == .confirm) {
793 selection.calibration_confirmations += 1;
794 const confirmation = try runCalibrationBatch(context, selection);
795 action = calibrationAction(selection.*, first, confirmation);
796 }
797 switch (action) {
798 .reached => {
799 selection.calibration_target_reached = true;
800 break;
801 },
802 .exhausted => break,
803 .grow => |evals| selection.evals = evals,
804 .confirm => unreachable,
805 }
806 }
807 if (steps == max_calibration_steps and
808 selection.evals != selection.policy.max_evals and
809 !selection.calibration_target_reached)
810 {
811 return error.CalibrationBoundExceeded;
812 }
813 calibration_phase.setValue(selection.evals);
814 tracy.plotInt("bench.calibration.evals", selection.evals);
815 }
816
817 fn runCalibrationBatch(context: DefinitionContext, selection: *AutomaticSelection) !u64 {
818 rerandomizeDefinitionLayout(context.runtime);
819 const elapsed = try measureDefinition(context, selection.evals);
820 selection.calibration_batches += 1;
821 selection.calibration_evals +|= selection.evals;
822 selection.calibration_ns +|= elapsed;
823 selection.calibration_last_ns = elapsed;
824 coz.progressNamed("bench.calibration");
825 return elapsed;
826 }
827
828 fn calibrationAction(
829 selection: AutomaticSelection,
830 first_ns: u64,
831 confirmation_ns: ?u64,
832 ) CalibrationAction {
833 if (first_ns >= selection.target_ns and confirmation_ns == null) return .confirm;
834 const observed = @min(first_ns, confirmation_ns orelse first_ns);
835 if (observed >= selection.target_ns) return .reached;
836 if (selection.evals == selection.policy.max_evals) return .exhausted;
837 return .{ .grow = nextEvalCount(
838 selection.evals,
839 observed,
840 selection.target_ns,
841 selection.policy.max_evals,
842 ) };
843 }
844
845 fn nextEvalCount(current: u32, elapsed_ns: u64, target_ns: u64, maximum: u32) u32 {
846 std.debug.assert(current > 0);
847 std.debug.assert(current <= maximum);
848 std.debug.assert(target_ns > 0);
849 const ratio = if (elapsed_ns == 0)
850 @as(u64, 10)
851 else
852 std.math.divCeil(u64, target_ns, elapsed_ns) catch 10;
853 const growth = @min(@max(ratio, 2), 10);
854 return @intCast(@min(@as(u64, current) * growth, maximum));
855 }
856
857 fn runWarmups(context: DefinitionContext, iterations: u32, evals: u32) !void {
858 for (0..iterations) |_| {
859 const warmup_phase = phaseAt("bench.warmup", @src());
860 warmup_phase.setName(context.name);
861 defer warmup_phase.end();
862 var warmup_trace = try enterTrace(context.trace, "warmup");
863 prepareDefinition(context);
864 callDefinitionEvals(context.runtime, context.func, context.alloc, evals);
865 warmup_trace.exit();
866 coz.progressNamed("bench.warmup");
867 }
868 }
869
870 fn collectSamples(
871 control_alloc: Allocator,
872 context: DefinitionContext,
873 cfg: Config,
874 evals: u32,
875 ) !CollectedSamples {
876 var collected: CollectedSamples = .{};
877 errdefer collected.deinit(control_alloc);
878 while ((collected.total_ns < cfg.min_time_ns or
879 collected.iterations < cfg.min_iterations) and
880 collected.iterations < cfg.max_iterations)
881 {
882 const batch = adaptiveBatchSize(
883 collected.iterations,
884 collected.total_ns,
885 cfg.min_time_ns,
886 cfg.max_iterations,
887 );
888 for (0..batch) |_| {
889 const sample = try measureSample(context, evals);
890 try collected.samples.append(control_alloc, sample.per_eval_ns);
891 try collected.totals.append(control_alloc, sample.total_ns);
892 collected.total_ns +|= sample.total_ns;
893 }
894 collected.iterations += batch;
895 if (collected.iterations >= cfg.min_iterations and
896 collected.total_ns >= cfg.min_time_ns) break;
897 }
898 return collected;
899 }
900
901 fn measureSample(context: DefinitionContext, evals: u32) !MeasuredSample {
902 rerandomizeDefinitionLayout(context.runtime);
903 prepareDefinition(context);
904 const sample_phase = phaseAt("bench.sample", @src());
905 sample_phase.setName(context.name);
906 defer sample_phase.end();
907 var sample_trace = try enterTrace(context.trace, "sample");
908 defer sample_trace.exit();
909 const elapsed = try measureDefinition(context, evals);
910 const per_eval = elapsed / evals;
911 sample_phase.setValue(per_eval);
912 tracy.plotInt("bench.sample.ns", saturatingI64(per_eval));
913 tracy.frameMarkNamed("bench.sample");
914 coz.progressNamed("bench.sample");
915 return .{ .per_eval_ns = per_eval, .total_ns = elapsed };
916 }
917
918 fn prepareDefinition(context: DefinitionContext) void {
919 const prepare = context.prepare orelse return;
920 callDefinition(context.runtime, prepare, context.prepare_alloc);
921 }
922
923 fn measureDefinition(context: DefinitionContext, evals: u32) !u64 {
924 std.debug.assert(evals > 0);
925 const started = try timing.readInstant();
926 callDefinitionEvals(context.runtime, context.func, context.alloc, evals);
927 const finished = try timing.readInstant();
928 return (try finished.elapsedSince(started)).asNanoseconds();
929 }
930
931 fn buildResult(
932 allocator: Allocator,
933 def: Definition,
934 collected: *CollectedSamples,
935 selection_value: EvaluationSelection,
936 clock_probe: timing.ClockProbe,
937 allocation_counts: allocation.Counts,
938 ) !Result {
939 var statistics_storage = try stats.Storage.init(allocator, .{
940 .samples = collected.samples.items.len,
941 });
942 defer statistics_storage.deinit(allocator);
943 statistics_storage.activate();
944 const sample_stats = try stats.computeSampleStats(
945 &statistics_storage,
946 collected.samples.items,
947 );
948 var selection = selection_value;
949 recordTargetCoverage(&selection, collected.totals.items);
950 const stddev_ns = sampleStddev(collected.samples.items, sample_stats.mean_ns);
951 coz.progressNamed("bench.definition.complete");
952 tracy.plotInt("bench.definition.median_ns", saturatingI64(sample_stats.median_ns));
953 tracy.message(def.name);
954 const owned_samples = try collected.samples.toOwnedSlice(allocator);
955 errdefer allocator.free(owned_samples);
956 const owned_totals = try collected.totals.toOwnedSlice(allocator);
957 const measured_evals =
958 @as(u64, collected.iterations) * @as(u64, selection.evalCount());
959 return .{
960 .name = def.name,
961 .iterations = collected.iterations,
962 .evals = selection.evalCount(),
963 .evaluation = selection,
964 .clock_probe = clock_probe,
965 .samples = owned_samples,
966 .sample_total_ns = owned_totals,
967 .min_ns = sample_stats.min_ns,
968 .max_ns = sample_stats.max_ns,
969 .mean_ns = sample_stats.mean_ns,
970 .median_ns = sample_stats.median_ns,
971 .stddev_ns = stddev_ns,
972 .p75_ns = sample_stats.p75_ns,
973 .p95_ns = sample_stats.p95_ns,
974 .p99_ns = sample_stats.p99_ns,
975 .total_ns = sample_stats.total_ns,
976 .warmup_iterations = def.config.warmup_iterations,
977 .confidence_intervals = sample_stats.confidence_intervals,
978 .allocation_attribution = def.config.allocation_attribution,
979 .alloc_count = if (def.config.allocation_attribution != .none) allocation_counts.alloc_count else null,
980 .free_count = if (def.config.allocation_attribution != .none) allocation_counts.free_count else null,
981 .alloc_bytes = if (def.config.allocation_attribution != .none) allocation_counts.alloc_bytes else null,
982 .alloc_count_per_eval = if (def.config.allocation_attribution != .none)
983 perEvaluation(allocation_counts.alloc_count, measured_evals)
984 else
985 null,
986 .free_count_per_eval = if (def.config.allocation_attribution != .none)
987 perEvaluation(allocation_counts.free_count, measured_evals)
988 else
989 null,
990 .alloc_bytes_per_eval = if (def.config.allocation_attribution != .none)
991 perEvaluation(allocation_counts.alloc_bytes, measured_evals)
992 else
993 null,
994 .layout_randomization = def.config.layout_randomization,
995 };
996 }
997
998 fn perEvaluation(total: u64, evaluations: u64) u64 {
999 std.debug.assert(evaluations > 0);
1000 return std.math.divCeil(u64, total, evaluations) catch unreachable;
1001 }
1002
1003 fn recordTargetCoverage(selection: *EvaluationSelection, totals: []const u64) void {
1004 std.debug.assert(totals.len <= std.math.maxInt(u32));
1005 switch (selection.*) {
1006 .automatic => |*automatic| {
1007 var below: u32 = 0;
1008 for (totals) |total| below += @intFromBool(total < automatic.target_ns);
1009 automatic.samples_below_target = below;
1010 },
1011 .fixed => {},
1012 }
1013 }
1014
1015 fn sampleStddev(samples: []const u64, mean_ns: f64) f64 {
1016 if (samples.len < 2) return 0;
1017 var variance_sum: f64 = 0;
1018 for (samples) |sample| {
1019 const difference = @as(f64, @floatFromInt(sample)) - mean_ns;
1020 variance_sum += difference * difference;
1021 }
1022 return @sqrt(variance_sum / @as(f64, @floatFromInt(samples.len - 1)));
1023 }
1024
1025 fn enterTrace(tracer: ?*memtrace.Tracer, label: []const u8) !memtrace.Span {
1026 return try memtrace.Context.init(tracer, null).enter(label);
1027 }
1028
1029 fn withOutputPath(options: JsonlOptions, path: []const u8) JsonlOptions {
1030 if (options.output_path.len != 0) return options;
1031 var updated = options;
1032 updated.output_path = path;
1033 return updated;
1034 }
1035
1036 fn writeJsonlStart(self: *const Suite, writer: *std.Io.Writer, options: JsonlOptions) !void {
1037 var stream = pretty_json.Writer.init(writer, .minified);
1038 const object = try stream.object();
1039 try object.field("event", "run_start");
1040 try object.field("protocol", "bench.suite/v1");
1041 try object.field("timestamp_ns", timing.realNowNs());
1042 try object.field("suite", options.suite_name);
1043 if (options.output_path.len != 0) try object.field("output", options.output_path);
1044 const metadata = options.metadata orelse metadata_mod.compiledBuildMetadata();
1045 try metadata_mod.writeBuildMetadata(try object.object("metadata"), metadata);
1046 if (self.clock_probe) |clock_probe| {
1047 try writeClockProbe(try object.object("clock"), clock_probe);
1048 }
1049 if (self.filter) |filter| try object.field("filter", filter);
1050 try object.field("definitions", self.benchmarks.items.len);
1051 try object.field("benchmarks", self.results.items.len);
1052 try object.endLine();
1053 }
1054
1055 fn writeJsonlResult(writer: *std.Io.Writer, options: JsonlOptions, result: Result) !void {
1056 const row = .{
1057 .warmup = result.warmup_iterations,
1058 .samples = result.iterations,
1059 .evals = result.evals,
1060 .sample_ns = result.samples,
1061 .sample_total_ns = result.sample_total_ns,
1062 .min_ns = result.min_ns,
1063 .max_ns = result.max_ns,
1064 .mean_ns = result.mean_ns,
1065 .median_ns = result.median_ns,
1066 .p75_ns = result.p75_ns,
1067 .p95_ns = result.p95_ns,
1068 .p99_ns = result.p99_ns,
1069 .total_ns = result.total_ns,
1070 .confidence_intervals = result.confidence_intervals,
1071 };
1072
1073 var stream = pretty_json.Writer.init(writer, .minified);
1074 const object = try stream.object();
1075 try object.field("schema", metric_schema);
1076 try object.field("event", "bench_end");
1077 try object.field("family", "timing");
1078 try object.field("metric", "duration_ns");
1079 try object.field("unit", "ns");
1080 try object.field("aggregation", "sample_distribution");
1081 try object.field("id", result.name);
1082 try object.field("path", options.suite_name);
1083 try object.field("name", result.name);
1084 try object.field("suite", options.suite_name);
1085 try event.writeSampledResultFields(@TypeOf(row), object, row);
1086 try writeClockProbe(try object.object("clock"), result.clock_probe);
1087 try writeEvaluation(try object.object("evaluation"), result.evaluation);
1088 if (result.layout_randomization) |layout_config| {
1089 try metadata_mod.writeStabilizerConfig(
1090 try object.object("stabilizer_config"),
1091 layout_config,
1092 );
1093 } else {
1094 try object.field("stabilizer_config", @as(?u8, null));
1095 }
1096 try evidence.writeBenchmarkEvidence(
1097 try object.object("agent_evidence"),
1098 result.layout_randomization,
1099 );
1100 try object.field("allocation_attribution", @tagName(result.allocation_attribution));
1101 try object.field("alloc_count", result.alloc_count);
1102 try object.field("free_count", result.free_count);
1103 try object.field("alloc_bytes", result.alloc_bytes);
1104 try object.field("alloc_count_per_eval", result.alloc_count_per_eval);
1105 try object.field("free_count_per_eval", result.free_count_per_eval);
1106 try object.field("alloc_bytes_per_eval", result.alloc_bytes_per_eval);
1107 try object.endLine();
1108 }
1109
1110 fn writeClockProbe(object: pretty_json.Object, probe: timing.ClockProbe) !void {
1111 try object.field("schema", "bench.clock/v1");
1112 try object.field("source", probe.source);
1113 try object.field("reads", probe.reads);
1114 try object.field("positive_deltas", probe.positive_deltas);
1115 try object.field("minimum_delta_ns", probe.minimum_delta_ns);
1116 try object.field("span_ns", probe.span_ns);
1117 try object.end();
1118 }
1119
1120 fn writeEvaluation(
1121 object: pretty_json.Object,
1122 selection: EvaluationSelection,
1123 ) !void {
1124 try object.field("schema", "bench.evaluation/v1");
1125 switch (selection) {
1126 .fixed => |evals| {
1127 try object.field("mode", "fixed");
1128 try object.field("evals", evals);
1129 },
1130 .automatic => |automatic| try writeAutomaticEvaluation(object, automatic),
1131 }
1132 try object.end();
1133 }
1134
1135 fn writeAutomaticEvaluation(
1136 object: pretty_json.Object,
1137 selection: AutomaticSelection,
1138 ) !void {
1139 try object.field("mode", "automatic");
1140 try object.field("evals", selection.evals);
1141 const policy = try object.object("policy");
1142 try policy.field("min_sample_time_ns", selection.policy.min_sample_time_ns);
1143 try policy.field("max_sample_time_ns", selection.policy.max_sample_time_ns);
1144 try policy.field(
1145 "clock_resolution_multiple",
1146 selection.policy.clock_resolution_multiple,
1147 );
1148 try policy.field("max_evals", selection.policy.max_evals);
1149 try policy.end();
1150 try object.field("clock_target_ns", selection.clock_target_ns);
1151 try object.field("target_ns", selection.target_ns);
1152 try object.field("target_capped", selection.target_capped);
1153 try object.field("calibration_batches", selection.calibration_batches);
1154 try object.field("calibration_confirmations", selection.calibration_confirmations);
1155 try object.field("calibration_evals", selection.calibration_evals);
1156 try object.field("calibration_ns", selection.calibration_ns);
1157 try object.field("calibration_last_ns", selection.calibration_last_ns);
1158 try object.field("calibration_target_reached", selection.calibration_target_reached);
1159 try object.field("samples_below_target", selection.samples_below_target);
1160 }
1161
1162 fn ensureParent(path: []const u8) !void {
1163 if (std.fs.path.dirname(path)) |dir| {
1164 if (dir.len > 0) try sys.fs.createDirPath(dir);
1165 }
1166 }
1167
1168 fn rerandomizeDefinitionLayout(runtime: *?stabilizer.Runtime) void {
1169 instrumentation.rerandomizeLayout(runtime, "bench.stabilizer.rerandomize");
1170 }
1171
1172 fn adaptiveBatchSize(iterations: u32, total_ns: u64, min_time_ns: u64, max_iterations: u32) u32 {
1173 if (iterations == 0) return 1;
1174 const remaining_iterations = max_iterations - iterations;
1175 const average_ns = total_ns / iterations;
1176 const target_ns = @min(min_time_ns -| total_ns, sample_batch_window_ns);
1177 const estimated: u64 = if (average_ns == 0) zero_average_batch_size else target_ns / average_ns;
1178 return @intCast(@min(@max(estimated, 1), remaining_iterations));
1179 }
1180
1181 fn callDefinitionEvals(runtime: *?stabilizer.Runtime, func: BenchFn, alloc: Allocator, evals: u32) void {
1182 for (0..evals) |_| callDefinition(runtime, func, alloc);
1183 }
1184
1185 fn callDefinition(runtime: *?stabilizer.Runtime, func: BenchFn, alloc: Allocator) void {
1186 if (runtime.*) |*active| {
1187 const pad = active.nextStackPad();
1188 callWithStackPad(pad.unit, func, .{alloc});
1189 } else {
1190 func(alloc);
1191 }
1192 }
1193
1194 fn test_noop(_: Allocator) void {}
1195
1196 fn test_teardown_noop() void {}
1197
1198 fn test_allocate_32_cc(allocator: Allocator) void {
1199 const bytes = allocator.alloc(u8, 32) catch unreachable;
1200 defer allocator.free(bytes);
1201 @memset(bytes, 0xcc);
1202 }
1203
1204 fn test_allocate_16_ee(allocator: Allocator) void {
1205 const bytes = allocator.alloc(u8, 16) catch unreachable;
1206 defer allocator.free(bytes);
1207 @memset(bytes, 0xee);
1208 }
1209
1210 fn test_allocate_8_aa(allocator: Allocator) void {
1211 const bytes = allocator.alloc(u8, 8) catch unreachable;
1212 defer allocator.free(bytes);
1213 @memset(bytes, 0xaa);
1214 }
1215
1216 fn test_allocate_16_dd(allocator: Allocator) void {
1217 const bytes = allocator.alloc(u8, 16) catch unreachable;
1218 defer allocator.free(bytes);
1219 @memset(bytes, 0xdd);
1220 }
1221
1222 fn test_allocate_64_cc(allocator: Allocator) void {
1223 const bytes = allocator.alloc(u8, 64) catch unreachable;
1224 defer allocator.free(bytes);
1225 @memset(bytes, 0xcc);
1226 }
1227
1228 const TestCallCounter = struct {
1229 var count: u32 = 0;
1230
1231 fn reset() void {
1232 count = 0;
1233 }
1234
1235 fn run(_: Allocator) void {
1236 count += 1;
1237 }
1238 };
1239
1240 const TestBackingWitness = struct {
1241 var sample: ?Allocator = null;
1242
1243 fn reset() void {
1244 sample = null;
1245 }
1246
1247 fn run(allocator: Allocator) void {
1248 sample = allocator;
1249 }
1250 };
1251
1252 const TestTeardownWitness = struct {
1253 var allocator: ?Allocator = null;
1254 var memory: ?[]u8 = null;
1255 var calls: usize = 0;
1256 var teardowns: usize = 0;
1257
1258 fn reset() void {
1259 allocator = null;
1260 memory = null;
1261 calls = 0;
1262 teardowns = 0;
1263 }
1264
1265 fn run(sample_allocator: Allocator) void {
1266 if (memory == null) {
1267 allocator = sample_allocator;
1268 memory = sample_allocator.alloc(u8, 32) catch @panic("bench teardown witness allocation failed");
1269 }
1270 calls += 1;
1271 }
1272
1273 fn teardown() void {
1274 allocator.?.free(memory.?);
1275 allocator = null;
1276 memory = null;
1277 teardowns += 1;
1278 }
1279 };
1280
1281 const TestPrepareWitness = struct {
1282 var prepares: usize = 0;
1283 var calls: usize = 0;
1284
1285 fn reset() void {
1286 prepares = 0;
1287 calls = 0;
1288 }
1289
1290 fn prepare(allocator: Allocator) void {
1291 const bytes = allocator.alloc(u8, 32) catch @panic("bench prepare witness allocation failed");
1292 defer allocator.free(bytes);
1293 @memset(bytes, 0xee);
1294 prepares += 1;
1295 }
1296
1297 fn run(_: Allocator) void {
1298 calls += 1;
1299 }
1300 };
1301
1302 const TestPrepareOwnerWitness = struct {
1303 var allocator: ?Allocator = null;
1304 var prepared_memory: ?[]u8 = null;
1305 var call_memory: ?[]u8 = null;
1306 var prepares: usize = 0;
1307 var calls: usize = 0;
1308 var teardowns: usize = 0;
1309
1310 fn reset() void {
1311 allocator = null;
1312 prepared_memory = null;
1313 call_memory = null;
1314 prepares = 0;
1315 calls = 0;
1316 teardowns = 0;
1317 }
1318
1319 fn prepare(owner_allocator: Allocator) void {
1320 release();
1321 allocator = owner_allocator;
1322 prepared_memory = owner_allocator.alloc(u8, 32) catch
1323 @panic("bench prepare owner allocation failed");
1324 @memset(prepared_memory.?, 0xcc);
1325 prepares += 1;
1326 }
1327
1328 fn run(sample_allocator: Allocator) void {
1329 const transient = sample_allocator.alloc(u8, 8) catch
1330 @panic("bench prepare owner sample allocation failed");
1331 defer sample_allocator.free(transient);
1332 call_memory = allocator.?.alloc(u8, 16) catch
1333 @panic("bench retained owner allocation failed");
1334 @memset(call_memory.?, 0xee);
1335 calls += 1;
1336 }
1337
1338 fn teardown() void {
1339 release();
1340 teardowns += 1;
1341 }
1342
1343 fn release() void {
1344 const owner_allocator = allocator orelse return;
1345 if (call_memory) |memory| owner_allocator.free(memory);
1346 if (prepared_memory) |memory| owner_allocator.free(memory);
1347 allocator = null;
1348 prepared_memory = null;
1349 call_memory = null;
1350 }
1351 };
1352
1353 const TestPrepareOwnerResizeWitness = struct {
1354 var allocator: ?Allocator = null;
1355 var memory: ?[]u8 = null;
1356
1357 fn reset() void {
1358 allocator = null;
1359 memory = null;
1360 }
1361
1362 fn prepare(owner_allocator: Allocator) void {
1363 release();
1364 allocator = owner_allocator;
1365 memory = owner_allocator.alloc(u8, 32) catch
1366 @panic("bench prepare owner resize allocation failed");
1367 }
1368
1369 fn run(sample_allocator: Allocator) void {
1370 const owner_allocator = allocator.?;
1371 const current = memory.?;
1372 if (!owner_allocator.resize(current, 48)) {
1373 @panic("bench prepare owner resize failed");
1374 }
1375 memory = current.ptr[0..48];
1376 const transient = sample_allocator.alloc(u8, 8) catch
1377 @panic("bench prepare owner resize sample allocation failed");
1378 sample_allocator.free(transient);
1379 }
1380
1381 fn teardown() void {
1382 release();
1383 }
1384
1385 fn release() void {
1386 const owner_allocator = allocator orelse return;
1387 owner_allocator.free(memory.?);
1388 allocator = null;
1389 memory = null;
1390 }
1391 };
1392
1393 test "default benchmark layout disables pointer validation bookkeeping" {
1394 try std.testing.expect(!(Config{}).layout_randomization.?.heap.pointer_validation);
1395 }
1396
1397 test "suite feeds workloads the production heap regardless of init allocator" {
1398 TestBackingWitness.reset();
1399 var s = Suite.init(std.testing.allocator, .{
1400 .min_time_ns = 0,
1401 .min_iterations = 1,
1402 .max_iterations = 1,
1403 .layout_randomization = null,
1404 });
1405 defer s.deinit();
1406
1407 try s.add("backing witness", TestBackingWitness.run, .{});
1408 _ = try s.runSilent();
1409
1410 const witness = TestBackingWitness.sample orelse return error.TestUnexpectedResult;
1411 const production = sampleBackingAllocator();
1412 try std.testing.expectEqual(production.ptr, witness.ptr);
1413 try std.testing.expectEqual(production.vtable, witness.vtable);
1414 try std.testing.expect(witness.vtable != std.testing.allocator.vtable);
1415 }
1416
1417 test "suite enables coz experiments from a nonempty environment value" {
1418 try std.testing.expect(experimentsEnabled("1"));
1419 try std.testing.expect(experimentsEnabled("true"));
1420 try std.testing.expect(!experimentsEnabled("0"));
1421 try std.testing.expect(!experimentsEnabled(""));
1422 try std.testing.expect(!experimentsEnabled(null));
1423 }
1424
1425 test "suite parses the minimum-time stretch from the environment value" {
1426 try std.testing.expectEqual(@as(?u64, null), try parseMinTime(null));
1427 try std.testing.expectEqual(@as(?u64, null), try parseMinTime(""));
1428 try std.testing.expectEqual(@as(?u64, 30_000_000_000), try parseMinTime("30000000000"));
1429 try std.testing.expectError(error.InvalidBenchMinTime, parseMinTime("30s"));
1430 }
1431
1432 test "suite parses the layout control from the environment value" {
1433 try std.testing.expectEqual(LayoutControl.inherit, try parseLayoutControl(null));
1434 try std.testing.expectEqual(LayoutControl.inherit, try parseLayoutControl(""));
1435 try std.testing.expectEqual(LayoutControl.disabled, try parseLayoutControl("off"));
1436 try std.testing.expectError(error.InvalidBenchLayout, parseLayoutControl("fixed"));
1437
1438 const def = Definition{
1439 .name = "layout control",
1440 .func = undefined,
1441 .config = .{},
1442 };
1443 const controlled = controlledDefinition(def, .{ .layout = .disabled });
1444 try std.testing.expect(controlled.config.layout_randomization == null);
1445 }
1446
1447 test "stretched definitions raise minimum time and iteration cap without shrinking" {
1448 const def = Definition{
1449 .name = "stretch",
1450 .func = undefined,
1451 .config = .{ .min_time_ns = 5_000_000, .max_iterations = 100 },
1452 };
1453
1454 const unchanged = stretchedDefinition(def, null);
1455 try std.testing.expectEqual(@as(u64, 5_000_000), unchanged.config.min_time_ns);
1456 try std.testing.expectEqual(@as(u32, 100), unchanged.config.max_iterations);
1457
1458 const stretched = stretchedDefinition(def, 30_000_000_000);
1459 try std.testing.expectEqual(@as(u64, 30_000_000_000), stretched.config.min_time_ns);
1460 try std.testing.expectEqual(stretched_iteration_limit, stretched.config.max_iterations);
1461
1462 const long = Definition{
1463 .name = "long",
1464 .func = undefined,
1465 .config = .{ .min_time_ns = 60_000_000_000, .max_iterations = 2_000_000 },
1466 };
1467 const kept = stretchedDefinition(long, 30_000_000_000);
1468 try std.testing.expectEqual(@as(u64, 60_000_000_000), kept.config.min_time_ns);
1469 try std.testing.expectEqual(@as(u32, 2_000_000), kept.config.max_iterations);
1470 }
1471
1472 test "adaptive sample batches stay inside the measurement window" {
1473 try std.testing.expectEqual(@as(u32, 1), adaptiveBatchSize(0, 0, std.time.ns_per_s, 1_000_000));
1474 try std.testing.expectEqual(@as(u32, 33), adaptiveBatchSize(1, 300_000, std.time.ns_per_s, 1_000_000));
1475 try std.testing.expectEqual(@as(u32, 5), adaptiveBatchSize(10, 20 * std.time.ns_per_ms, std.time.ns_per_s, 1_000_000));
1476 try std.testing.expectEqual(zero_average_batch_size, adaptiveBatchSize(1, 0, std.time.ns_per_s, 1_000_000));
1477 try std.testing.expectEqual(@as(u32, 1), adaptiveBatchSize(10, std.time.ns_per_s, std.time.ns_per_s, 1_000_000));
1478 try std.testing.expectEqual(@as(u32, 3), adaptiveBatchSize(97, 97_000, std.time.ns_per_s, 100));
1479 }
1480
1481 test "automatic eval growth is bounded and progressive" {
1482 try std.testing.expectEqual(@as(u32, 10), nextEvalCount(1, 0, 100, 1000));
1483 try std.testing.expectEqual(@as(u32, 100), nextEvalCount(10, 10, 100, 1000));
1484 try std.testing.expectEqual(@as(u32, 20), nextEvalCount(10, 60, 100, 1000));
1485 try std.testing.expectEqual(@as(u32, 25), nextEvalCount(20, 10, 100, 25));
1486
1487 var evals: u32 = 1;
1488 var steps: u32 = 1;
1489 while (evals < automatic_eval_limit) : (steps += 1) {
1490 evals = nextEvalCount(evals, 60, 100, automatic_eval_limit);
1491 }
1492 try std.testing.expectEqual(automatic_eval_limit, evals);
1493 try std.testing.expectEqual(max_calibration_steps, steps);
1494 try std.testing.expectEqual(max_calibration_steps * 2, max_calibration_batches);
1495 }
1496
1497 test "automatic calibration requires a confirming target observation" {
1498 var selection = AutomaticSelection{
1499 .policy = .{ .max_evals = 100 },
1500 .evals = 1,
1501 .clock_target_ns = 100,
1502 .target_ns = 100,
1503 .target_capped = false,
1504 .calibration_batches = 0,
1505 .calibration_confirmations = 0,
1506 .calibration_evals = 0,
1507 .calibration_ns = 0,
1508 .calibration_last_ns = 0,
1509 .calibration_target_reached = false,
1510 };
1511 try std.testing.expect(calibrationAction(selection, 150, null) == .confirm);
1512 const cold_action = calibrationAction(selection, 150, 60);
1513 try std.testing.expectEqual(@as(u32, 2), cold_action.grow);
1514 try std.testing.expect(calibrationAction(selection, 150, 120) == .reached);
1515
1516 selection.evals = selection.policy.max_evals;
1517 try std.testing.expect(calibrationAction(selection, 99, null) == .exhausted);
1518 }
1519
1520 test "automatic evaluation policy enforces absolute bounds" {
1521 try std.testing.expectError(
1522 error.InvalidEvaluationPolicy,
1523 validateAutomaticEvaluation(.{ .min_sample_time_ns = 0 }),
1524 );
1525 try std.testing.expectError(
1526 error.InvalidEvaluationPolicy,
1527 validateAutomaticEvaluation(.{
1528 .min_sample_time_ns = 2,
1529 .max_sample_time_ns = 1,
1530 }),
1531 );
1532 try std.testing.expectError(
1533 error.InvalidEvaluationPolicy,
1534 validateAutomaticEvaluation(.{
1535 .max_sample_time_ns = automatic_sample_time_limit_ns + 1,
1536 }),
1537 );
1538 try std.testing.expectError(
1539 error.InvalidEvaluationPolicy,
1540 validateAutomaticEvaluation(.{ .clock_resolution_multiple = 0 }),
1541 );
1542 try std.testing.expectError(
1543 error.InvalidEvaluationPolicy,
1544 validateAutomaticEvaluation(.{ .max_evals = automatic_eval_limit + 1 }),
1545 );
1546 }
1547
1548 test "suite basic run" {
1549 const alloc = std.testing.allocator;
1550 var s = Suite.init(alloc, .{ .min_time_ns = 1_000_000, .min_iterations = 3 });
1551 defer s.deinit();
1552
1553 try s.add("noop", test_noop, .{});
1554
1555 const results = try s.runSilent();
1556 try std.testing.expect(results.len == 1);
1557 try std.testing.expect(results[0].iterations >= 3);
1558 try std.testing.expect(results[0].evals > 0);
1559 try std.testing.expect(results[0].evaluation == .automatic);
1560 try std.testing.expect(results[0].clock_probe.minimum_delta_ns > 0);
1561 const selection = results[0].evaluation.automatic;
1562 try std.testing.expect(selection.target_ns >= selection.policy.min_sample_time_ns);
1563 try std.testing.expect(selection.target_ns <= selection.policy.max_sample_time_ns);
1564 try std.testing.expect(selection.calibration_batches > 0);
1565 try std.testing.expect(selection.calibration_batches <= max_calibration_batches);
1566 try std.testing.expect(selection.calibration_confirmations > 0);
1567 try std.testing.expect(selection.calibration_evals >= selection.evals);
1568 if (selection.calibration_target_reached) {
1569 try std.testing.expect(selection.calibration_last_ns >= selection.target_ns);
1570 }
1571 try std.testing.expect(selection.samples_below_target <= results[0].iterations);
1572 try std.testing.expect(std.mem.eql(u8, results[0].name, "noop"));
1573 }
1574
1575 test "build result preserves measured sample order" {
1576 const allocator = std.testing.allocator;
1577 var collected: CollectedSamples = .{};
1578 defer collected.deinit(allocator);
1579 try collected.samples.appendSlice(allocator, &.{ 30, 10, 20 });
1580 try collected.totals.appendSlice(allocator, &.{ 90, 30, 60 });
1581 collected.total_ns = 180;
1582 collected.iterations = 3;
1583 const result = try buildResult(
1584 allocator,
1585 .{ .name = "ordered", .func = test_noop, .config = .{} },
1586 &collected,
1587 .{ .fixed = 3 },
1588 .{
1589 .reads = 2,
1590 .positive_deltas = 1,
1591 .minimum_delta_ns = 1,
1592 .span_ns = 1,
1593 },
1594 .{},
1595 );
1596 defer allocator.free(result.samples);
1597 defer allocator.free(result.sample_total_ns);
1598 try std.testing.expectEqualSlices(u64, &.{ 30, 10, 20 }, result.samples);
1599 try std.testing.expectEqualSlices(u64, &.{ 90, 30, 60 }, result.sample_total_ns);
1600 try std.testing.expectEqual(@as(u64, 10), result.min_ns);
1601 try std.testing.expectEqual(@as(u64, 20), result.median_ns);
1602 try std.testing.expectEqual(@as(u64, 30), result.max_ns);
1603 }
1604
1605 test "suite honors minimum iterations after time target" {
1606 const alloc = std.testing.allocator;
1607 var s = Suite.init(alloc, .{
1608 .min_time_ns = 0,
1609 .min_iterations = 3,
1610 .max_iterations = 3,
1611 });
1612 defer s.deinit();
1613
1614 try s.add("noop", test_noop, .{});
1615
1616 const results = try s.runSilent();
1617 try std.testing.expectEqual(@as(u32, 3), results[0].iterations);
1618 }
1619
1620 test "suite applies per-row config overrides" {
1621 const alloc = std.testing.allocator;
1622 var s = Suite.init(alloc, .{
1623 .min_time_ns = 0,
1624 .min_iterations = 1,
1625 .max_iterations = 5,
1626 });
1627 defer s.deinit();
1628
1629 try s.add("default", test_noop, .{});
1630 try s.add("override", test_noop, .{
1631 .min_iterations = 3,
1632 .max_iterations = 3,
1633 .evaluation = Evaluation{ .fixed = 2 },
1634 });
1635
1636 const results = try s.runSilent();
1637 try std.testing.expectEqual(@as(usize, 2), results.len);
1638 try std.testing.expectEqual(@as(u32, 1), results[0].iterations);
1639 try std.testing.expectEqual(@as(u32, 3), results[1].iterations);
1640 try std.testing.expect(results[0].evaluation == .automatic);
1641 try std.testing.expectEqual(@as(u32, 2), results[1].evals);
1642 try std.testing.expect(results[1].evaluation == .fixed);
1643 }
1644
1645 test "suite filters benchmark definitions" {
1646 const alloc = std.testing.allocator;
1647 var s = Suite.init(alloc, .{
1648 .min_time_ns = 0,
1649 .min_iterations = 1,
1650 .max_iterations = 1,
1651 .layout_randomization = null,
1652 });
1653 defer s.deinit();
1654 s.setFilter("keep");
1655
1656 try s.add("drop row", test_noop, .{});
1657 try s.add("keep row", test_noop, .{});
1658
1659 const results = try s.runSilent();
1660 try std.testing.expectEqual(@as(usize, 1), results.len);
1661 try std.testing.expectEqualStrings("keep row", results[0].name);
1662 }
1663
1664 test "suite tears down retained sample storage before definition instrumentation" {
1665 TestTeardownWitness.reset();
1666 var s = Suite.init(std.testing.allocator, .{
1667 .min_time_ns = 0,
1668 .min_iterations = 1,
1669 .max_iterations = 1,
1670 .warmup_iterations = 1,
1671 .evaluation = .{ .fixed = 1 },
1672 });
1673 defer s.deinit();
1674
1675 try s.add("retained fixture", TestTeardownWitness.run, .{
1676 .teardown = TestTeardownWitness.teardown,
1677 });
1678 _ = try s.runSilent();
1679
1680 try std.testing.expectEqual(@as(usize, 2), TestTeardownWitness.calls);
1681 try std.testing.expectEqual(@as(usize, 1), TestTeardownWitness.teardowns);
1682 try std.testing.expect(TestTeardownWitness.memory == null);
1683 try std.testing.expect(TestTeardownWitness.allocator == null);
1684 }
1685
1686 test "suite preserves null allocation fields for unattributed prepared rows" {
1687 TestPrepareWitness.reset();
1688 var s = Suite.init(std.testing.allocator, .{
1689 .min_time_ns = 0,
1690 .min_iterations = 3,
1691 .max_iterations = 3,
1692 .warmup_iterations = 1,
1693 .evaluation = .{ .fixed = 1 },
1694 .layout_randomization = null,
1695 });
1696 defer s.deinit();
1697
1698 try s.add("prepared fixture", TestPrepareWitness.run, .{
1699 .prepare = TestPrepareWitness.prepare,
1700 });
1701 const results = try s.runSilent();
1702
1703 try std.testing.expectEqual(@as(usize, 4), TestPrepareWitness.prepares);
1704 try std.testing.expectEqual(@as(usize, 4), TestPrepareWitness.calls);
1705 try std.testing.expectEqual(AllocationAttribution.none, results[0].allocation_attribution);
1706 try std.testing.expectEqual(@as(?u64, null), results[0].alloc_count);
1707 try std.testing.expectEqual(@as(?u64, null), results[0].alloc_bytes);
1708 try std.testing.expectEqual(@as(?u64, null), results[0].alloc_count_per_eval);
1709 try std.testing.expectEqual(@as(?u64, null), results[0].alloc_bytes_per_eval);
1710
1711 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
1712 defer out.deinit();
1713 try s.writeJsonl(&out.writer, .{ .suite_name = "unattributed-suite" });
1714 const jsonl = out.written();
1715 try std.testing.expect(
1716 std.mem.indexOf(u8, jsonl, "\"allocation_attribution\":\"none\"") != null,
1717 );
1718 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"alloc_count\":null") != null);
1719 }
1720
1721 test "suite attributes prepare owner traffic outside sample timing" {
1722 TestPrepareOwnerWitness.reset();
1723 var s = Suite.init(std.testing.allocator, .{
1724 .min_time_ns = 0,
1725 .min_iterations = 3,
1726 .max_iterations = 3,
1727 .warmup_iterations = 1,
1728 .evaluation = .{ .fixed = 1 },
1729 .allocation_attribution = .prepare_owner,
1730 .layout_randomization = null,
1731 });
1732 defer s.deinit();
1733
1734 try s.add("prepare owner fixture", TestPrepareOwnerWitness.run, .{
1735 .prepare = TestPrepareOwnerWitness.prepare,
1736 .teardown = TestPrepareOwnerWitness.teardown,
1737 });
1738 const results = try s.runSilent();
1739
1740 try std.testing.expectEqual(@as(usize, 4), TestPrepareOwnerWitness.prepares);
1741 try std.testing.expectEqual(@as(usize, 4), TestPrepareOwnerWitness.calls);
1742 try std.testing.expectEqual(@as(usize, 2), TestPrepareOwnerWitness.teardowns);
1743 try std.testing.expect(TestPrepareOwnerWitness.allocator == null);
1744 try std.testing.expectEqual(
1745 AllocationAttribution.prepare_owner,
1746 results[0].allocation_attribution,
1747 );
1748 try std.testing.expectEqual(@as(?u64, 6), results[0].alloc_count);
1749 try std.testing.expectEqual(@as(?u64, 6), results[0].free_count);
1750 try std.testing.expectEqual(@as(?u64, 144), results[0].alloc_bytes);
1751 try std.testing.expectEqual(@as(?u64, 2), results[0].alloc_count_per_eval);
1752 try std.testing.expectEqual(@as(?u64, 2), results[0].free_count_per_eval);
1753 try std.testing.expectEqual(@as(?u64, 48), results[0].alloc_bytes_per_eval);
1754 }
1755
1756 test "suite attributes prepare owner resize traffic" {
1757 TestPrepareOwnerResizeWitness.reset();
1758 var backing: [512]u8 = undefined;
1759 var fixed = std.heap.FixedBufferAllocator.init(&backing);
1760 const result = try runOne(
1761 std.testing.allocator,
1762 fixed.allocator(),
1763 null,
1764 .{
1765 .name = "prepare owner resize",
1766 .func = TestPrepareOwnerResizeWitness.run,
1767 .config = .{
1768 .min_time_ns = 0,
1769 .min_iterations = 2,
1770 .max_iterations = 2,
1771 .warmup_iterations = 1,
1772 .evaluation = .{ .fixed = 1 },
1773 .prepare = TestPrepareOwnerResizeWitness.prepare,
1774 .teardown = TestPrepareOwnerResizeWitness.teardown,
1775 .allocation_attribution = .prepare_owner,
1776 .layout_randomization = null,
1777 },
1778 },
1779 try timing.probeClock(),
1780 );
1781 defer std.testing.allocator.free(result.samples);
1782 defer std.testing.allocator.free(result.sample_total_ns);
1783
1784 try std.testing.expect(TestPrepareOwnerResizeWitness.allocator == null);
1785 try std.testing.expectEqual(@as(?u64, 2), result.alloc_count);
1786 try std.testing.expectEqual(@as(?u64, 2), result.free_count);
1787 try std.testing.expectEqual(@as(?u64, 96), result.alloc_bytes);
1788 try std.testing.expectEqual(@as(?u64, 1), result.alloc_count_per_eval);
1789 try std.testing.expectEqual(@as(?u64, 1), result.free_count_per_eval);
1790 try std.testing.expectEqual(@as(?u64, 48), result.alloc_bytes_per_eval);
1791 }
1792
1793 test "suite rejects prepared batched evaluations" {
1794 var s = Suite.init(std.testing.allocator, .{
1795 .min_time_ns = 0,
1796 .min_iterations = 1,
1797 .max_iterations = 1,
1798 .evaluation = .{ .fixed = 2 },
1799 .layout_randomization = null,
1800 });
1801 defer s.deinit();
1802
1803 try s.add("invalid prepared fixture", test_noop, .{
1804 .prepare = test_noop,
1805 });
1806 try std.testing.expectError(error.InvalidPreparedEvaluation, s.runSilent());
1807 }
1808
1809 test "suite rejects incomplete prepare owner attribution" {
1810 var missing_teardown = Suite.init(std.testing.allocator, .{
1811 .min_time_ns = 0,
1812 .min_iterations = 1,
1813 .max_iterations = 1,
1814 .evaluation = .{ .fixed = 1 },
1815 .allocation_attribution = .prepare_owner,
1816 .layout_randomization = null,
1817 });
1818 defer missing_teardown.deinit();
1819 try missing_teardown.add("missing teardown", test_noop, .{
1820 .prepare = test_noop,
1821 });
1822 try std.testing.expectError(
1823 error.InvalidPrepareOwnerAttribution,
1824 missing_teardown.runSilent(),
1825 );
1826
1827 var missing_prepare = Suite.init(std.testing.allocator, .{
1828 .min_time_ns = 0,
1829 .min_iterations = 1,
1830 .max_iterations = 1,
1831 .evaluation = .{ .fixed = 1 },
1832 .allocation_attribution = .prepare_owner,
1833 .layout_randomization = null,
1834 });
1835 defer missing_prepare.deinit();
1836 try missing_prepare.add("missing prepare", test_noop, .{
1837 .teardown = test_teardown_noop,
1838 });
1839 try std.testing.expectError(
1840 error.InvalidPrepareOwnerAttribution,
1841 missing_prepare.runSilent(),
1842 );
1843 }
1844
1845 test "suite applies per-row layout randomization overrides" {
1846 const default_config: Config = .{
1847 .min_time_ns = 0,
1848 .min_iterations = 1,
1849 .max_iterations = 1,
1850 .layout_randomization = null,
1851 };
1852
1853 const enabled = applyConfigOverride(default_config, .{
1854 .layout_randomization = @as(?stabilizer.Config, .{
1855 .seed = 77,
1856 .heap = .{ .shuffle_slots = 3 },
1857 .code = .{ .enabled = false },
1858 }),
1859 });
1860 try std.testing.expectEqual(@as(u64, 77), enabled.layout_randomization.?.seed);
1861 try std.testing.expectEqual(@as(usize, 3), enabled.layout_randomization.?.heap.shuffle_slots);
1862
1863 const disabled = applyConfigOverride(.{}, .{
1864 .layout_randomization = @as(?stabilizer.Config, null),
1865 });
1866 try std.testing.expect(disabled.layout_randomization == null);
1867 }
1868
1869 test "suite tracks measured allocation counts" {
1870 const alloc = std.testing.allocator;
1871 var s = Suite.init(alloc, .{
1872 .min_time_ns = 0,
1873 .min_iterations = 2,
1874 .max_iterations = 2,
1875 .warmup_iterations = 1,
1876 .evaluation = .{ .fixed = 1 },
1877 .allocation_attribution = .sample_call,
1878 .layout_randomization = null,
1879 });
1880 defer s.deinit();
1881
1882 try s.add("alloc", test_allocate_32_cc, .{});
1883
1884 const results = try s.runSilent();
1885 try std.testing.expectEqual(@as(u32, 2), results[0].iterations);
1886 try std.testing.expectEqual(@as(u32, 1), results[0].evals);
1887 try std.testing.expectEqual(@as(u64, 2), results[0].alloc_count.?);
1888 try std.testing.expectEqual(@as(u64, 2), results[0].free_count.?);
1889 try std.testing.expectEqual(@as(u64, 64), results[0].alloc_bytes.?);
1890 try std.testing.expectEqual(@as(u64, 1), results[0].alloc_count_per_eval.?);
1891 try std.testing.expectEqual(@as(u64, 1), results[0].free_count_per_eval.?);
1892 try std.testing.expectEqual(@as(u64, 32), results[0].alloc_bytes_per_eval.?);
1893 }
1894
1895 test "suite batches evals per timed sample" {
1896 const alloc = std.testing.allocator;
1897 TestCallCounter.reset();
1898
1899 var s = Suite.init(alloc, .{
1900 .min_time_ns = 0,
1901 .min_iterations = 3,
1902 .max_iterations = 3,
1903 .evaluation = .{ .fixed = 4 },
1904 .warmup_iterations = 1,
1905 .layout_randomization = null,
1906 });
1907 defer s.deinit();
1908
1909 try s.add("batched", TestCallCounter.run, .{});
1910
1911 const results = try s.runSilent();
1912 try std.testing.expectEqual(@as(u32, 3), results[0].iterations);
1913 try std.testing.expectEqual(@as(u32, 4), results[0].evals);
1914 try std.testing.expectEqual(@as(u32, 16), TestCallCounter.count);
1915 try std.testing.expectEqual(@as(usize, 3), results[0].samples.len);
1916 try std.testing.expectEqual(@as(usize, 3), results[0].sample_total_ns.len);
1917 for (results[0].samples, results[0].sample_total_ns) |sample_ns, total_ns| {
1918 try std.testing.expect(total_ns >= sample_ns);
1919 }
1920
1921 var out = std.Io.Writer.Allocating.init(alloc);
1922 defer out.deinit();
1923 try s.writeJsonl(&out.writer, .{ .suite_name = "batched-suite" });
1924 const jsonl = out.written();
1925 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"evals\":4") != null);
1926 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"sample_total_ns\":[") != null);
1927 }
1928
1929 test "suite tracks allocation counts across batched evals" {
1930 const alloc = std.testing.allocator;
1931 var s = Suite.init(alloc, .{
1932 .min_time_ns = 0,
1933 .min_iterations = 2,
1934 .max_iterations = 2,
1935 .evaluation = .{ .fixed = 3 },
1936 .allocation_attribution = .sample_call,
1937 .layout_randomization = null,
1938 });
1939 defer s.deinit();
1940
1941 try s.add("alloc batched", test_allocate_16_ee, .{});
1942
1943 const results = try s.runSilent();
1944 try std.testing.expectEqual(@as(u32, 2), results[0].iterations);
1945 try std.testing.expectEqual(@as(u32, 3), results[0].evals);
1946 try std.testing.expectEqual(@as(u64, 6), results[0].alloc_count.?);
1947 try std.testing.expectEqual(@as(u64, 6), results[0].free_count.?);
1948 try std.testing.expectEqual(@as(u64, 96), results[0].alloc_bytes.?);
1949 try std.testing.expectEqual(@as(u64, 1), results[0].alloc_count_per_eval.?);
1950 try std.testing.expectEqual(@as(u64, 1), results[0].free_count_per_eval.?);
1951 try std.testing.expectEqual(@as(u64, 16), results[0].alloc_bytes_per_eval.?);
1952 }
1953
1954 test "suite rejects zero eval batches" {
1955 const alloc = std.testing.allocator;
1956 var s = Suite.init(alloc, .{
1957 .min_time_ns = 0,
1958 .min_iterations = 1,
1959 .max_iterations = 1,
1960 .evaluation = .{ .fixed = 0 },
1961 });
1962 defer s.deinit();
1963
1964 try s.add("noop", test_noop, .{});
1965
1966 try std.testing.expectError(error.InvalidEvalCount, s.runSilent());
1967 }
1968
1969 test "suite writes comparable JSONL results" {
1970 const alloc = std.testing.allocator;
1971 var s = Suite.init(alloc, .{
1972 .min_time_ns = 0,
1973 .min_iterations = 2,
1974 .max_iterations = 2,
1975 .warmup_iterations = 1,
1976 .allocation_attribution = .sample_call,
1977 .layout_randomization = null,
1978 });
1979 defer s.deinit();
1980
1981 try s.add("json row", test_allocate_8_aa, .{});
1982
1983 _ = try s.runSilent();
1984
1985 var out = std.Io.Writer.Allocating.init(alloc);
1986 defer out.deinit();
1987 try s.writeJsonl(&out.writer, .{
1988 .suite_name = "suite-json-test",
1989 .output_path = "suite.jsonl",
1990 .metadata = .{
1991 .git_sha = "abc123",
1992 .git_dirty = true,
1993 .optimize = "ReleaseFast",
1994 .target = "native",
1995 .zig_version = "0.16.0",
1996 .host_os = "linux",
1997 .host_arch = "x86_64",
1998 },
1999 });
2000
2001 const jsonl = out.written();
2002 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"protocol\":\"bench.suite/v1\"") != null);
2003 try std.testing.expect(
2004 std.mem.indexOf(u8, jsonl, "\"metadata\":{\"git_sha\":\"abc123\"") != null,
2005 );
2006 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"event\":\"bench_end\"") != null);
2007 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"id\":\"json row\"") != null);
2008 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"median_ns\":") != null);
2009 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"p95_ns\":") != null);
2010 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"clock\":{") != null);
2011 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"schema\":\"bench.clock/v1\"") != null);
2012 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"evaluation\":{") != null);
2013 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"mode\":\"automatic\"") != null);
2014 try std.testing.expect(
2015 std.mem.indexOf(u8, jsonl, "\"calibration_confirmations\":") != null,
2016 );
2017 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"sample_ns\":[") != null);
2018 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"sample_total_ns\":[") != null);
2019 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"agent_evidence\":") != null);
2020 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"code\":\"not_recorded\"") != null);
2021 try std.testing.expect(
2022 std.mem.indexOf(
2023 u8,
2024 jsonl,
2025 "\"allocation_attribution\":\"sample_call\"",
2026 ) != null,
2027 );
2028 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"alloc_count\":") != null);
2029 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"event\":\"run_end\"") != null);
2030 }
2031
2032 test "suite writes row evidence from effective layout config" {
2033 const alloc = std.testing.allocator;
2034 var s = Suite.init(alloc, .{
2035 .min_time_ns = 0,
2036 .min_iterations = 1,
2037 .max_iterations = 1,
2038 .layout_randomization = .{
2039 .seed = 77,
2040 .heap = .{ .shuffle_slots = 5 },
2041 .code = .{ .enabled = false },
2042 },
2043 });
2044 defer s.deinit();
2045
2046 try s.add("default evidence", test_noop, .{});
2047 try s.add(
2048 "override evidence",
2049 test_noop,
2050 .{ .layout_randomization = @as(?stabilizer.Config, null) },
2051 );
2052 _ = try s.runSilent();
2053
2054 var out = std.Io.Writer.Allocating.init(alloc);
2055 defer out.deinit();
2056 try s.writeJsonl(&out.writer, .{ .suite_name = "evidence-suite" });
2057
2058 const jsonl = out.written();
2059 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"agent_evidence\":") != null);
2060 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"stabilizer_config\":{") != null);
2061 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"pointer_validation\":true") != null);
2062 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"stabilizer_config\":null") != null);
2063 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"code\":\"not_randomized\"") != null);
2064 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"code\":\"not_recorded\"") != null);
2065 }
2066
2067 test "suite appends JSONL files" {
2068 const alloc = std.testing.allocator;
2069 var tmp = std.testing.tmpDir(.{});
2070 defer tmp.cleanup();
2071
2072 const root = try tmp.parent_dir.realPathFileAlloc(std.Options.debug_io, tmp.sub_path[0..], alloc);
2073 defer alloc.free(root);
2074 const path = try std.fs.path.join(alloc, &.{ root, "suite.jsonl" });
2075 defer alloc.free(path);
2076
2077 var s = Suite.init(alloc, .{
2078 .min_time_ns = 0,
2079 .min_iterations = 1,
2080 .max_iterations = 1,
2081 .layout_randomization = null,
2082 });
2083 defer s.deinit();
2084 try s.add("append row", test_noop, .{});
2085 _ = try s.runSilent();
2086
2087 try s.appendJsonlFile(path, .{ .suite_name = "append-suite" });
2088 try s.appendJsonlFile(path, .{ .suite_name = "append-suite" });
2089
2090 const jsonl = try sys.fs.readFileAlloc(alloc, path, 1024 * 1024);
2091 defer alloc.free(jsonl);
2092 var count: usize = 0;
2093 var rest = jsonl;
2094 while (std.mem.indexOf(u8, rest, "\"event\":\"bench_end\"")) |index| {
2095 count += 1;
2096 rest = rest[index + 1 ..];
2097 }
2098 try std.testing.expectEqual(@as(usize, 2), count);
2099 }
2100
2101 test "suite can leave environment JSONL to caller" {
2102 if (comptime !@hasDecl(std.process.Environ.Block, "view")) return error.SkipZigTest;
2103
2104 const alloc = std.testing.allocator;
2105 var tmp = std.testing.tmpDir(.{});
2106 defer tmp.cleanup();
2107
2108 const root = try tmp.parent_dir.realPathFileAlloc(std.Options.debug_io, tmp.sub_path[0..], alloc);
2109 defer alloc.free(root);
2110 const path = try std.fs.path.join(alloc, &.{ root, "suite.jsonl" });
2111 defer alloc.free(path);
2112
2113 var map = sys.env.Map.init(alloc);
2114 defer map.deinit();
2115 try map.put(jsonl_output_env, path);
2116
2117 const block = try map.createPosixBlock(alloc, .{});
2118 defer block.deinit(alloc);
2119
2120 const previous = sys.env.current();
2121 sys.env.installProcessEnvironment(.{ .block = block });
2122 defer sys.env.installProcessEnvironment(previous);
2123
2124 var s = Suite.init(alloc, .{
2125 .min_time_ns = 0,
2126 .min_iterations = 1,
2127 .max_iterations = 1,
2128 .layout_randomization = null,
2129 });
2130 defer s.deinit();
2131 try s.add("manual row", test_noop, .{});
2132
2133 try s.runWithOptions(.{
2134 .write_stdout = false,
2135 .write_env_jsonl = false,
2136 });
2137 try std.testing.expectError(error.FileNotFound, sys.fs.readFileAlloc(alloc, path, 1024 * 1024));
2138
2139 try s.appendJsonlFile(path, .{ .suite_name = "manual-suite" });
2140 const jsonl = try sys.fs.readFileAlloc(alloc, path, 1024 * 1024);
2141 defer alloc.free(jsonl);
2142 var count: usize = 0;
2143 var rest = jsonl;
2144 while (std.mem.indexOf(u8, rest, "\"event\":\"bench_end\"")) |index| {
2145 count += 1;
2146 rest = rest[index + 1 ..];
2147 }
2148 try std.testing.expectEqual(@as(usize, 1), count);
2149 try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"suite\":\"manual-suite\"") != null);
2150 }
2151
2152 test "suite reads Coz artifact environment paths" {
2153 if (comptime !@hasDecl(std.process.Environ.Block, "view")) return error.SkipZigTest;
2154
2155 const alloc = std.testing.allocator;
2156 var map = sys.env.Map.init(alloc);
2157 defer map.deinit();
2158 try map.put(coz_output_env, "out.coz.jsonl");
2159 try map.put(coz_analysis_env, "out.coz.analysis.json");
2160
2161 const block = try map.createPosixBlock(alloc, .{});
2162 defer block.deinit(alloc);
2163
2164 const previous = sys.env.current();
2165 sys.env.installProcessEnvironment(.{ .block = block });
2166 defer sys.env.installProcessEnvironment(previous);
2167
2168 var env = try CozEnv.read(alloc);
2169 defer env.deinit(alloc);
2170 try std.testing.expectEqualStrings("out.coz.jsonl", env.output_path.?);
2171 try std.testing.expectEqualStrings("out.coz.analysis.json", env.analysis_path.?);
2172 }
2173
2174 test "suite reads Tracy artifact environment paths" {
2175 if (comptime !@hasDecl(std.process.Environ.Block, "view")) return error.SkipZigTest;
2176
2177 const alloc = std.testing.allocator;
2178 var map = sys.env.Map.init(alloc);
2179 defer map.deinit();
2180 try map.put(tracy_output_env, "out.tracy.jsonl");
2181 try map.put(tracy_summary_env, "out.tracy.summary.jsonl");
2182
2183 const block = try map.createPosixBlock(alloc, .{});
2184 defer block.deinit(alloc);
2185
2186 const previous = sys.env.current();
2187 sys.env.installProcessEnvironment(.{ .block = block });
2188 defer sys.env.installProcessEnvironment(previous);
2189
2190 var env = try TracyEnv.read(alloc);
2191 defer env.deinit(alloc);
2192 try std.testing.expectEqualStrings("out.tracy.jsonl", env.output_path.?);
2193 try std.testing.expectEqualStrings("out.tracy.summary.jsonl", env.summary_path.?);
2194 }
2195
2196 test "suite rejects a Tracy summary without an event stream" {
2197 if (comptime !@hasDecl(std.process.Environ.Block, "view")) return error.SkipZigTest;
2198
2199 const alloc = std.testing.allocator;
2200 var map = sys.env.Map.init(alloc);
2201 defer map.deinit();
2202 try map.put(tracy_summary_env, "out.tracy.summary.jsonl");
2203 const block = try map.createPosixBlock(alloc, .{});
2204 defer block.deinit(alloc);
2205 const previous = sys.env.current();
2206 sys.env.installProcessEnvironment(.{ .block = block });
2207 defer sys.env.installProcessEnvironment(previous);
2208
2209 try std.testing.expectError(error.InvalidTracyEnvironment, TracyEnv.read(alloc));
2210 }
2211
2212 test "suite owns enabled Tracy capture lifecycle" {
2213 if (comptime !@hasDecl(std.process.Environ.Block, "view")) return error.SkipZigTest;
2214 if (!tracy.enabled) return error.SkipZigTest;
2215
2216 const alloc = std.testing.allocator;
2217 var tmp = std.testing.tmpDir(.{});
2218 defer tmp.cleanup();
2219 const root = try tmp.parent_dir.realPathFileAlloc(std.Options.debug_io, tmp.sub_path[0..], alloc);
2220 defer alloc.free(root);
2221 const trace_path = try std.fs.path.join(alloc, &.{ root, "bench.tracy.jsonl" });
2222 defer alloc.free(trace_path);
2223 const summary_path = try std.fs.path.join(alloc, &.{ root, "bench.tracy.summary.jsonl" });
2224 defer alloc.free(summary_path);
2225
2226 var map = sys.env.Map.init(alloc);
2227 defer map.deinit();
2228 try map.put(tracy_output_env, trace_path);
2229 try map.put(tracy_summary_env, summary_path);
2230 const block = try map.createPosixBlock(alloc, .{});
2231 defer block.deinit(alloc);
2232 const previous = sys.env.current();
2233 sys.env.installProcessEnvironment(.{ .block = block });
2234 defer sys.env.installProcessEnvironment(previous);
2235
2236 var suite = Suite.init(alloc, .{
2237 .suite_name = "tracy-lifecycle",
2238 .min_time_ns = 0,
2239 .min_iterations = 1,
2240 .max_iterations = 1,
2241 .layout_randomization = null,
2242 });
2243 defer suite.deinit();
2244 try suite.add("capture", test_noop, .{});
2245 try suite.runWithOptions(.{ .write_stdout = false, .write_env_jsonl = false });
2246
2247 const trace_jsonl = try sys.fs.readFileAlloc(alloc, trace_path, 1024 * 1024);
2248 defer alloc.free(trace_jsonl);
2249 const summary_jsonl = try sys.fs.readFileAlloc(alloc, summary_path, 1024 * 1024);
2250 defer alloc.free(summary_jsonl);
2251 try std.testing.expect(std.mem.indexOf(u8, trace_jsonl, "\"kind\":\"start\"") != null);
2252 try std.testing.expect(std.mem.indexOf(u8, trace_jsonl, "bench.suite.run") != null);
2253 try std.testing.expect(std.mem.indexOf(u8, trace_jsonl, "\"kind\":\"plot.config\"") != null);
2254 try std.testing.expect(std.mem.indexOf(u8, trace_jsonl, "\"plot_unit\":\"count\"") != null);
2255 try std.testing.expect(
2256 std.mem.indexOf(u8, trace_jsonl, "\"plot_unit\":\"nanoseconds\"") != null,
2257 );
2258 try std.testing.expect(std.mem.indexOf(u8, summary_jsonl, "\"kind\":\"summary\"") != null);
2259 try std.testing.expect(std.mem.indexOf(u8, summary_jsonl, "\"kind\":\"zone\"") != null);
2260 try std.testing.expect(
2261 std.mem.indexOf(u8, summary_jsonl, "\"plot_configurations\":3") != null,
2262 );
2263 try std.testing.expect(std.mem.indexOf(u8, summary_jsonl, "\"unit\":\"count\"") != null);
2264 try std.testing.expect(
2265 std.mem.indexOf(u8, summary_jsonl, "\"unit\":\"nanoseconds\"") != null,
2266 );
2267 }
2268
2269 test "suite writes scoped allocation memtrace events" {
2270 if (comptime !@hasDecl(std.process.Environ.Block, "view")) return error.SkipZigTest;
2271
2272 const alloc = std.testing.allocator;
2273 var tmp = std.testing.tmpDir(.{});
2274 defer tmp.cleanup();
2275
2276 const root = try tmp.parent_dir.realPathFileAlloc(std.Options.debug_io, tmp.sub_path[0..], alloc);
2277 defer alloc.free(root);
2278 const path = try std.fs.path.join(alloc, &.{ root, "scoped-allocations.jsonl" });
2279 defer alloc.free(path);
2280
2281 var map = sys.env.Map.init(alloc);
2282 defer map.deinit();
2283 try map.put(scoped_allocations_env, path);
2284
2285 const block = try map.createPosixBlock(alloc, .{});
2286 defer block.deinit(alloc);
2287
2288 const previous = sys.env.current();
2289 sys.env.installProcessEnvironment(.{ .block = block });
2290 defer sys.env.installProcessEnvironment(previous);
2291
2292 var s = Suite.init(alloc, .{
2293 .min_time_ns = 0,
2294 .min_iterations = 1,
2295 .max_iterations = 1,
2296 .allocation_attribution = .sample_call,
2297 .evaluation = .{ .fixed = 1 },
2298 .layout_randomization = null,
2299 });
2300 defer s.deinit();
2301
2302 try s.add("scoped row", test_allocate_32_cc, .{});
2303
2304 _ = try s.runSilent();
2305
2306 const events = try sys.fs.readFileAlloc(alloc, path, 1024 * 1024);
2307 defer alloc.free(events);
2308 try std.testing.expect(std.mem.indexOf(u8, events, "\"kind\":\"alloc\"") != null);
2309 try std.testing.expect(std.mem.indexOf(u8, events, "\"scope\":\"root/scoped row/sample\"") != null);
2310 }
2311
2312 test "suite tracks allocations through layout randomization" {
2313 const alloc = std.testing.allocator;
2314 var s = Suite.init(alloc, .{
2315 .min_time_ns = 0,
2316 .min_iterations = 3,
2317 .max_iterations = 3,
2318 .evaluation = .{ .fixed = 1 },
2319 .allocation_attribution = .sample_call,
2320 .layout_randomization = .{
2321 .seed = 123,
2322 .heap = .{
2323 .shuffle_slots = 4,
2324 .max_shuffled_size = 1024,
2325 },
2326 .code = .{ .enabled = false },
2327 },
2328 });
2329 defer s.deinit();
2330
2331 try s.add("alloc", test_allocate_16_dd, .{});
2332
2333 const results = try s.runSilent();
2334 try std.testing.expectEqual(@as(u32, 3), results[0].iterations);
2335 try std.testing.expectEqual(@as(u32, 1), results[0].evals);
2336 try std.testing.expectEqual(@as(u64, 3), results[0].alloc_count.?);
2337 try std.testing.expectEqual(@as(u64, 3), results[0].free_count.?);
2338 try std.testing.expectEqual(@as(u64, 48), results[0].alloc_bytes.?);
2339 }
2340
2341 test "suite runs definitions through layout randomized allocator" {
2342 const alloc = std.testing.allocator;
2343 var s = Suite.init(alloc, .{
2344 .min_time_ns = 0,
2345 .min_iterations = 4,
2346 .max_iterations = 4,
2347 });
2348 defer s.deinit();
2349
2350 try s.add("alloc", test_allocate_64_cc, .{});
2351
2352 const results = try s.runSilent();
2353 try std.testing.expectEqual(@as(usize, 1), results.len);
2354 try std.testing.expectEqual(@as(u32, 4), results[0].iterations);
2355 }