lib/bench/src/compare/test.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const compare = @import("root.zig");
3 const engine = @import("engine.zig");
4
5 const Allocator = std.mem.Allocator;
6 const Storage = compare.Storage;
7 const ExecutionObservation = compare.ExecutionObservation;
8 const ExecutionStats = compare.ExecutionStats;
9 const Classification = compare.Classification;
10 const EffectInterval = compare.EffectInterval;
11 const default_bootstrap_iterations = compare.default_bootstrap_iterations;
12 const analyzeExecutions = compare.analyzeExecutions;
13 const summarizeExecutions = compare.summarizeExecutions;
14 const parseArgs = compare.parseArgs;
15 const SummaryComparison = compare.SummaryComparison;
16 const duplicateObjectString = compare.duplicateObjectString;
17 const objectNumberOptional = compare.objectNumberOptional;
18 const pretty_json = @import("pretty").json;
19 const classifyChange = engine.classifyChange;
20 const loadRows = engine.loadRows;
21 const deinitRows = engine.deinitRows;
22 const compareRows = engine.compareRows;
23
24 const comparison_baseline = [_]ExecutionObservation{
25 .{ .mean = 100, .median = 100 },
26 .{ .mean = 105, .median = 105 },
27 .{ .mean = 95, .median = 95 },
28 .{ .mean = 102, .median = 102 },
29 .{ .mean = 98, .median = 98 },
30 .{ .mean = 103, .median = 103 },
31 .{ .mean = 97, .median = 97 },
32 .{ .mean = 101, .median = 101 },
33 };
34
35 const comparison_candidate = [_]ExecutionObservation{
36 .{ .mean = 97, .median = 97 },
37 .{ .mean = 99, .median = 99 },
38 .{ .mean = 101, .median = 101 },
39 .{ .mean = 96, .median = 96 },
40 .{ .mean = 100, .median = 100 },
41 .{ .mean = 98, .median = 98 },
42 .{ .mean = 102, .median = 102 },
43 .{ .mean = 95, .median = 95 },
44 };
45
46 test "comparison storage rejects every input boundary before mutation" {
47 comptime {
48 @stardustClaim(
49 @import("alloc_phase").capacity.witness(@import("./root.zig").ComparisonStorage, "bench_comparison_boundaries"),
50 null,
51 null,
52 null,
53 null,
54 null,
55 null,
56 );
57 }
58
59 var storage = try Storage.init(std.testing.allocator, .{
60 .max_samples_per_series = 3,
61 .max_bootstrap_iterations = 5,
62 });
63 defer storage.deinit(std.testing.allocator);
64 storage.activate();
65 @memset(storage.bytes, 42);
66
67 try std.testing.expectError(
68 error.EmptyExecutionSet,
69 analyzeExecutions(&storage, &.{}, &.{.{ .mean = 1, .median = 1 }}, .{}),
70 );
71 try std.testing.expectError(
72 error.EmptyExecutionSet,
73 analyzeExecutions(&storage, &.{.{ .mean = 1, .median = 1 }}, &.{}, .{}),
74 );
75 try std.testing.expectError(
76 error.InvalidBootstrapIterations,
77 analyzeExecutions(
78 &storage,
79 &.{.{ .mean = 1, .median = 1 }},
80 &.{.{ .mean = 1, .median = 1 }},
81 .{ .bootstrap_iterations = 0 },
82 ),
83 );
84 try std.testing.expectError(
85 error.BaseSampleCapacityExceeded,
86 analyzeExecutions(
87 &storage,
88 &.{
89 .{ .mean = 1, .median = 1 },
90 .{ .mean = 2, .median = 2 },
91 .{ .mean = 3, .median = 3 },
92 .{ .mean = 4, .median = 4 },
93 },
94 &.{.{ .mean = 1, .median = 1 }},
95 .{ .min_runs = 1, .bootstrap_iterations = 5 },
96 ),
97 );
98 try std.testing.expectError(
99 error.CandidateSampleCapacityExceeded,
100 analyzeExecutions(
101 &storage,
102 &.{.{ .mean = 1, .median = 1 }},
103 &.{
104 .{ .mean = 1, .median = 1 },
105 .{ .mean = 2, .median = 2 },
106 .{ .mean = 3, .median = 3 },
107 .{ .mean = 4, .median = 4 },
108 },
109 .{ .min_runs = 1, .bootstrap_iterations = 5 },
110 ),
111 );
112 try std.testing.expectError(
113 error.BootstrapIterationCapacityExceeded,
114 analyzeExecutions(
115 &storage,
116 &.{.{ .mean = 1, .median = 1 }},
117 &.{.{ .mean = 1, .median = 1 }},
118 .{ .min_runs = 1, .bootstrap_iterations = 6 },
119 ),
120 );
121 try std.testing.expectError(
122 error.SampleCapacityExceeded,
123 summarizeExecutions(&storage, &.{
124 .{ .mean = 1, .median = 1 },
125 .{ .mean = 2, .median = 2 },
126 .{ .mean = 3, .median = 3 },
127 .{ .mean = 4, .median = 4 },
128 }),
129 );
130 for (storage.bytes) |byte| try std.testing.expectEqual(@as(u8, 42), byte);
131 _ = try storage.acquireF64(1, 1, 1);
132 try std.testing.expectError(
133 error.ComparisonStorageInUse,
134 analyzeExecutions(
135 &storage,
136 &.{.{ .mean = 1, .median = 1 }},
137 &.{.{ .mean = 1, .median = 1 }},
138 .{ .min_runs = 1, .bootstrap_iterations = 1 },
139 ),
140 );
141 for (storage.bytes) |byte| try std.testing.expectEqual(@as(u8, 42), byte);
142 storage.reset();
143 _ = try analyzeExecutions(
144 &storage,
145 &.{.{ .mean = 1, .median = 1 }},
146 &.{.{ .mean = 1, .median = 1 }},
147 .{ .min_runs = 1, .bootstrap_iterations = 5 },
148 );
149 try std.testing.expect(!storage.status().in_use);
150 }
151
152 test "comparison storage preserves execution output and reusable state" {
153 comptime {
154 @stardustClaim(
155 @import("alloc_phase").capacity.witness(@import("./root.zig").ComparisonStorage, "bench_comparison_reuse"),
156 null,
157 null,
158 null,
159 null,
160 null,
161 null,
162 );
163 }
164
165 var storage = try Storage.init(std.testing.allocator, .{
166 .max_samples_per_series = comparison_baseline.len,
167 .max_bootstrap_iterations = default_bootstrap_iterations,
168 });
169 defer storage.deinit(std.testing.allocator);
170 storage.activate();
171 const first = try analyzeExecutions(
172 &storage,
173 &comparison_baseline,
174 &comparison_candidate,
175 .{ .min_runs = comparison_baseline.len },
176 );
177 const second = try analyzeExecutions(
178 &storage,
179 &comparison_baseline,
180 &comparison_candidate,
181 .{ .min_runs = comparison_baseline.len },
182 );
183
184 try std.testing.expectEqual(first, second);
185 try std.testing.expectEqual(Classification.uncertain, first.classification);
186 try std.testing.expectEqual(ExecutionStats{
187 .runs = 8,
188 .mean = 100.125,
189 .median = 100.5,
190 .min = 95,
191 .max = 105,
192 }, first.baseline);
193 try std.testing.expectEqual(ExecutionStats{
194 .runs = 8,
195 .mean = 98.5,
196 .median = 98.5,
197 .min = 95,
198 .max = 102,
199 }, first.candidate);
200 try std.testing.expectEqual(@as(f64, 0.9800995024875622), first.ratio.?);
201 try std.testing.expectEqual(@as(f64, -5.3398058252427205), first.effect.?.low_percent);
202 try std.testing.expectEqual(@as(f64, 2.040816326530617), first.effect.?.high_percent);
203 try std.testing.expectEqual(default_bootstrap_iterations, first.bootstrap_iterations);
204 try std.testing.expectEqualStrings("execution_run_effect_interval", first.evidence);
205 try std.testing.expect(!storage.status().in_use);
206 }
207
208 test "activated execution comparison performs no backing allocation" {
209 comptime {
210 @stardustClaim(
211 @import("alloc_phase").capacity.witness(@import("./root.zig").ComparisonStorage, "bench_comparison_sealed_f64_transitive_risk"),
212 null,
213 null,
214 null,
215 null,
216 null,
217 null,
218 );
219 }
220 comptime {
221 @stardustClaim(
222 @import("alloc_phase").capacity.witness(@import("./root.zig").ComparisonStorage, "bench_comparison_sealed_f64_foreign_risk"),
223 null,
224 null,
225 null,
226 null,
227 null,
228 null,
229 );
230 }
231
232 var counting = std.testing.FailingAllocator.init(std.testing.allocator, .{});
233 var storage = try Storage.init(counting.allocator(), .{
234 .max_samples_per_series = comparison_baseline.len,
235 .max_bootstrap_iterations = default_bootstrap_iterations,
236 });
237 defer storage.deinit(counting.allocator());
238 storage.activate();
239 const allocations = counting.alloc_index;
240 const bytes = counting.allocated_bytes;
241
242 const result = try analyzeExecutions(
243 &storage,
244 &comparison_baseline,
245 &comparison_candidate,
246 .{ .min_runs = comparison_baseline.len },
247 );
248
249 try std.testing.expectEqual(Classification.uncertain, result.classification);
250 try std.testing.expectEqual(@as(usize, 1), allocations);
251 try std.testing.expectEqual(@as(usize, 8_096), bytes);
252 try std.testing.expectEqual(allocations, counting.alloc_index);
253 try std.testing.expectEqual(bytes, counting.allocated_bytes);
254 }
255
256 const TestRow = struct {
257 key: []u8,
258 summary: []u8,
259 name: []u8,
260 mean: f64,
261 median: f64,
262 min: f64,
263 max: f64,
264
265 pub fn deinit(self: TestRow, allocator: Allocator) void {
266 allocator.free(self.key);
267 allocator.free(self.summary);
268 allocator.free(self.name);
269 }
270 };
271
272 const TestSpec = struct {
273 pub const Row: type = TestRow;
274 pub const benchmark_name = "bench.test";
275 pub const protocol_name = "bench.test_compare/v1";
276
277 pub fn parseRow(allocator: Allocator, object: std.json.ObjectMap) !TestRow {
278 const summary = try duplicateObjectString(allocator, object, "summary");
279 errdefer allocator.free(summary);
280 const name = try duplicateObjectString(allocator, object, "name");
281 errdefer allocator.free(name);
282 const key = try std.fmt.allocPrint(allocator, "{s}|{s}", .{ summary, name });
283 return .{
284 .key = key,
285 .summary = summary,
286 .name = name,
287 .mean = objectNumberOptional(object, "mean") orelse return error.InvalidSummary,
288 .median = objectNumberOptional(object, "median") orelse
289 return error.InvalidSummary,
290 .min = objectNumberOptional(object, "min") orelse return error.InvalidSummary,
291 .max = objectNumberOptional(object, "max") orelse return error.InvalidSummary,
292 };
293 }
294
295 pub fn writeIdentity(_: pretty_json.Object, _: TestRow) !void {}
296 };
297
298 const TestEngine = SummaryComparison(TestSpec);
299 const test_run_start =
300 "{\"kind\":\"run\",\"benchmark\":\"bench.test\",\"event\":\"start\"}\n";
301 const test_run_end =
302 "{\"kind\":\"run\",\"benchmark\":\"bench.test\",\"event\":\"end\"}\n";
303 const test_run_invalid =
304 "{\"kind\":\"run\",\"benchmark\":\"bench.test\",\"event\":\"pause\"}\n";
305 const test_summary_10 =
306 "{\"kind\":\"summary\",\"benchmark\":\"bench.test\"," ++
307 "\"summary\":\"phase\",\"name\":\"total\",\"mean\":10," ++
308 "\"median\":10,\"min\":10,\"max\":10}\n";
309 const test_summary_11 =
310 "{\"kind\":\"summary\",\"benchmark\":\"bench.test\"," ++
311 "\"summary\":\"phase\",\"name\":\"total\",\"mean\":11," ++
312 "\"median\":11,\"min\":11,\"max\":11}\n";
313 const test_summary_parse_20 =
314 "{\"kind\":\"summary\",\"benchmark\":\"bench.test\"," ++
315 "\"summary\":\"phase\",\"name\":\"parse\",\"mean\":20," ++
316 "\"median\":20,\"min\":20,\"max\":20}\n";
317
318 test "parseArgs accepts material threshold execution minimum and bootstrap limit" {
319 const options = try parseArgs(&.{
320 "base.jsonl",
321 "candidate.jsonl",
322 "--threshold",
323 "1.2",
324 "--min-runs",
325 "12",
326 "--bootstrap-iterations",
327 "500",
328 });
329 try std.testing.expectEqual(@as(f64, 1.2), options.threshold);
330 try std.testing.expectEqual(@as(u32, 12), options.min_runs);
331 try std.testing.expectEqual(@as(u32, 500), options.bootstrap_iterations);
332 }
333
334 test "parseArgs rejects invalid comparison evidence options" {
335 try std.testing.expectError(error.InvalidArguments, parseArgs(&.{}));
336 try std.testing.expectError(
337 error.InvalidArguments,
338 parseArgs(&.{ "base", "candidate", "--threshold", "nan" }),
339 );
340 try std.testing.expectError(
341 error.InvalidArguments,
342 parseArgs(&.{ "base", "candidate", "--min-runs", "0" }),
343 );
344 try std.testing.expectError(
345 error.InvalidArguments,
346 parseArgs(&.{ "base", "candidate", "--bootstrap-iterations", "0" }),
347 );
348 try std.testing.expectError(
349 error.InvalidArguments,
350 parseArgs(&.{ "base", "candidate", "--noise-threshold", "0.2" }),
351 );
352 }
353
354 test "effect intervals classify supported, equivalent, and uncertain changes" {
355 try std.testing.expectEqual(
356 Classification.insufficient,
357 classifyChange(.{ .low_percent = 20, .high_percent = 20 }, false, 1.05),
358 );
359 try std.testing.expectEqual(
360 Classification.regression,
361 classifyChange(.{ .low_percent = 5, .high_percent = 8 }, true, 1.05),
362 );
363 try std.testing.expectEqual(
364 Classification.improvement,
365 classifyChange(.{ .low_percent = -8, .high_percent = -5 }, true, 1.05),
366 );
367 try std.testing.expectEqual(
368 Classification.unchanged,
369 classifyChange(.{ .low_percent = -2, .high_percent = 2 }, true, 1.05),
370 );
371 try std.testing.expectEqual(
372 Classification.uncertain,
373 classifyChange(.{ .low_percent = 2, .high_percent = 8 }, true, 1.05),
374 );
375 try std.testing.expectEqual(
376 Classification.unchanged,
377 classifyChange(.{ .low_percent = 0, .high_percent = 0 }, true, 1.0),
378 );
379 }
380
381 test "repeated run summaries produce supported effect intervals" {
382 var baseline = std.Io.Writer.Allocating.init(std.testing.allocator);
383 defer baseline.deinit();
384 var candidate = std.Io.Writer.Allocating.init(std.testing.allocator);
385 defer candidate.deinit();
386 try writeTestRuns(&baseline.writer, 10, 100);
387 try writeTestRuns(&candidate.writer, 10, 120);
388 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
389 defer out.deinit();
390 const counts = try TestEngine.compareSources(
391 std.testing.allocator,
392 &out.writer,
393 .{
394 .baseline_path = "base",
395 .candidate_path = "candidate",
396 .bootstrap_iterations = 500,
397 },
398 baseline.written(),
399 candidate.written(),
400 );
401 try std.testing.expectEqual(@as(u64, 1), counts.regressions);
402 try std.testing.expect(std.mem.indexOf(
403 u8,
404 out.written(),
405 "\"evidence\":\"execution_run_effect_interval\"",
406 ) != null);
407 try std.testing.expect(std.mem.indexOf(u8, out.written(), "\"baseline_runs\":10") != null);
408 try std.testing.expect(std.mem.indexOf(u8, out.written(), "\"effect_low_percent\":20") != null);
409 try std.testing.expect(std.mem.indexOf(u8, out.written(), "\"bootstrap_iterations\":500") != null);
410 try std.testing.expect(std.mem.indexOf(u8, out.written(), "\"effect_bootstrap_iterations\":500") != null);
411 }
412
413 test "summary comparison reuses one exact region across rows" {
414 comptime {
415 @stardustClaim(
416 @import("alloc_phase").capacity.witness(@import("./root.zig").ComparisonStorage, "bench_comparison_rows"),
417 null,
418 null,
419 null,
420 null,
421 null,
422 null,
423 );
424 }
425
426 var source = std.Io.Writer.Allocating.init(std.testing.allocator);
427 defer source.deinit();
428 for (0..10) |_| {
429 try source.writer.writeAll(
430 test_run_start ++ test_summary_10 ++ test_summary_parse_20 ++ test_run_end,
431 );
432 }
433 var baseline = try loadRows(TestSpec, std.testing.allocator, source.written());
434 defer deinitRows(TestRow, std.testing.allocator, &baseline);
435 var candidate = try loadRows(TestSpec, std.testing.allocator, source.written());
436 defer deinitRows(TestRow, std.testing.allocator, &candidate);
437 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
438 defer out.deinit();
439 var counting = std.testing.FailingAllocator.init(std.testing.allocator, .{});
440
441 const counts = try compareRows(
442 TestSpec,
443 counting.allocator(),
444 &out.writer,
445 .{
446 .baseline_path = "base",
447 .candidate_path = "candidate",
448 .bootstrap_iterations = 500,
449 },
450 &baseline,
451 &candidate,
452 );
453
454 try std.testing.expectEqual(@as(u64, 2), counts.compared);
455 try std.testing.expectEqual(@as(usize, 3), counting.alloc_index);
456 try std.testing.expectEqual(@as(usize, 4_152), counting.allocated_bytes);
457 try std.testing.expect(std.mem.indexOf(u8, out.written(), "\"bootstrap_iterations\":500") != null);
458 }
459
460 test "single run threshold crossings are insufficient" {
461 var baseline = std.Io.Writer.Allocating.init(std.testing.allocator);
462 defer baseline.deinit();
463 var candidate = std.Io.Writer.Allocating.init(std.testing.allocator);
464 defer candidate.deinit();
465 try writeTestRuns(&baseline.writer, 1, 100);
466 try writeTestRuns(&candidate.writer, 1, 120);
467 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
468 defer out.deinit();
469 const counts = try TestEngine.compareSources(
470 std.testing.allocator,
471 &out.writer,
472 .{ .baseline_path = "base", .candidate_path = "candidate" },
473 baseline.written(),
474 candidate.written(),
475 );
476 try std.testing.expectEqual(@as(u64, 1), counts.insufficient);
477 try std.testing.expect(std.mem.indexOf(
478 u8,
479 out.written(),
480 "\"classification\":\"insufficient\"",
481 ) != null);
482 }
483
484 test "duplicate summaries inside one run are rejected" {
485 const source = test_run_start ++ test_summary_10 ++ test_summary_11 ++ test_run_end;
486 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
487 defer out.deinit();
488 try std.testing.expectError(
489 error.DuplicateSummaryInRun,
490 TestEngine.compareSources(
491 std.testing.allocator,
492 &out.writer,
493 .{ .baseline_path = "base", .candidate_path = "candidate" },
494 source,
495 source,
496 ),
497 );
498 }
499
500 test "complete execution scopes are required" {
501 try expectSourceError(error.MissingRunStart, test_summary_10);
502 try expectSourceError(error.MissingRunEnd, test_run_start ++ test_summary_10);
503 try expectSourceError(error.UnmatchedRunEnd, test_run_end);
504 try expectSourceError(error.NestedRun, test_run_start ++ test_run_start);
505 try expectSourceError(error.InvalidRunEvent, test_run_invalid);
506 try expectSourceError(error.MissingSummaryInRun, test_run_start ++ test_run_end);
507 try expectSourceError(error.InvalidProfileJsonl, "not-json\n");
508 }
509
510 fn expectSourceError(expected: anyerror, source: []const u8) !void {
511 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
512 defer out.deinit();
513 try std.testing.expectError(
514 expected,
515 TestEngine.compareSources(
516 std.testing.allocator,
517 &out.writer,
518 .{ .baseline_path = "base", .candidate_path = "candidate" },
519 source,
520 source,
521 ),
522 );
523 }
524
525 test "comparison releases ownership on every allocation failure" {
526 try std.testing.checkAllAllocationFailures(
527 std.testing.allocator,
528 compareTestSources,
529 .{},
530 );
531 }
532
533 fn compareTestSources(allocator: Allocator) !void {
534 const source = test_run_start ++ test_summary_10 ++ test_run_end;
535 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
536 defer out.deinit();
537 _ = try TestEngine.compareSources(
538 allocator,
539 &out.writer,
540 .{
541 .baseline_path = "base",
542 .candidate_path = "candidate",
543 .min_runs = 1,
544 },
545 source,
546 source,
547 );
548 }
549
550 fn writeTestRuns(writer: *std.Io.Writer, count: u32, value: u64) !void {
551 for (0..count) |_| {
552 try writeTestRunMarker(writer, "start");
553 var stream = pretty_json.Writer.init(writer, .minified);
554 const object = try stream.object();
555 try object.field("kind", "summary");
556 try object.field("benchmark", "bench.test");
557 try object.field("summary", "phase");
558 try object.field("name", "total");
559 try object.field("mean", value);
560 try object.field("median", value);
561 try object.field("min", value);
562 try object.field("max", value);
563 try object.endLine();
564 try writeTestRunMarker(writer, "end");
565 }
566 }
567
568 fn writeTestRunMarker(writer: *std.Io.Writer, event: []const u8) !void {
569 var stream = pretty_json.Writer.init(writer, .minified);
570 const object = try stream.object();
571 try object.field("kind", "run");
572 try object.field("benchmark", "bench.test");
573 try object.field("event", event);
574 try object.endLine();
575 }