lib/coz/src/analysis.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const sys = @import("sys");
3 const pretty = @import("pretty");
4 const pretty_json = pretty.json;
5 const profile = @import("profile.zig");
6
7 pub const schema = "coz.analysis/v2";
8 pub const support_method = "within_run_experiment_counts";
9
10 pub const Options = struct {
11 min_delta: u64 = 5,
12 min_points: usize = 1,
13 };
14
15 pub const default_max_profile_bytes = 128 * 1024 * 1024;
16
17 pub const PointKind = enum {
18 throughput,
19 latency,
20 };
21
22 pub const SupportStatus = enum {
23 point_only,
24 unreplicated_curve,
25 within_run_repeated_curve,
26 };
27
28 pub const Support = struct {
29 status: SupportStatus,
30 speedup_point_count: u64,
31 experiment_count: u64,
32 baseline_experiment_count: u64,
33 minimum_experiments_per_point: u64,
34 };
35
36 pub const Measurement = struct {
37 virtual_speedup: f64,
38 program_speedup: f64,
39 experiment_count: u64,
40 selected_samples: u64,
41 observations: u64,
42 duration_ns: u64,
43 value_ns: f64,
44 throughput_delta: ?u64 = null,
45 latency_arrivals: ?u64 = null,
46 latency_departures: ?u64 = null,
47 latency_outstanding: ?f64 = null,
48 };
49
50 pub const Result = struct {
51 kind: PointKind,
52 selected: profile.Location,
53 progress_point: []const u8,
54 baseline_virtual_speedup: f64,
55 baseline_value_ns: f64,
56 min_program_speedup: f64,
57 max_program_speedup: f64,
58 total_selected_samples: u64,
59 slope: ?f64,
60 support: Support,
61 measurements: []Measurement,
62
63 pub fn deinit(self: *Result, allocator: std.mem.Allocator) void {
64 allocator.free(self.selected.file);
65 allocator.free(self.progress_point);
66 allocator.free(self.measurements);
67 self.* = undefined;
68 }
69 };
70
71 pub const CaptureIntegrity = struct {
72 status: []const u8,
73 method: []const u8,
74 action: []const u8,
75 message: ?[]const u8,
76 record_count: u64,
77 sample_record_count: u64,
78 lost_record_count: u64,
79 lost_event_count: u64,
80 lost_samples_record_count: u64,
81 lost_samples_count: u64,
82 throttle_record_count: u64,
83 unthrottle_record_count: u64,
84 loss_counter_value: ?u64,
85 terminal_status: []const u8,
86 };
87
88 pub const Summary = struct {
89 capture_integrity: CaptureIntegrity,
90 results: []Result,
91
92 pub fn deinit(self: *Summary, allocator: std.mem.Allocator) void {
93 for (self.results) |*result| result.deinit(allocator);
94 allocator.free(self.results);
95 self.* = undefined;
96 }
97
98 pub fn writeJson(self: Summary, writer: *std.Io.Writer) !void {
99 var stringify = pretty_json.Writer.init(writer, .minified);
100 try stringify.beginObject();
101 try stringify.objectField("schema");
102 try stringify.write(schema);
103 try stringify.objectField("capture_integrity");
104 try writeCaptureIntegrity(&stringify, self.capture_integrity);
105 try stringify.objectField("results");
106 try stringify.beginArray();
107 for (self.results) |result| {
108 try writeResultJson(&stringify, result);
109 }
110 try stringify.endArray();
111 try stringify.endObject();
112 try writer.writeByte('\n');
113 }
114
115 pub fn writePretty(
116 self: Summary,
117 allocator: std.mem.Allocator,
118 writer: *std.Io.Writer,
119 options: pretty.LayoutOptions,
120 ) !void {
121 var arena_state = std.heap.ArenaAllocator.init(allocator);
122 defer arena_state.deinit();
123 const builder = pretty.Builder.init(arena_state.allocator());
124 try pretty.write(writer, try summaryDocAlloc(builder, self), options);
125 }
126 };
127
128 fn writeResultJson(stringify: *pretty_json.Writer, result: Result) !void {
129 try stringify.beginObject();
130 try stringify.objectField("selected");
131 try writeLocation(stringify, result.selected);
132 try writeStringField(stringify, "progress_point", result.progress_point);
133 try writeStringField(stringify, "kind", kindName(result.kind));
134 try writeStringField(stringify, "goal", goalName(result.kind));
135 try writeStringField(stringify, "value_name", valueName(result.kind));
136 try stringify.objectField("baseline_virtual_speedup");
137 try stringify.write(result.baseline_virtual_speedup);
138 try stringify.objectField("baseline_value_ns");
139 try stringify.write(result.baseline_value_ns);
140 try stringify.objectField(if (result.kind == .throughput)
141 "baseline_period_ns"
142 else
143 "baseline_average_latency_ns");
144 try stringify.write(result.baseline_value_ns);
145 try stringify.objectField("min_program_speedup");
146 try stringify.write(result.min_program_speedup);
147 try stringify.objectField("max_program_speedup");
148 try stringify.write(result.max_program_speedup);
149 try writeU64Field(stringify, "total_selected_samples", result.total_selected_samples);
150 try stringify.objectField("slope");
151 try stringify.write(result.slope);
152 try stringify.objectField("support");
153 try writeSupportJson(stringify, result.support);
154 try stringify.objectField("measurements");
155 try stringify.beginArray();
156 for (result.measurements) |measurement| try writeMeasurementJson(stringify, measurement);
157 try stringify.endArray();
158 try stringify.endObject();
159 }
160
161 fn writeMeasurementJson(stringify: *pretty_json.Writer, measurement: Measurement) !void {
162 try stringify.beginObject();
163 try stringify.objectField("virtual_speedup");
164 try stringify.write(measurement.virtual_speedup);
165 try stringify.objectField("program_speedup");
166 try stringify.write(measurement.program_speedup);
167 try writeU64Field(stringify, "experiment_count", measurement.experiment_count);
168 try writeU64Field(stringify, "selected_samples", measurement.selected_samples);
169 try writeU64Field(stringify, "observations", measurement.observations);
170 try writeU64Field(stringify, "duration_ns", measurement.duration_ns);
171 try stringify.objectField("value_ns");
172 try stringify.write(measurement.value_ns);
173 if (measurement.throughput_delta) |delta| {
174 try writeU64Field(stringify, "delta", delta);
175 try stringify.objectField("period_ns");
176 try stringify.write(measurement.value_ns);
177 }
178 if (measurement.latency_arrivals) |arrivals| {
179 try writeU64Field(stringify, "arrivals", arrivals);
180 try stringify.objectField("average_latency_ns");
181 try stringify.write(measurement.value_ns);
182 }
183 if (measurement.latency_departures) |departures| {
184 try writeU64Field(stringify, "departures", departures);
185 }
186 if (measurement.latency_outstanding) |outstanding| {
187 try stringify.objectField("outstanding");
188 try stringify.write(outstanding);
189 }
190 try stringify.endObject();
191 }
192
193 fn writeSupportJson(stringify: *pretty_json.Writer, support: Support) !void {
194 try stringify.beginObject();
195 try writeStringField(stringify, "status", @tagName(support.status));
196 try writeStringField(stringify, "method", support_method);
197 try writeStringField(stringify, "action", supportAction(support.status));
198 try writeU64Field(stringify, "speedup_point_count", support.speedup_point_count);
199 try writeU64Field(stringify, "experiment_count", support.experiment_count);
200 try writeU64Field(
201 stringify,
202 "baseline_experiment_count",
203 support.baseline_experiment_count,
204 );
205 try writeU64Field(
206 stringify,
207 "minimum_experiments_per_point",
208 support.minimum_experiments_per_point,
209 );
210 try stringify.endObject();
211 }
212
213 pub fn supportAction(status: SupportStatus) []const u8 {
214 return switch (status) {
215 .point_only => "collect_baseline_and_perturbed_points",
216 .unreplicated_curve => "collect_repeated_experiments_per_point",
217 .within_run_repeated_curve => "validate_with_independent_runs",
218 };
219 }
220
221 fn writeCaptureIntegrity(
222 stringify: *pretty_json.Writer,
223 integrity: CaptureIntegrity,
224 ) !void {
225 try stringify.beginObject();
226 try writeStringField(stringify, "status", integrity.status);
227 try writeStringField(stringify, "method", integrity.method);
228 try writeStringField(stringify, "action", integrity.action);
229 try stringify.objectField("message");
230 try stringify.write(integrity.message);
231 try writeU64Field(stringify, "record_count", integrity.record_count);
232 try writeU64Field(stringify, "sample_record_count", integrity.sample_record_count);
233 try writeU64Field(stringify, "lost_record_count", integrity.lost_record_count);
234 try writeU64Field(stringify, "lost_event_count", integrity.lost_event_count);
235 try writeU64Field(
236 stringify,
237 "lost_samples_record_count",
238 integrity.lost_samples_record_count,
239 );
240 try writeU64Field(stringify, "lost_samples_count", integrity.lost_samples_count);
241 try writeU64Field(stringify, "throttle_record_count", integrity.throttle_record_count);
242 try writeU64Field(
243 stringify,
244 "unthrottle_record_count",
245 integrity.unthrottle_record_count,
246 );
247 try stringify.objectField("loss_counter_value");
248 try stringify.write(integrity.loss_counter_value);
249 try writeStringField(stringify, "terminal_status", integrity.terminal_status);
250 try stringify.endObject();
251 }
252
253 fn writeStringField(
254 stringify: *pretty_json.Writer,
255 name: []const u8,
256 value: []const u8,
257 ) !void {
258 try stringify.objectField(name);
259 try stringify.write(value);
260 }
261
262 fn writeU64Field(stringify: *pretty_json.Writer, name: []const u8, value: u64) !void {
263 try stringify.objectField(name);
264 try stringify.write(value);
265 }
266
267 fn captureIntegrity(sampling_value: ?profile.Sampling) CaptureIntegrity {
268 const sampling = sampling_value orelse return integrityResult(
269 .{},
270 "missing_sampling_summary",
271 "sampling_summary_missing",
272 "rerun_with_sampling_integrity_enabled",
273 "profile has no sampling transport summary",
274 );
275 if (!samplingCountsValid(sampling)) return integrityResult(
276 sampling,
277 "invalid_sampling_summary",
278 lossCounterMethod(sampling.loss_counter),
279 "inspect_profile_artifact_and_rerun",
280 "sampling record counts are internally inconsistent",
281 );
282 if (terminalIntegrity(sampling)) |integrity| return integrity;
283 if (samplingLost(sampling)) return integrityResult(
284 sampling,
285 "sample_loss",
286 lossCounterMethod(sampling.loss_counter),
287 "repeat_with_lower_sample_rate_or_larger_ring",
288 "perf reported lost sampling events",
289 );
290 if (sampling.throttle_record_count != 0 or sampling.unthrottle_record_count != 0) {
291 return integrityResult(
292 sampling,
293 "sampling_throttled",
294 lossCounterMethod(sampling.loss_counter),
295 "repeat_with_lower_sample_rate",
296 "perf throttled the sampling event",
297 );
298 }
299 return switch (sampling.loss_counter) {
300 .available => integrityResult(
301 sampling,
302 "complete",
303 "perf_format_lost_and_record_audit",
304 "use_profile",
305 null,
306 ),
307 .unsupported => integrityResult(
308 sampling,
309 "loss_counter_unavailable",
310 "perf_record_audit_without_terminal_counter",
311 "treat_as_partial_or_rerun_on_linux_6_plus",
312 "kernel did not provide a terminal cumulative loss counter",
313 ),
314 .read_failed => integrityResult(
315 sampling,
316 "loss_counter_read_failed",
317 "perf_record_audit_after_counter_read_failure",
318 "inspect_perf_event_and_rerun",
319 "terminal cumulative loss counter could not be read",
320 ),
321 };
322 }
323
324 fn terminalIntegrity(sampling: profile.Sampling) ?CaptureIntegrity {
325 return switch (sampling.terminal_status) {
326 .complete => null,
327 .not_started => integrityResult(
328 sampling,
329 "sampling_not_started",
330 "sampling_terminal_state",
331 "rerun_with_sampling_enabled",
332 "profile runtime did not start a sampler",
333 ),
334 .stop_failed => integrityResult(
335 sampling,
336 "sampling_stop_failed",
337 "sampling_terminal_state",
338 "inspect_sampler_shutdown_and_rerun",
339 "sampling event could not be stopped before the terminal audit",
340 ),
341 .drain_failed => integrityResult(
342 sampling,
343 "sampling_drain_failed",
344 "sampling_terminal_state",
345 "inspect_perf_ring_and_rerun",
346 "perf ring could not be drained at shutdown",
347 ),
348 };
349 }
350
351 fn samplingCountsValid(sampling: profile.Sampling) bool {
352 const known = @as(u128, sampling.sample_record_count) +
353 sampling.lost_record_count + sampling.lost_samples_record_count +
354 sampling.throttle_record_count + sampling.unthrottle_record_count;
355 return known <= sampling.record_count;
356 }
357
358 fn samplingLost(sampling: profile.Sampling) bool {
359 if (sampling.lost_record_count != 0 or sampling.lost_event_count != 0) return true;
360 if (sampling.lost_samples_record_count != 0 or sampling.lost_samples_count != 0) return true;
361 return switch (sampling.loss_counter) {
362 .available => |value| value != 0,
363 .unsupported, .read_failed => false,
364 };
365 }
366
367 fn lossCounterMethod(counter: profile.LossCounter) []const u8 {
368 return switch (counter) {
369 .available => "perf_format_lost_and_record_audit",
370 .unsupported => "perf_record_audit_without_terminal_counter",
371 .read_failed => "perf_record_audit_after_counter_read_failure",
372 };
373 }
374
375 fn integrityResult(
376 sampling: profile.Sampling,
377 status: []const u8,
378 method: []const u8,
379 action: []const u8,
380 message: ?[]const u8,
381 ) CaptureIntegrity {
382 return .{
383 .status = status,
384 .method = method,
385 .action = action,
386 .message = message,
387 .record_count = sampling.record_count,
388 .sample_record_count = sampling.sample_record_count,
389 .lost_record_count = sampling.lost_record_count,
390 .lost_event_count = sampling.lost_event_count,
391 .lost_samples_record_count = sampling.lost_samples_record_count,
392 .lost_samples_count = sampling.lost_samples_count,
393 .throttle_record_count = sampling.throttle_record_count,
394 .unthrottle_record_count = sampling.unthrottle_record_count,
395 .loss_counter_value = switch (sampling.loss_counter) {
396 .available => |value| value,
397 .unsupported, .read_failed => null,
398 },
399 .terminal_status = @tagName(sampling.terminal_status),
400 };
401 }
402
403 pub const Accumulator = struct {
404 current_experiment: ?ExperimentContext = null,
405 curves: std.StringHashMapUnmanaged(Curve) = .empty,
406 sampling: ?profile.Sampling = null,
407
408 pub fn deinit(self: *Accumulator, allocator: std.mem.Allocator) void {
409 self.clearCurrentExperiment(allocator);
410 var iter = self.curves.iterator();
411 while (iter.next()) |entry| {
412 allocator.free(entry.key_ptr.*);
413 entry.value_ptr.deinit(allocator);
414 }
415 self.curves.deinit(allocator);
416 self.* = .{};
417 }
418
419 pub fn observe(self: *Accumulator, allocator: std.mem.Allocator, event: profile.Event) !void {
420 switch (event) {
421 .experiment => |experiment| {
422 self.clearCurrentExperiment(allocator);
423 const selected_file = try allocator.dupe(u8, experiment.selected.file);
424 self.current_experiment = .{
425 .selected = .{
426 .file = selected_file,
427 .line = experiment.selected.line,
428 },
429 .virtual_speedup = experiment.virtual_speedup,
430 .duration_ns = experiment.duration_ns,
431 .selected_samples = experiment.selected_samples,
432 };
433 },
434 .throughput => |throughput| {
435 if (self.current_experiment) |experiment| {
436 try self.addThroughput(allocator, experiment, throughput);
437 }
438 },
439 .latency => |latency| {
440 if (self.current_experiment) |experiment| {
441 try self.addLatency(allocator, experiment, latency);
442 }
443 },
444 .sampling => |sampling| {
445 if (self.sampling != null) return error.DuplicateSamplingSummary;
446 self.sampling = sampling;
447 },
448 .startup, .runtime, .sample => {},
449 }
450 }
451
452 pub fn summarize(self: *const Accumulator, allocator: std.mem.Allocator, options: Options) !Summary {
453 var results: std.ArrayListUnmanaged(Result) = .empty;
454 errdefer {
455 for (results.items) |*result| result.deinit(allocator);
456 results.deinit(allocator);
457 }
458
459 var iter = self.curves.valueIterator();
460 while (iter.next()) |curve| {
461 if (try curve.result(allocator, options)) |result| {
462 try results.append(allocator, result);
463 }
464 }
465
466 std.mem.sort(Result, results.items, {}, resultLessThan);
467 return .{
468 .capture_integrity = captureIntegrity(self.sampling),
469 .results = try results.toOwnedSlice(allocator),
470 };
471 }
472
473 fn clearCurrentExperiment(self: *Accumulator, allocator: std.mem.Allocator) void {
474 if (self.current_experiment) |experiment| {
475 allocator.free(experiment.selected.file);
476 self.current_experiment = null;
477 }
478 }
479
480 fn addThroughput(
481 self: *Accumulator,
482 allocator: std.mem.Allocator,
483 experiment: ExperimentContext,
484 throughput: profile.Throughput,
485 ) !void {
486 const curve = try self.getCurve(allocator, .throughput, experiment.selected, throughput.name);
487 try curve.addThroughput(
488 allocator,
489 experiment.virtual_speedup,
490 experiment.duration_ns,
491 experiment.selected_samples,
492 throughput.delta,
493 );
494 }
495
496 fn addLatency(
497 self: *Accumulator,
498 allocator: std.mem.Allocator,
499 experiment: ExperimentContext,
500 latency: profile.Latency,
501 ) !void {
502 const curve = try self.getCurve(allocator, .latency, experiment.selected, latency.name);
503 try curve.addLatency(
504 allocator,
505 experiment.virtual_speedup,
506 experiment.duration_ns,
507 experiment.selected_samples,
508 latency.arrivals,
509 latency.departures,
510 latency.outstanding,
511 );
512 }
513
514 fn getCurve(
515 self: *Accumulator,
516 allocator: std.mem.Allocator,
517 kind: PointKind,
518 selected: profile.Location,
519 progress_point: []const u8,
520 ) !*Curve {
521 const key = try std.fmt.allocPrint(
522 allocator,
523 "{s}\x1f{s}:{d}\x1f{s}",
524 .{ kindName(kind), selected.file, selected.line, progress_point },
525 );
526 var owned_key: ?[]u8 = key;
527 errdefer if (owned_key) |remaining| allocator.free(remaining);
528
529 const entry = try self.curves.getOrPut(allocator, key);
530 if (entry.found_existing) {
531 allocator.free(key);
532 owned_key = null;
533 } else {
534 owned_key = null;
535 errdefer {
536 if (self.curves.fetchRemove(key)) |removed| allocator.free(removed.key);
537 }
538 entry.value_ptr.* = try Curve.init(allocator, kind, selected, progress_point);
539 }
540
541 return entry.value_ptr;
542 }
543 };
544
545 pub fn summarizeJsonLines(
546 allocator: std.mem.Allocator,
547 bytes: []const u8,
548 options: Options,
549 ) !Summary {
550 var accumulator: Accumulator = .{};
551 defer accumulator.deinit(allocator);
552
553 var lines = std.mem.splitScalar(u8, bytes, '\n');
554 while (lines.next()) |raw_line| {
555 const line = std.mem.trim(u8, raw_line, " \t\r");
556 if (line.len == 0) continue;
557 var parsed = try profile.parseJsonLine(allocator, line);
558 defer parsed.deinit(allocator);
559 try accumulator.observe(allocator, parsed.event);
560 }
561
562 return try accumulator.summarize(allocator, options);
563 }
564
565 pub fn summarizeFileAlloc(
566 allocator: std.mem.Allocator,
567 path: []const u8,
568 max_bytes: usize,
569 options: Options,
570 ) !Summary {
571 const bytes = try sys.fs.readFileAlloc(allocator, path, max_bytes);
572 defer allocator.free(bytes);
573 return try summarizeJsonLines(allocator, bytes, options);
574 }
575
576 const ExperimentContext = struct {
577 selected: profile.Location,
578 virtual_speedup: f64,
579 duration_ns: u64,
580 selected_samples: u64,
581 };
582
583 const SpeedupAggregate = struct {
584 virtual_speedup: f64,
585 experiment_count: u64,
586 duration_ns: u64,
587 selected_samples: u64,
588 observations: u64,
589 departures: u64 = 0,
590 outstanding_duration_ns: f64 = 0,
591
592 fn addThroughput(self: *SpeedupAggregate, duration_ns: u64, selected_samples: u64, delta: u64) void {
593 self.experiment_count +|= 1;
594 self.duration_ns += duration_ns;
595 self.selected_samples += selected_samples;
596 self.observations += delta;
597 }
598
599 fn addLatency(self: *SpeedupAggregate, duration_ns: u64, selected_samples: u64, arrivals: u64, departures: u64, outstanding: u64) void {
600 self.experiment_count +|= 1;
601 self.duration_ns += duration_ns;
602 self.selected_samples += selected_samples;
603 self.observations += arrivals;
604 self.departures += departures;
605 self.outstanding_duration_ns += @as(f64, @floatFromInt(outstanding)) * @as(f64, @floatFromInt(duration_ns));
606 }
607
608 fn valueNs(self: SpeedupAggregate, kind: PointKind) f64 {
609 return switch (kind) {
610 .throughput => @as(f64, @floatFromInt(self.duration_ns)) / @as(f64, @floatFromInt(self.observations)),
611 .latency => self.outstanding_duration_ns / @as(f64, @floatFromInt(self.observations)),
612 };
613 }
614
615 fn latencyOutstanding(self: SpeedupAggregate) f64 {
616 if (self.duration_ns == 0) return 0;
617 return self.outstanding_duration_ns / @as(f64, @floatFromInt(self.duration_ns));
618 }
619 };
620
621 const Curve = struct {
622 kind: PointKind,
623 selected: profile.Location,
624 progress_point: []u8,
625 speedups: std.ArrayListUnmanaged(SpeedupAggregate) = .empty,
626
627 fn init(allocator: std.mem.Allocator, kind: PointKind, selected: profile.Location, progress_point: []const u8) !Curve {
628 const selected_file = try allocator.dupe(u8, selected.file);
629 errdefer allocator.free(selected_file);
630
631 const owned_progress_point = try allocator.dupe(u8, progress_point);
632
633 return .{
634 .kind = kind,
635 .selected = .{
636 .file = selected_file,
637 .line = selected.line,
638 },
639 .progress_point = owned_progress_point,
640 };
641 }
642
643 fn deinit(self: *Curve, allocator: std.mem.Allocator) void {
644 allocator.free(self.selected.file);
645 allocator.free(self.progress_point);
646 self.speedups.deinit(allocator);
647 self.* = undefined;
648 }
649
650 fn addThroughput(
651 self: *Curve,
652 allocator: std.mem.Allocator,
653 virtual_speedup: f64,
654 duration_ns: u64,
655 selected_samples: u64,
656 delta: u64,
657 ) !void {
658 std.debug.assert(self.kind == .throughput);
659 for (self.speedups.items) |*aggregate| {
660 if (aggregate.virtual_speedup == virtual_speedup) {
661 aggregate.addThroughput(duration_ns, selected_samples, delta);
662 return;
663 }
664 }
665
666 var aggregate = SpeedupAggregate{
667 .virtual_speedup = virtual_speedup,
668 .experiment_count = 0,
669 .duration_ns = 0,
670 .selected_samples = 0,
671 .observations = 0,
672 };
673 aggregate.addThroughput(duration_ns, selected_samples, delta);
674 try self.speedups.append(allocator, aggregate);
675 }
676
677 fn addLatency(
678 self: *Curve,
679 allocator: std.mem.Allocator,
680 virtual_speedup: f64,
681 duration_ns: u64,
682 selected_samples: u64,
683 arrivals: u64,
684 departures: u64,
685 outstanding: u64,
686 ) !void {
687 std.debug.assert(self.kind == .latency);
688 for (self.speedups.items) |*aggregate| {
689 if (aggregate.virtual_speedup == virtual_speedup) {
690 aggregate.addLatency(duration_ns, selected_samples, arrivals, departures, outstanding);
691 return;
692 }
693 }
694
695 var aggregate = SpeedupAggregate{
696 .virtual_speedup = virtual_speedup,
697 .experiment_count = 0,
698 .duration_ns = 0,
699 .selected_samples = 0,
700 .observations = 0,
701 };
702 aggregate.addLatency(duration_ns, selected_samples, arrivals, departures, outstanding);
703 try self.speedups.append(allocator, aggregate);
704 }
705
706 fn result(self: *const Curve, allocator: std.mem.Allocator, options: Options) !?Result {
707 var valid: std.ArrayListUnmanaged(SpeedupAggregate) = .empty;
708 defer valid.deinit(allocator);
709
710 for (self.speedups.items) |aggregate| {
711 if (aggregate.observations >= options.min_delta) try valid.append(allocator, aggregate);
712 }
713 if (valid.items.len < options.min_points) return null;
714
715 std.mem.sort(SpeedupAggregate, valid.items, {}, speedupLessThan);
716 const baseline = chooseBaseline(valid.items) orelse return null;
717 const baseline_value_ns = baseline.valueNs(self.kind);
718 if (!validDataPoint(baseline_value_ns) or baseline_value_ns == 0) return null;
719
720 var data = try self.measurementsFor(allocator, valid.items, baseline_value_ns);
721 defer data.measurements.deinit(allocator);
722 if (data.measurements.items.len < options.min_points) return null;
723
724 const owned_file = try allocator.dupe(u8, self.selected.file);
725 errdefer allocator.free(owned_file);
726
727 const owned_progress_point = try allocator.dupe(u8, self.progress_point);
728 errdefer allocator.free(owned_progress_point);
729
730 const result_slope = slope(data.measurements.items);
731 const support = supportFor(data.measurements.items, baseline.virtual_speedup);
732 const owned_measurements = try data.measurements.toOwnedSlice(allocator);
733 errdefer allocator.free(owned_measurements);
734
735 return .{
736 .kind = self.kind,
737 .selected = .{
738 .file = owned_file,
739 .line = self.selected.line,
740 },
741 .progress_point = owned_progress_point,
742 .baseline_virtual_speedup = baseline.virtual_speedup,
743 .baseline_value_ns = baseline_value_ns,
744 .min_program_speedup = data.min_program_speedup,
745 .max_program_speedup = data.max_program_speedup,
746 .total_selected_samples = data.total_selected_samples,
747 .slope = result_slope,
748 .support = support,
749 .measurements = owned_measurements,
750 };
751 }
752
753 fn measurementsFor(
754 self: *const Curve,
755 allocator: std.mem.Allocator,
756 aggregates: []const SpeedupAggregate,
757 baseline_value_ns: f64,
758 ) !CurveData {
759 var data: CurveData = .{};
760 errdefer data.measurements.deinit(allocator);
761 for (aggregates) |aggregate| {
762 const value_ns = aggregate.valueNs(self.kind);
763 if (!validDataPoint(value_ns)) continue;
764 var program_speedup = (baseline_value_ns - value_ns) / baseline_value_ns;
765 if (self.kind == .latency) program_speedup = -program_speedup;
766 if (program_speedup < -1 or program_speedup > 2) continue;
767 data.max_program_speedup = @max(data.max_program_speedup, program_speedup);
768 data.min_program_speedup = @min(data.min_program_speedup, program_speedup);
769 data.total_selected_samples += aggregate.selected_samples;
770 var measurement = Measurement{
771 .virtual_speedup = aggregate.virtual_speedup,
772 .program_speedup = program_speedup,
773 .experiment_count = aggregate.experiment_count,
774 .selected_samples = aggregate.selected_samples,
775 .observations = aggregate.observations,
776 .duration_ns = aggregate.duration_ns,
777 .value_ns = value_ns,
778 };
779 switch (self.kind) {
780 .throughput => measurement.throughput_delta = aggregate.observations,
781 .latency => {
782 measurement.latency_arrivals = aggregate.observations;
783 measurement.latency_departures = aggregate.departures;
784 measurement.latency_outstanding = aggregate.latencyOutstanding();
785 },
786 }
787 try data.measurements.append(allocator, measurement);
788 }
789 return data;
790 }
791 };
792
793 const CurveData = struct {
794 measurements: std.ArrayListUnmanaged(Measurement) = .empty,
795 min_program_speedup: f64 = std.math.inf(f64),
796 max_program_speedup: f64 = -std.math.inf(f64),
797 total_selected_samples: u64 = 0,
798 };
799
800 fn supportFor(measurements: []const Measurement, baseline_virtual_speedup: f64) Support {
801 std.debug.assert(measurements.len > 0);
802 var experiment_count: u64 = 0;
803 var baseline_experiment_count: u64 = 0;
804 var minimum_experiments_per_point: u64 = std.math.maxInt(u64);
805 for (measurements) |measurement| {
806 experiment_count +|= measurement.experiment_count;
807 minimum_experiments_per_point = @min(
808 minimum_experiments_per_point,
809 measurement.experiment_count,
810 );
811 if (measurement.virtual_speedup == baseline_virtual_speedup) {
812 baseline_experiment_count = measurement.experiment_count;
813 }
814 }
815 const status: SupportStatus = if (measurements.len < 2)
816 .point_only
817 else if (minimum_experiments_per_point < 2)
818 .unreplicated_curve
819 else
820 .within_run_repeated_curve;
821 return .{
822 .status = status,
823 .speedup_point_count = @intCast(measurements.len),
824 .experiment_count = experiment_count,
825 .baseline_experiment_count = baseline_experiment_count,
826 .minimum_experiments_per_point = minimum_experiments_per_point,
827 };
828 }
829
830 fn chooseBaseline(items: []const SpeedupAggregate) ?SpeedupAggregate {
831 for (items) |item| {
832 if (item.virtual_speedup == 0) return item;
833 }
834 return null;
835 }
836
837 fn validDataPoint(value: f64) bool {
838 return std.math.isFinite(value) and !std.math.isNan(value);
839 }
840
841 fn slope(measurements: []const Measurement) ?f64 {
842 if (measurements.len < 2) return null;
843
844 var sum_x: f64 = 0;
845 var sum_y: f64 = 0;
846 var sum_xy: f64 = 0;
847 var sum_x2: f64 = 0;
848 for (measurements) |measurement| {
849 const x = measurement.virtual_speedup;
850 const y = measurement.program_speedup;
851 sum_x += x;
852 sum_y += y;
853 sum_xy += x * y;
854 sum_x2 += x * x;
855 }
856
857 const n: f64 = @floatFromInt(measurements.len);
858 const denominator = n * sum_x2 - sum_x * sum_x;
859 if (denominator == 0) return null;
860 return (n * sum_xy - sum_x * sum_y) / denominator;
861 }
862
863 fn speedupLessThan(_: void, lhs: SpeedupAggregate, rhs: SpeedupAggregate) bool {
864 return lhs.virtual_speedup < rhs.virtual_speedup;
865 }
866
867 fn resultLessThan(_: void, lhs: Result, rhs: Result) bool {
868 const file_order = std.mem.order(u8, lhs.selected.file, rhs.selected.file);
869 if (file_order != .eq) return file_order == .lt;
870 if (lhs.selected.line != rhs.selected.line) return lhs.selected.line < rhs.selected.line;
871 if (lhs.kind != rhs.kind) return @backingInt(lhs.kind) < @backingInt(rhs.kind);
872 return std.mem.lessThan(u8, lhs.progress_point, rhs.progress_point);
873 }
874
875 fn kindName(kind: PointKind) []const u8 {
876 return switch (kind) {
877 .throughput => "throughput",
878 .latency => "latency",
879 };
880 }
881
882 fn goalName(kind: PointKind) []const u8 {
883 return switch (kind) {
884 .throughput => "maximize",
885 .latency => "minimize",
886 };
887 }
888
889 fn valueName(kind: PointKind) []const u8 {
890 return switch (kind) {
891 .throughput => "period_ns",
892 .latency => "average_latency_ns",
893 };
894 }
895
896 fn baselineValueLabel(kind: PointKind) []const u8 {
897 return switch (kind) {
898 .throughput => "baseline_period",
899 .latency => "baseline_average_latency",
900 };
901 }
902
903 fn measurementObservationLabel(measurement: Measurement) []const u8 {
904 if (measurement.throughput_delta != null) return "delta";
905 return "arrivals";
906 }
907
908 fn measurementValueLabel(measurement: Measurement) []const u8 {
909 if (measurement.throughput_delta != null) return "period";
910 return "average_latency";
911 }
912
913 fn writeLocation(stringify: *pretty_json.Writer, location: profile.Location) !void {
914 try stringify.beginObject();
915 try stringify.objectField("file");
916 try stringify.write(location.file);
917 try stringify.objectField("line");
918 try stringify.write(location.line);
919 try stringify.endObject();
920 }
921
922 fn summaryDocAlloc(builder: pretty.Builder, summary: Summary) !pretty.Doc {
923 var parts: std.ArrayListUnmanaged(pretty.Doc) = .empty;
924 defer parts.deinit(builder.allocator);
925
926 try parts.append(builder.allocator, try summaryHeaderDocAlloc(builder, summary));
927 for (summary.results) |result| {
928 try parts.append(builder.allocator, try resultDocAlloc(builder, result));
929 }
930 return try builder.concat(parts.items);
931 }
932
933 fn summaryHeaderDocAlloc(builder: pretty.Builder, summary: Summary) !pretty.Doc {
934 return try builder.concat(&.{
935 try builder.styledText(.title, "coz profiler"),
936 try builder.punct(":"),
937 builder.text(" "),
938 try builder.styledText(.attribute, "integrity"),
939 try builder.punct("="),
940 try builder.styledText(
941 if (std.mem.eql(u8, summary.capture_integrity.status, "complete")) .value else .warning,
942 summary.capture_integrity.status,
943 ),
944 try builder.punct(";"),
945 builder.text(" "),
946 try builder.styledText(.attribute, "results"),
947 try builder.punct("="),
948 try builder.styledFmt(.number, "{d}", .{summary.results.len}),
949 pretty.hardline,
950 });
951 }
952
953 fn resultDocAlloc(builder: pretty.Builder, result: Result) !pretty.Doc {
954 const fields = [_]pretty.Doc{
955 try metricDocAlloc(builder, "kind", try builder.styledText(.attribute, kindName(result.kind))),
956 try metricDocAlloc(
957 builder,
958 "support",
959 try builder.styledText(
960 supportStyle(result.support.status),
961 @tagName(result.support.status),
962 ),
963 ),
964 try u64MetricDocAlloc(
965 builder,
966 "speedup_points",
967 result.support.speedup_point_count,
968 .number,
969 ),
970 try u64MetricDocAlloc(
971 builder,
972 "experiments",
973 result.support.experiment_count,
974 .number,
975 ),
976 try u64MetricDocAlloc(
977 builder,
978 "baseline_experiments",
979 result.support.baseline_experiment_count,
980 .number,
981 ),
982 try u64MetricDocAlloc(
983 builder,
984 "minimum_experiments_per_point",
985 result.support.minimum_experiments_per_point,
986 .number,
987 ),
988 try percentMetricDocAlloc(builder, "baseline_virtual_speedup", result.baseline_virtual_speedup, .number),
989 try floatNsMetricDocAlloc(builder, baselineValueLabel(result.kind), result.baseline_value_ns, .number),
990 try percentMetricDocAlloc(builder, "max_program_speedup", result.max_program_speedup, speedupStyle(result.max_program_speedup)),
991 try u64MetricDocAlloc(builder, "total_selected_samples", result.total_selected_samples, .number),
992 try optionalFloatMetricDocAlloc(builder, "slope", result.slope),
993 };
994
995 var parts: std.ArrayListUnmanaged(pretty.Doc) = .empty;
996 defer parts.deinit(builder.allocator);
997
998 try parts.append(builder.allocator, try builder.concat(&.{
999 try builder.spaces(2),
1000 try locationDocAlloc(builder, result.selected),
1001 builder.text(" "),
1002 try builder.punct("->"),
1003 builder.text(" "),
1004 try builder.styledText(.name, result.progress_point),
1005 try builder.punct(":"),
1006 builder.text(" "),
1007 try builder.nest(4, try builder.group(try builder.join(&fields, try metricSeparatorDocAlloc(builder)))),
1008 pretty.hardline,
1009 }));
1010
1011 for (result.measurements, 0..) |measurement, index| {
1012 try parts.append(builder.allocator, try measurementDocAlloc(builder, measurement, index + 1));
1013 }
1014
1015 return try builder.concat(parts.items);
1016 }
1017
1018 fn locationDocAlloc(builder: pretty.Builder, location: profile.Location) !pretty.Doc {
1019 return try builder.concat(&.{
1020 try builder.styledText(.source, location.file),
1021 try builder.punct(":"),
1022 try builder.styledFmt(.number, "{d}", .{location.line}),
1023 });
1024 }
1025
1026 fn measurementDocAlloc(builder: pretty.Builder, measurement: Measurement, index: usize) !pretty.Doc {
1027 const fields = [_]pretty.Doc{
1028 try percentMetricDocAlloc(builder, "virtual_speedup", measurement.virtual_speedup, .number),
1029 try percentMetricDocAlloc(builder, "program_speedup", measurement.program_speedup, speedupStyle(measurement.program_speedup)),
1030 try u64MetricDocAlloc(builder, "experiments", measurement.experiment_count, .number),
1031 try u64MetricDocAlloc(builder, "selected_samples", measurement.selected_samples, .number),
1032 try u64MetricDocAlloc(builder, measurementObservationLabel(measurement), measurement.observations, .number),
1033 try u64NsMetricDocAlloc(builder, "duration", measurement.duration_ns, .number),
1034 try floatNsMetricDocAlloc(builder, measurementValueLabel(measurement), measurement.value_ns, .number),
1035 };
1036
1037 return try builder.concat(&.{
1038 try builder.spaces(4),
1039 try builder.styledFmt(.number, "{d}", .{index}),
1040 try builder.punct("."),
1041 builder.text(" "),
1042 try builder.nest(4, try builder.group(try builder.join(&fields, try metricSeparatorDocAlloc(builder)))),
1043 pretty.hardline,
1044 });
1045 }
1046
1047 fn supportStyle(status: SupportStatus) pretty.Style {
1048 return switch (status) {
1049 .point_only, .unreplicated_curve => .warning,
1050 .within_run_repeated_curve => .value,
1051 };
1052 }
1053
1054 fn metricSeparatorDocAlloc(builder: pretty.Builder) !pretty.Doc {
1055 return try builder.concat(&.{
1056 try builder.punct(";"),
1057 pretty.softline,
1058 });
1059 }
1060
1061 fn percentMetricDocAlloc(
1062 builder: pretty.Builder,
1063 label: []const u8,
1064 value: f64,
1065 value_style: pretty.Style,
1066 ) !pretty.Doc {
1067 return try metricDocAlloc(
1068 builder,
1069 label,
1070 try builder.styledFmt(value_style, "{d:.2}%", .{value * 100.0}),
1071 );
1072 }
1073
1074 fn floatNsMetricDocAlloc(
1075 builder: pretty.Builder,
1076 label: []const u8,
1077 value: f64,
1078 value_style: pretty.Style,
1079 ) !pretty.Doc {
1080 return try metricDocAlloc(
1081 builder,
1082 label,
1083 try builder.styledFmt(value_style, "{d:.2} ns", .{value}),
1084 );
1085 }
1086
1087 fn u64NsMetricDocAlloc(
1088 builder: pretty.Builder,
1089 label: []const u8,
1090 value: u64,
1091 value_style: pretty.Style,
1092 ) !pretty.Doc {
1093 return try metricDocAlloc(
1094 builder,
1095 label,
1096 try builder.styledFmt(value_style, "{d} ns", .{value}),
1097 );
1098 }
1099
1100 fn u64MetricDocAlloc(
1101 builder: pretty.Builder,
1102 label: []const u8,
1103 value: u64,
1104 value_style: pretty.Style,
1105 ) !pretty.Doc {
1106 return try metricDocAlloc(
1107 builder,
1108 label,
1109 try builder.styledFmt(value_style, "{d}", .{value}),
1110 );
1111 }
1112
1113 fn optionalFloatMetricDocAlloc(builder: pretty.Builder, label: []const u8, value: ?f64) !pretty.Doc {
1114 if (value) |actual| {
1115 return try metricDocAlloc(builder, label, try builder.styledFmt(.number, "{d:.4}", .{actual}));
1116 }
1117 return try metricDocAlloc(builder, label, try builder.styledText(.muted, "n/a"));
1118 }
1119
1120 fn metricDocAlloc(builder: pretty.Builder, label: []const u8, value_doc: pretty.Doc) !pretty.Doc {
1121 return try builder.concat(&.{
1122 try builder.styledText(.attribute, label),
1123 try builder.punct("="),
1124 value_doc,
1125 });
1126 }
1127
1128 fn speedupStyle(value: f64) pretty.Style {
1129 if (value < 0) return .danger;
1130 if (value > 0) return .value;
1131 return .number;
1132 }
1133
1134 test "accumulator experiment replacement survives allocation failures" {
1135 try std.testing.checkAllAllocationFailures(
1136 std.testing.allocator,
1137 checkExperimentReplacementAllocationFailures,
1138 .{},
1139 );
1140 }
1141
1142 fn checkExperimentReplacementAllocationFailures(allocator: std.mem.Allocator) !void {
1143 var accumulator: Accumulator = .{};
1144 defer accumulator.deinit(allocator);
1145
1146 try accumulator.observe(allocator, .{ .experiment = .{
1147 .selected = .{ .file = "src/first.zig", .line = 10 },
1148 .virtual_speedup = 0,
1149 .duration_ns = 100,
1150 .selected_samples = 1,
1151 } });
1152 try accumulator.observe(allocator, .{ .experiment = .{
1153 .selected = .{ .file = "src/second.zig", .line = 20 },
1154 .virtual_speedup = 0.5,
1155 .duration_ns = 80,
1156 .selected_samples = 2,
1157 } });
1158
1159 try std.testing.expectEqualStrings(
1160 "src/second.zig",
1161 accumulator.current_experiment.?.selected.file,
1162 );
1163 }
1164
1165 test "accumulator computes program speedup from throughput periods" {
1166 var accumulator: Accumulator = .{};
1167 defer accumulator.deinit(std.testing.allocator);
1168
1169 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1170 .selected = .{ .file = "src/work.zig", .line = 10 },
1171 .virtual_speedup = 0,
1172 .duration_ns = 100,
1173 .selected_samples = 2,
1174 } });
1175 try accumulator.observe(std.testing.allocator, .{ .throughput = .{
1176 .name = "items",
1177 .delta = 10,
1178 } });
1179 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1180 .selected = .{ .file = "src/work.zig", .line = 10 },
1181 .virtual_speedup = 0.5,
1182 .duration_ns = 80,
1183 .selected_samples = 3,
1184 } });
1185 try accumulator.observe(std.testing.allocator, .{ .throughput = .{
1186 .name = "items",
1187 .delta = 10,
1188 } });
1189
1190 var summary = try accumulator.summarize(std.testing.allocator, .{ .min_delta = 1, .min_points = 2 });
1191 defer summary.deinit(std.testing.allocator);
1192
1193 try std.testing.expectEqual(@as(usize, 1), summary.results.len);
1194 const result = summary.results[0];
1195 try std.testing.expectEqual(PointKind.throughput, result.kind);
1196 try std.testing.expectEqualStrings("src/work.zig", result.selected.file);
1197 try std.testing.expectEqual(@as(u64, 10), result.selected.line);
1198 try std.testing.expectEqualStrings("items", result.progress_point);
1199 try std.testing.expectEqual(@as(f64, 10), result.baseline_value_ns);
1200 try std.testing.expectEqual(@as(f64, 0), result.min_program_speedup);
1201 try std.testing.expectApproxEqAbs(@as(f64, 0.2), result.max_program_speedup, 0.000001);
1202 try std.testing.expectEqual(@as(u64, 5), result.total_selected_samples);
1203 try std.testing.expectEqual(@as(usize, 2), result.measurements.len);
1204 try std.testing.expectEqual(SupportStatus.unreplicated_curve, result.support.status);
1205 try std.testing.expectEqual(@as(u64, 2), result.support.experiment_count);
1206 try std.testing.expectEqual(@as(u64, 1), result.support.baseline_experiment_count);
1207 try std.testing.expectEqual(@as(u64, 1), result.support.minimum_experiments_per_point);
1208 try std.testing.expectEqual(@as(u64, 1), result.measurements[0].experiment_count);
1209 try std.testing.expectEqual(@as(u64, 2), result.measurements[0].selected_samples);
1210 try std.testing.expectEqual(@as(u64, 3), result.measurements[1].selected_samples);
1211 try std.testing.expectApproxEqAbs(@as(f64, 0.4), result.slope.?, 0.000001);
1212 }
1213
1214 test "accumulator computes latency speedups with upstream Little's Law metric" {
1215 var accumulator: Accumulator = .{};
1216 defer accumulator.deinit(std.testing.allocator);
1217
1218 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1219 .selected = .{ .file = "src/work.zig", .line = 10 },
1220 .virtual_speedup = 0,
1221 .duration_ns = 100,
1222 .selected_samples = 2,
1223 } });
1224 try accumulator.observe(std.testing.allocator, .{ .latency = .{
1225 .name = "request",
1226 .arrivals = 10,
1227 .departures = 9,
1228 .outstanding = 5,
1229 } });
1230 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1231 .selected = .{ .file = "src/work.zig", .line = 10 },
1232 .virtual_speedup = 0.5,
1233 .duration_ns = 80,
1234 .selected_samples = 3,
1235 } });
1236 try accumulator.observe(std.testing.allocator, .{ .latency = .{
1237 .name = "request",
1238 .arrivals = 10,
1239 .departures = 10,
1240 .outstanding = 4,
1241 } });
1242
1243 var summary = try accumulator.summarize(std.testing.allocator, .{ .min_delta = 1, .min_points = 2 });
1244 defer summary.deinit(std.testing.allocator);
1245
1246 try std.testing.expectEqual(@as(usize, 1), summary.results.len);
1247 const result = summary.results[0];
1248 try std.testing.expectEqual(PointKind.latency, result.kind);
1249 try std.testing.expectEqualStrings("request", result.progress_point);
1250 try std.testing.expectEqual(@as(f64, 50), result.baseline_value_ns);
1251 try std.testing.expectApproxEqAbs(@as(f64, -0.36), result.min_program_speedup, 0.000001);
1252 try std.testing.expectEqual(@as(f64, 0), result.max_program_speedup);
1253 try std.testing.expectEqual(@as(u64, 5), result.total_selected_samples);
1254 try std.testing.expectEqual(@as(usize, 2), result.measurements.len);
1255 try std.testing.expectEqual(@as(u64, 2), result.measurements[0].selected_samples);
1256 try std.testing.expectEqual(@as(u64, 3), result.measurements[1].selected_samples);
1257 try std.testing.expectEqual(@as(u64, 10), result.measurements[1].observations);
1258 try std.testing.expectEqual(@as(u64, 10), result.measurements[1].latency_arrivals.?);
1259 try std.testing.expectEqual(@as(u64, 10), result.measurements[1].latency_departures.?);
1260 try std.testing.expectEqual(@as(f64, 4), result.measurements[1].latency_outstanding.?);
1261 try std.testing.expectApproxEqAbs(@as(f64, 32), result.measurements[1].value_ns, 0.000001);
1262 try std.testing.expectApproxEqAbs(@as(f64, -0.36), result.measurements[1].program_speedup, 0.000001);
1263 }
1264
1265 test "accumulator merges repeated speedups and reports repeated curve support" {
1266 var accumulator: Accumulator = .{};
1267 defer accumulator.deinit(std.testing.allocator);
1268
1269 for ([_]f64{ 0, 0.5 }) |virtual_speedup| {
1270 for (0..2) |_| {
1271 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1272 .selected = .{ .file = "src/work.zig", .line = 10 },
1273 .virtual_speedup = virtual_speedup,
1274 .duration_ns = if (virtual_speedup == 0) 100 else 80,
1275 .selected_samples = 1,
1276 } });
1277 try accumulator.observe(std.testing.allocator, .{ .throughput = .{
1278 .name = "items",
1279 .delta = 10,
1280 } });
1281 }
1282 }
1283
1284 var summary = try accumulator.summarize(
1285 std.testing.allocator,
1286 .{ .min_delta = 1, .min_points = 2 },
1287 );
1288 defer summary.deinit(std.testing.allocator);
1289
1290 try std.testing.expectEqual(@as(usize, 1), summary.results.len);
1291 try std.testing.expectEqual(@as(u64, 4), summary.results[0].total_selected_samples);
1292 try std.testing.expectEqual(
1293 SupportStatus.within_run_repeated_curve,
1294 summary.results[0].support.status,
1295 );
1296 try std.testing.expectEqual(@as(u64, 2), summary.results[0].support.speedup_point_count);
1297 try std.testing.expectEqual(@as(u64, 4), summary.results[0].support.experiment_count);
1298 try std.testing.expectEqual(@as(u64, 2), summary.results[0].support.baseline_experiment_count);
1299 try std.testing.expectEqual(
1300 @as(u64, 2),
1301 summary.results[0].support.minimum_experiments_per_point,
1302 );
1303 const measurement = summary.results[0].measurements[0];
1304 try std.testing.expectEqual(@as(u64, 2), measurement.experiment_count);
1305 try std.testing.expectEqual(@as(u64, 2), measurement.selected_samples);
1306 try std.testing.expectEqual(@as(u64, 20), measurement.observations);
1307 try std.testing.expectEqual(@as(u64, 20), measurement.throughput_delta.?);
1308 try std.testing.expectEqual(@as(u64, 200), measurement.duration_ns);
1309 try std.testing.expectEqual(@as(f64, 10), measurement.value_ns);
1310 }
1311
1312 test "accumulator requires a zero-speedup baseline after low-delta filtering" {
1313 var accumulator: Accumulator = .{};
1314 defer accumulator.deinit(std.testing.allocator);
1315
1316 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1317 .selected = .{ .file = "src/work.zig", .line = 10 },
1318 .virtual_speedup = 0,
1319 .duration_ns = 100,
1320 .selected_samples = 1,
1321 } });
1322 try accumulator.observe(std.testing.allocator, .{ .throughput = .{ .name = "items", .delta = 1 } });
1323 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1324 .selected = .{ .file = "src/work.zig", .line = 10 },
1325 .virtual_speedup = 0.25,
1326 .duration_ns = 90,
1327 .selected_samples = 2,
1328 } });
1329 try accumulator.observe(std.testing.allocator, .{ .throughput = .{ .name = "items", .delta = 10 } });
1330 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1331 .selected = .{ .file = "src/work.zig", .line = 10 },
1332 .virtual_speedup = 0.5,
1333 .duration_ns = 80,
1334 .selected_samples = 3,
1335 } });
1336 try accumulator.observe(std.testing.allocator, .{ .throughput = .{ .name = "items", .delta = 10 } });
1337
1338 var summary = try accumulator.summarize(std.testing.allocator, .{ .min_delta = 5, .min_points = 2 });
1339 defer summary.deinit(std.testing.allocator);
1340
1341 try std.testing.expectEqual(@as(usize, 0), summary.results.len);
1342 }
1343
1344 test "summary JSON exposes agent-friendly result fields" {
1345 var accumulator: Accumulator = .{};
1346 defer accumulator.deinit(std.testing.allocator);
1347
1348 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1349 .selected = .{ .file = "src/work.zig", .line = 10 },
1350 .virtual_speedup = 0,
1351 .duration_ns = 100,
1352 .selected_samples = 1,
1353 } });
1354 try accumulator.observe(std.testing.allocator, .{ .throughput = .{ .name = "items", .delta = 10 } });
1355
1356 var summary = try accumulator.summarize(std.testing.allocator, .{ .min_delta = 1, .min_points = 1 });
1357 defer summary.deinit(std.testing.allocator);
1358
1359 var buffer: [2048]u8 = undefined;
1360 var writer = std.Io.Writer.fixed(&buffer);
1361 try summary.writeJson(&writer);
1362 const json = writer.buffered();
1363
1364 try std.testing.expect(std.mem.indexOf(u8, json, "\"schema\":\"coz.analysis/v2\"") != null);
1365 try std.testing.expect(std.mem.indexOf(
1366 u8,
1367 json,
1368 "\"status\":\"missing_sampling_summary\"",
1369 ) != null);
1370 try std.testing.expect(std.mem.indexOf(u8, json, "\"progress_point\":\"items\"") != null);
1371 try std.testing.expect(std.mem.indexOf(u8, json, "\"kind\":\"throughput\"") != null);
1372 try std.testing.expect(std.mem.indexOf(u8, json, "\"goal\":\"maximize\"") != null);
1373 try std.testing.expect(std.mem.indexOf(u8, json, "\"baseline_value_ns\":10") != null);
1374 try std.testing.expect(std.mem.indexOf(u8, json, "\"baseline_period_ns\":10") != null);
1375 try std.testing.expect(std.mem.indexOf(u8, json, "\"total_selected_samples\":1") != null);
1376 try std.testing.expect(std.mem.indexOf(u8, json, "\"status\":\"point_only\"") != null);
1377 try std.testing.expect(std.mem.indexOf(u8, json, "\"experiment_count\":1") != null);
1378 try std.testing.expect(std.mem.indexOf(
1379 u8,
1380 json,
1381 "\"action\":\"collect_baseline_and_perturbed_points\"",
1382 ) != null);
1383 try std.testing.expect(std.mem.indexOf(u8, json, "\"selected_samples\":1") != null);
1384 try std.testing.expect(std.mem.indexOf(u8, json, "\"measurements\"") != null);
1385 }
1386
1387 test "summary JSON exposes latency-specific agent fields" {
1388 var accumulator: Accumulator = .{};
1389 defer accumulator.deinit(std.testing.allocator);
1390
1391 try accumulator.observe(std.testing.allocator, .{ .experiment = .{
1392 .selected = .{ .file = "src/work.zig", .line = 10 },
1393 .virtual_speedup = 0,
1394 .duration_ns = 100,
1395 .selected_samples = 1,
1396 } });
1397 try accumulator.observe(std.testing.allocator, .{ .latency = .{
1398 .name = "request",
1399 .arrivals = 10,
1400 .departures = 8,
1401 .outstanding = 4,
1402 } });
1403
1404 var summary = try accumulator.summarize(std.testing.allocator, .{ .min_delta = 1, .min_points = 1 });
1405 defer summary.deinit(std.testing.allocator);
1406
1407 var buffer: [2048]u8 = undefined;
1408 var writer = std.Io.Writer.fixed(&buffer);
1409 try summary.writeJson(&writer);
1410 const json = writer.buffered();
1411
1412 try std.testing.expect(std.mem.indexOf(u8, json, "\"kind\":\"latency\"") != null);
1413 try std.testing.expect(std.mem.indexOf(u8, json, "\"goal\":\"minimize\"") != null);
1414 try std.testing.expect(std.mem.indexOf(u8, json, "\"value_name\":\"average_latency_ns\"") != null);
1415 try std.testing.expect(std.mem.indexOf(u8, json, "\"arrivals\":10") != null);
1416 try std.testing.expect(std.mem.indexOf(u8, json, "\"departures\":8") != null);
1417 try std.testing.expect(std.mem.indexOf(u8, json, "\"average_latency_ns\":40") != null);
1418 try std.testing.expect(std.mem.indexOf(u8, json, "\"total_selected_samples\":1") != null);
1419 try std.testing.expect(std.mem.indexOf(u8, json, "\"selected_samples\":1") != null);
1420 }
1421
1422 test "summary can be built from profile JSON lines" {
1423 const text =
1424 \\{"schema":"coz.profile/v1","event":"startup","timestamp_ns":1}
1425 \\{"schema":"coz.profile/v1","event":"experiment","selected":{"file":"src/work.zig","line":10},"virtual_speedup":0,"duration_ns":100,"selected_samples":1}
1426 \\{"schema":"coz.profile/v1","event":"throughput","name":"items","delta":10}
1427 \\{"schema":"coz.profile/v1","event":"experiment","selected":{"file":"src/work.zig","line":10},"virtual_speedup":0.5,"duration_ns":80,"selected_samples":2}
1428 \\{"schema":"coz.profile/v1","event":"throughput","name":"items","delta":10}
1429 \\{"schema":"coz.profile/v1","event":"runtime","duration_ns":200}
1430 \\
1431 ;
1432
1433 var summary = try summarizeJsonLines(std.testing.allocator, text, .{ .min_delta = 1, .min_points = 2 });
1434 defer summary.deinit(std.testing.allocator);
1435
1436 try std.testing.expectEqual(@as(usize, 1), summary.results.len);
1437 try std.testing.expectEqualStrings("src/work.zig", summary.results[0].selected.file);
1438 try std.testing.expectEqualStrings("items", summary.results[0].progress_point);
1439 try std.testing.expectApproxEqAbs(@as(f64, 0.2), summary.results[0].max_program_speedup, 0.000001);
1440 try std.testing.expectEqualStrings(
1441 "missing_sampling_summary",
1442 summary.capture_integrity.status,
1443 );
1444 }
1445
1446 test "capture integrity distinguishes complete loss throttle and counter gaps" {
1447 const complete = captureIntegrity(.{
1448 .record_count = 20,
1449 .sample_record_count = 20,
1450 .loss_counter = .{ .available = 0 },
1451 .terminal_status = .complete,
1452 });
1453 const loss = captureIntegrity(.{
1454 .record_count = 21,
1455 .sample_record_count = 20,
1456 .lost_record_count = 1,
1457 .lost_event_count = 3,
1458 .loss_counter = .{ .available = 3 },
1459 .terminal_status = .complete,
1460 });
1461 const throttled = captureIntegrity(.{
1462 .record_count = 22,
1463 .sample_record_count = 20,
1464 .throttle_record_count = 1,
1465 .unthrottle_record_count = 1,
1466 .loss_counter = .{ .available = 0 },
1467 .terminal_status = .complete,
1468 });
1469 const unsupported = captureIntegrity(.{
1470 .record_count = 20,
1471 .sample_record_count = 20,
1472 .loss_counter = .unsupported,
1473 .terminal_status = .complete,
1474 });
1475 const failed = captureIntegrity(.{
1476 .record_count = 20,
1477 .sample_record_count = 20,
1478 .loss_counter = .read_failed,
1479 .terminal_status = .complete,
1480 });
1481 const drain_failed = captureIntegrity(.{
1482 .loss_counter = .{ .available = 0 },
1483 .terminal_status = .drain_failed,
1484 });
1485
1486 try std.testing.expectEqualStrings("complete", complete.status);
1487 try std.testing.expectEqualStrings("sample_loss", loss.status);
1488 try std.testing.expectEqual(@as(?u64, 3), loss.loss_counter_value);
1489 try std.testing.expectEqualStrings("sampling_throttled", throttled.status);
1490 try std.testing.expectEqualStrings("loss_counter_unavailable", unsupported.status);
1491 try std.testing.expectEqualStrings("loss_counter_read_failed", failed.status);
1492 try std.testing.expectEqualStrings("sampling_drain_failed", drain_failed.status);
1493 }
1494
1495 test "capture integrity rejects inconsistent and duplicate sampling summaries" {
1496 const invalid = captureIntegrity(.{
1497 .record_count = 1,
1498 .sample_record_count = 2,
1499 .loss_counter = .{ .available = 0 },
1500 .terminal_status = .complete,
1501 });
1502 try std.testing.expectEqualStrings("invalid_sampling_summary", invalid.status);
1503
1504 var accumulator: Accumulator = .{};
1505 defer accumulator.deinit(std.testing.allocator);
1506 const sampling = profile.Event{ .sampling = .{
1507 .loss_counter = .{ .available = 0 },
1508 .terminal_status = .complete,
1509 } };
1510 try accumulator.observe(std.testing.allocator, sampling);
1511 try std.testing.expectError(
1512 error.DuplicateSamplingSummary,
1513 accumulator.observe(std.testing.allocator, sampling),
1514 );
1515 }
1516
1517 test "summary pretty report renders profiler analysis with grouped metrics" {
1518 var summary = try sampleSummary(std.testing.allocator);
1519 defer summary.deinit(std.testing.allocator);
1520
1521 var buffer: [4096]u8 = undefined;
1522 var writer = std.Io.Writer.fixed(&buffer);
1523 try summary.writePretty(std.testing.allocator, &writer, .{ .width = 360 });
1524
1525 try std.testing.expectEqualStrings(
1526 "coz profiler: integrity=complete; results=1\n" ++
1527 " src/work.zig:10 -> items: kind=throughput; " ++
1528 "support=unreplicated_curve; speedup_points=2; experiments=2; " ++
1529 "baseline_experiments=1; " ++
1530 "minimum_experiments_per_point=1; baseline_virtual_speedup=0.00%; " ++
1531 "baseline_period=10.00 ns; max_program_speedup=20.00%; " ++
1532 "total_selected_samples=5; slope=0.4000\n" ++
1533 " 1. virtual_speedup=0.00%; program_speedup=0.00%; " ++
1534 "experiments=1; selected_samples=2; delta=10; duration=100 ns; " ++
1535 "period=10.00 ns\n" ++
1536 " 2. virtual_speedup=50.00%; program_speedup=20.00%; " ++
1537 "experiments=1; selected_samples=3; delta=10; duration=80 ns; " ++
1538 "period=8.00 ns\n",
1539 writer.buffered(),
1540 );
1541 }
1542
1543 test "summary pretty report breaks metric groups under narrow widths" {
1544 var summary = try sampleSummary(std.testing.allocator);
1545 defer summary.deinit(std.testing.allocator);
1546
1547 var buffer: [4096]u8 = undefined;
1548 var writer = std.Io.Writer.fixed(&buffer);
1549 try summary.writePretty(std.testing.allocator, &writer, .{ .width = 48 });
1550 const text = writer.buffered();
1551
1552 try std.testing.expect(std.mem.indexOf(u8, text, "\n baseline_period=10.00 ns") != null);
1553 try std.testing.expect(std.mem.indexOf(u8, text, "\n program_speedup=20.00%") != null);
1554 }
1555
1556 fn sampleSummary(allocator: std.mem.Allocator) !Summary {
1557 var accumulator: Accumulator = .{};
1558 defer accumulator.deinit(allocator);
1559
1560 try accumulator.observe(allocator, .{ .experiment = .{
1561 .selected = .{ .file = "src/work.zig", .line = 10 },
1562 .virtual_speedup = 0,
1563 .duration_ns = 100,
1564 .selected_samples = 2,
1565 } });
1566 try accumulator.observe(allocator, .{ .throughput = .{
1567 .name = "items",
1568 .delta = 10,
1569 } });
1570 try accumulator.observe(allocator, .{ .experiment = .{
1571 .selected = .{ .file = "src/work.zig", .line = 10 },
1572 .virtual_speedup = 0.5,
1573 .duration_ns = 80,
1574 .selected_samples = 3,
1575 } });
1576 try accumulator.observe(allocator, .{ .throughput = .{
1577 .name = "items",
1578 .delta = 10,
1579 } });
1580 try accumulator.observe(allocator, .{ .sampling = .{
1581 .loss_counter = .{ .available = 0 },
1582 .terminal_status = .complete,
1583 } });
1584
1585 return try accumulator.summarize(allocator, .{ .min_delta = 1, .min_points = 2 });
1586 }