lib/bench/src/summary.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const pretty_json = @import("pretty").json;
3
4 const jsonl_mod = @import("jsonl.zig");
5 const stats_mod = @import("stats/root.zig");
6
7 pub const Unit = jsonl_mod.Unit;
8
9 pub const Record = struct {
10 benchmark: []const u8,
11 summary: []const u8,
12 name: []const u8,
13 samples: u64,
14 stats: stats_mod.SampleStats,
15 units: []const Unit,
16 };
17
18 pub const PassIrMeans = struct {
19 before: f64,
20 after: f64,
21 delta: f64,
22 };
23
24 pub const PassMemoryMeans = struct {
25 alloc_count: f64,
26 free_count: f64,
27 alloc_bytes: f64,
28 };
29
30 pub const PassRecord = struct {
31 benchmark: []const u8,
32 name: []const u8,
33 samples: u64,
34 pass_ir_sizes: bool,
35 stats: stats_mod.SampleStats,
36 units: []const Unit,
37 ir: PassIrMeans,
38 memory: PassMemoryMeans,
39 };
40
41 pub fn passIrMeans(totals: anytype, index: usize, sample_count: u32) PassIrMeans {
42 return .{
43 .before = totals.meanBefore(index, sample_count),
44 .after = totals.meanAfter(index, sample_count),
45 .delta = totals.meanDelta(index, sample_count),
46 };
47 }
48
49 pub fn passMemoryMeans(totals: anytype, index: usize, sample_count: u32) PassMemoryMeans {
50 return .{
51 .alloc_count = totals.meanAllocCount(index, sample_count),
52 .free_count = totals.meanFreeCount(index, sample_count),
53 .alloc_bytes = totals.meanAllocBytes(index, sample_count),
54 };
55 }
56
57 pub fn writeTiming(writer: *std.Io.Writer, record: Record, identity: anytype) !void {
58 var stream = pretty_json.Writer.init(writer, .minified);
59 const object = try stream.object();
60 try writeHeader(object, record.benchmark, record.summary, record.name);
61 try identity.writeBeforeSamples(object);
62 try object.field("samples", record.samples);
63 try identity.writeAfterSamples(object);
64 try writeTimingStats(object, record.stats, record.units);
65 try object.endLine();
66 }
67
68 pub fn writeMemory(writer: *std.Io.Writer, record: Record, identity: anytype) !void {
69 var stream = pretty_json.Writer.init(writer, .minified);
70 const object = try stream.object();
71 try writeHeader(object, record.benchmark, "memory", record.name);
72 try identity.writeBeforeSamples(object);
73 try object.field("samples", record.samples);
74 try identity.writeAfterSamples(object);
75 try writeMemoryStats(object, record.stats, record.units);
76 try object.endLine();
77 }
78
79 pub fn writePass(writer: *std.Io.Writer, record: PassRecord, identity: anytype) !void {
80 var stream = pretty_json.Writer.init(writer, .minified);
81 const object = try stream.object();
82 try writeHeader(object, record.benchmark, "pass", record.name);
83 try identity.writeBeforeSamples(object);
84 try object.field("samples", record.samples);
85 try identity.writeAfterSamples(object);
86 try object.field("pass_ir_sizes", record.pass_ir_sizes);
87 try object.field("min_ns", record.stats.min_ns);
88 try object.field("median_ns", record.stats.median_ns);
89 try object.field("mean_ns", record.stats.mean_ns);
90 try jsonl_mod.writePerUnitFields(object, "mean_ns", record.stats.mean_ns, record.units);
91 try object.field("mean_pass_choir_ops_before", record.ir.before);
92 try object.field("mean_pass_choir_ops_after", record.ir.after);
93 try object.field("mean_pass_choir_op_delta", record.ir.delta);
94 try object.field("mean_alloc_count", record.memory.alloc_count);
95 try object.field("mean_free_count", record.memory.free_count);
96 try object.field("mean_alloc_bytes", record.memory.alloc_bytes);
97 try jsonl_mod.writePerUnitFields(
98 object,
99 "mean_alloc_bytes",
100 record.memory.alloc_bytes,
101 record.units,
102 );
103 try writePassRates(object, record);
104 try writeTimingTail(object, record.stats);
105 try object.endLine();
106 }
107
108 fn writePassRates(object: pretty_json.Object, record: PassRecord) !void {
109 try object.field(
110 "mean_alloc_bytes_per_pass_choir_op_before",
111 stats_mod.valuePerFloatUnit(record.memory.alloc_bytes, record.ir.before),
112 );
113 try object.field(
114 "mean_alloc_bytes_per_pass_choir_op_after",
115 stats_mod.valuePerFloatUnit(record.memory.alloc_bytes, record.ir.after),
116 );
117 try object.field(
118 "mean_ns_per_pass_choir_op_before",
119 stats_mod.nsPerFloatUnit(record.stats.mean_ns, record.ir.before),
120 );
121 try object.field(
122 "mean_ns_per_pass_choir_op_after",
123 stats_mod.nsPerFloatUnit(record.stats.mean_ns, record.ir.after),
124 );
125 }
126
127 fn writeHeader(
128 object: pretty_json.Object,
129 benchmark: []const u8,
130 summary: []const u8,
131 name: []const u8,
132 ) !void {
133 try object.field("kind", "summary");
134 try object.field("benchmark", benchmark);
135 try object.field("summary", summary);
136 try object.field("name", name);
137 }
138
139 fn writeTimingStats(
140 object: pretty_json.Object,
141 sample_stats: stats_mod.SampleStats,
142 units: []const Unit,
143 ) !void {
144 try object.field("min_ns", sample_stats.min_ns);
145 try object.field("median_ns", sample_stats.median_ns);
146 try object.field("mean_ns", sample_stats.mean_ns);
147 try jsonl_mod.writePerUnitFields(object, "mean_ns", sample_stats.mean_ns, units);
148 try writeTimingTail(object, sample_stats);
149 }
150
151 fn writeTimingTail(object: pretty_json.Object, sample_stats: stats_mod.SampleStats) !void {
152 try object.field("p95_ns", sample_stats.p95_ns);
153 try object.field("p99_ns", sample_stats.p99_ns);
154 try object.field("max_ns", sample_stats.max_ns);
155 try object.field("total_ns", sample_stats.total_ns);
156 }
157
158 fn writeMemoryStats(
159 object: pretty_json.Object,
160 sample_stats: stats_mod.SampleStats,
161 units: []const Unit,
162 ) !void {
163 try object.field("min", sample_stats.min_ns);
164 try object.field("median", sample_stats.median_ns);
165 try object.field("mean", sample_stats.mean_ns);
166 try jsonl_mod.writePerUnitFields(object, "mean", sample_stats.mean_ns, units);
167 try object.field("p95", sample_stats.p95_ns);
168 try object.field("p99", sample_stats.p99_ns);
169 try object.field("max", sample_stats.max_ns);
170 try object.field("total", sample_stats.total_ns);
171 }
172
173 const TestIdentity = struct {
174 pub fn writeBeforeSamples(_: @This(), object: pretty_json.Object) !void {
175 try object.field("workload", "demo");
176 }
177
178 pub fn writeAfterSamples(_: @This(), object: pretty_json.Object) !void {
179 try object.field("source_ops", 4);
180 }
181 };
182
183 test "timing summary writes stats and unit rates" {
184 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
185 defer out.deinit();
186
187 const units = [_]Unit{
188 .{ .name = "source_op", .count = 4 },
189 .{ .name = "final_choir_op", .count = 8 },
190 };
191 try writeTiming(&out.writer, .{
192 .benchmark = "bench.demo",
193 .summary = "phase",
194 .name = "total",
195 .samples = 2,
196 .stats = sampleStats(),
197 .units = &units,
198 }, TestIdentity{});
199
200 const text = out.written();
201 try std.testing.expect(std.mem.indexOf(u8, text, "\"summary\":\"phase\"") != null);
202 try std.testing.expect(std.mem.indexOf(u8, text, "\"workload\":\"demo\"") != null);
203 try std.testing.expect(std.mem.indexOf(u8, text, "\"mean_ns_per_source_op\":30") != null);
204 try std.testing.expect(std.mem.indexOf(u8, text, "\"mean_ns_per_final_choir_op\":15") != null);
205 }
206
207 test "pass summary writes IR and allocation means" {
208 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
209 defer out.deinit();
210
211 const units = [_]Unit{.{ .name = "source_op", .count = 4 }};
212 try writePass(&out.writer, .{
213 .benchmark = "bench.demo",
214 .name = "pass",
215 .samples = 2,
216 .pass_ir_sizes = true,
217 .stats = sampleStats(),
218 .units = &units,
219 .ir = .{ .before = 2, .after = 4, .delta = 2 },
220 .memory = .{ .alloc_count = 2, .free_count = 1, .alloc_bytes = 64 },
221 }, TestIdentity{});
222
223 const text = out.written();
224 try std.testing.expect(std.mem.indexOf(u8, text, "\"pass_ir_sizes\":true") != null);
225 try std.testing.expect(std.mem.indexOf(u8, text, "\"mean_pass_choir_ops_before\":2") != null);
226 try std.testing.expect(std.mem.indexOf(u8, text, "\"mean_alloc_bytes_per_source_op\":16") != null);
227 try std.testing.expect(std.mem.indexOf(u8, text, "\"mean_ns_per_pass_choir_op_after\":30") != null);
228 }
229
230 fn sampleStats() stats_mod.SampleStats {
231 return .{
232 .min_ns = 100,
233 .max_ns = 140,
234 .mean_ns = 120,
235 .median_ns = 100,
236 .p75_ns = 140,
237 .p95_ns = 140,
238 .p99_ns = 140,
239 .total_ns = 240,
240 };
241 }