lib/bench/src/output.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const pretty = @import("pretty");
3 const sys = @import("sys");
4
5 const stats = @import("stats/root.zig");
6 const OutputTestEvaluation = union(enum) {
7 automatic: struct { evals: u32 },
8 fixed: u32,
9 };
10 const OutputTestAllocationAttribution = enum {
11 none,
12 sample_call,
13 prepare_owner,
14 };
15 const result_header =
16 "benchmark samples batch median" ++
17 " median ci p75 p99 mean/sd" ++
18 " min..max alloc owner alloc/free alloc bytes";
19 const result_rule =
20 "--------------------------------------------------------------------------------" ++
21 "--------------------------------------------------------------------------------" ++
22 "--------------------------";
23
24 pub fn writeHeader() !void {
25 const parts = [_]pretty.Doc{
26 pretty.hardline,
27 .{ .text = result_header },
28 pretty.hardline,
29 .{ .text = result_rule },
30 pretty.hardline,
31 };
32 try writeStdoutDoc(.{ .concat = &parts });
33 }
34
35 pub fn writeResult(result: anytype) !void {
36 var line_buf: [512]u8 = undefined;
37 const line = try formatResultLine(&line_buf, result);
38 try writeStdoutDoc(.{ .text = line });
39 }
40
41 pub fn section(comptime name: []const u8) void {
42 const parts = [_]pretty.Doc{
43 pretty.hardline,
44 .{ .text = "== " },
45 .{ .text = name },
46 .{ .text = " ==" },
47 pretty.hardline,
48 };
49 writeStdoutDoc(.{ .concat = &parts }) catch {};
50 writeHeader() catch {};
51 }
52
53 pub fn stdout(comptime fmt: []const u8, args: anytype) void {
54 writeStdoutFmt(fmt, args) catch {};
55 }
56
57 pub fn stderr(comptime fmt: []const u8, args: anytype) void {
58 writeStderrFmt(fmt, args) catch {};
59 }
60
61 pub fn writeStdoutFmt(comptime fmt: []const u8, args: anytype) !void {
62 try writeFileFmt(.stdout, fmt, args);
63 }
64
65 pub fn writeStderrFmt(comptime fmt: []const u8, args: anytype) !void {
66 try writeFileFmt(.stderr, fmt, args);
67 }
68
69 pub fn writeStdoutDoc(doc: pretty.Doc) !void {
70 try writeFileDoc(.stdout, doc);
71 }
72
73 pub fn writeStderrDoc(doc: pretty.Doc) !void {
74 try writeFileDoc(.stderr, doc);
75 }
76
77 fn formatResultLine(line_buf: []u8, result: anytype) ![]u8 {
78 var avg_buf: [16]u8 = undefined;
79 var sd_buf: [16]u8 = undefined;
80 var median_buf: [16]u8 = undefined;
81 var median_ci_buf: [40]u8 = undefined;
82 var min_buf: [16]u8 = undefined;
83 var max_buf: [16]u8 = undefined;
84 var p75_buf: [16]u8 = undefined;
85 var p99_buf: [16]u8 = undefined;
86 var alloc_buf: [32]u8 = undefined;
87 var bytes_buf: [32]u8 = undefined;
88 var evaluation_buf: [32]u8 = undefined;
89
90 const avg_str = fmtTime(&avg_buf, result.mean_ns);
91 const sd_str = fmtTime(&sd_buf, result.stddev_ns);
92 const median_str = fmtTime(&median_buf, @floatFromInt(result.median_ns));
93 const median_ci_str = fmtMedianInterval(&median_ci_buf, result);
94 const min_str = fmtTime(&min_buf, @floatFromInt(result.min_ns));
95 const max_str = fmtTime(&max_buf, @floatFromInt(result.max_ns));
96 const p75_str = fmtTime(&p75_buf, @floatFromInt(result.p75_ns));
97 const p99_str = fmtTime(&p99_buf, @floatFromInt(result.p99_ns));
98 const alloc_count = if (@hasField(@TypeOf(result), "alloc_count_per_eval"))
99 result.alloc_count_per_eval orelse result.alloc_count
100 else
101 result.alloc_count;
102 const free_count = if (@hasField(@TypeOf(result), "free_count_per_eval"))
103 result.free_count_per_eval orelse result.free_count
104 else
105 result.free_count;
106 const alloc_bytes = if (@hasField(@TypeOf(result), "alloc_bytes_per_eval"))
107 result.alloc_bytes_per_eval orelse result.alloc_bytes
108 else
109 result.alloc_bytes;
110 const alloc_str = fmtAllocPair(&alloc_buf, alloc_count, free_count);
111 const bytes_str = fmtOptionalBytes(&bytes_buf, alloc_bytes);
112 const evaluation_str = fmtEvaluation(&evaluation_buf, result);
113 const allocation_attribution = fmtAllocationAttribution(result);
114
115 var avg_sd_buf: [40]u8 = undefined;
116 const avg_sd = std.fmt.bufPrint(&avg_sd_buf, "{s} +/- {s}", .{ avg_str, sd_str }) catch "???";
117
118 var range_buf: [40]u8 = undefined;
119 const range = std.fmt.bufPrint(&range_buf, "({s} ... {s})", .{ min_str, max_str }) catch "???";
120
121 return try std.fmt.bufPrint(
122 line_buf,
123 "{s:<39} {d:>10} {s:>12} {s:>12} {s:>18} {s:>10} {s:>10} {s:>22} {s:>20} {s:>13} {s:>12} {s:>12}\n",
124 .{
125 result.name,
126 result.iterations,
127 evaluation_str,
128 median_str,
129 median_ci_str,
130 p75_str,
131 p99_str,
132 avg_sd,
133 range,
134 allocation_attribution,
135 alloc_str,
136 bytes_str,
137 },
138 );
139 }
140
141 fn fmtAllocationAttribution(result: anytype) []const u8 {
142 if (comptime @hasField(@TypeOf(result), "allocation_attribution")) {
143 return switch (result.allocation_attribution) {
144 .none => "-",
145 .sample_call => "sample-call",
146 .prepare_owner => "prepare-owner",
147 };
148 }
149 return "-";
150 }
151
152 fn fmtEvaluation(buf: []u8, result: anytype) []const u8 {
153 if (comptime @hasField(@TypeOf(result), "evaluation")) {
154 return switch (result.evaluation) {
155 .automatic => |selection| std.fmt.bufPrint(
156 buf,
157 "auto:{d}",
158 .{selection.evals},
159 ) catch "?",
160 .fixed => |evals| std.fmt.bufPrint(buf, "fixed:{d}", .{evals}) catch "?",
161 };
162 }
163 if (comptime @hasField(@TypeOf(result), "evals")) {
164 return std.fmt.bufPrint(buf, "{d}", .{result.evals}) catch "?";
165 }
166 return "-";
167 }
168
169 fn fmtMedianInterval(buf: []u8, result: anytype) []const u8 {
170 if (comptime @hasField(@TypeOf(result), "confidence_intervals")) {
171 if (result.confidence_intervals) |intervals| {
172 return fmtTimeInterval(buf, intervals.median_ns.low_ns, intervals.median_ns.high_ns);
173 }
174 }
175 return "-";
176 }
177
178 fn fmtTimeInterval(buf: []u8, low_ns: f64, high_ns: f64) []const u8 {
179 var low_buf: [16]u8 = undefined;
180 var high_buf: [16]u8 = undefined;
181 const low = fmtTime(&low_buf, low_ns);
182 const high = fmtTime(&high_buf, high_ns);
183 return std.fmt.bufPrint(buf, "{s}..{s}", .{ low, high }) catch "?";
184 }
185
186 fn fmtAllocPair(buf: []u8, alloc_count: ?u64, free_count: ?u64) []const u8 {
187 const allocs = alloc_count orelse return "-";
188 const frees = free_count orelse return "-";
189 return std.fmt.bufPrint(buf, "{d}/{d}", .{ allocs, frees }) catch "?";
190 }
191
192 fn fmtOptionalBytes(buf: []u8, alloc_bytes: ?u64) []const u8 {
193 const bytes = alloc_bytes orelse return "-";
194 if (bytes < 1024) return std.fmt.bufPrint(buf, "{d}B", .{bytes}) catch "?";
195 if (bytes < 1024 * 1024) {
196 const kib = @as(f64, @floatFromInt(bytes)) / 1024.0;
197 return std.fmt.bufPrint(buf, "{d:.1}KiB", .{kib}) catch "?";
198 }
199 if (bytes < 1024 * 1024 * 1024) {
200 const mib = @as(f64, @floatFromInt(bytes)) / (1024.0 * 1024.0);
201 return std.fmt.bufPrint(buf, "{d:.1}MiB", .{mib}) catch "?";
202 }
203 const gib = @as(f64, @floatFromInt(bytes)) / (1024.0 * 1024.0 * 1024.0);
204 return std.fmt.bufPrint(buf, "{d:.1}GiB", .{gib}) catch "?";
205 }
206
207 fn fmtTime(buf: []u8, ns: f64) []const u8 {
208 if (ns < 1_000) {
209 return std.fmt.bufPrint(buf, "{d:.0}ns", .{ns}) catch "?";
210 } else if (ns < 1_000_000) {
211 return std.fmt.bufPrint(buf, "{d:.1}us", .{ns / 1_000.0}) catch "?";
212 } else if (ns < 1_000_000_000) {
213 return std.fmt.bufPrint(buf, "{d:.2}ms", .{ns / 1_000_000.0}) catch "?";
214 } else {
215 return std.fmt.bufPrint(buf, "{d:.2}s", .{ns / 1_000_000_000.0}) catch "?";
216 }
217 }
218
219 const OutputTarget = enum {
220 stdout,
221 stderr,
222 };
223
224 fn writeFileFmt(target: OutputTarget, comptime fmt: []const u8, args: anytype) !void {
225 var buffer: [8192]u8 = undefined;
226 const text = try std.fmt.bufPrint(&buffer, fmt, args);
227 try writeFileDoc(target, .{ .text = text });
228 }
229
230 fn writeFileDoc(target: OutputTarget, doc: pretty.Doc) !void {
231 const file = switch (target) {
232 .stdout => sys.stdio.stdout(),
233 .stderr => sys.stdio.stderr(),
234 };
235 try writeDocToFile(file, doc);
236 }
237
238 fn writeDocToFile(file: std.Io.File, doc: pretty.Doc) !void {
239 var buffer: [4096]u8 = undefined;
240 var file_writer = file.writerStreaming(sys.stdio.debugIo(), &buffer);
241 try pretty.write(&file_writer.interface, doc, .{});
242 try file_writer.interface.flush();
243 }
244
245 test "time formatting ranges" {
246 var buf: [16]u8 = undefined;
247 const ns = fmtTime(&buf, 500);
248 try std.testing.expect(std.mem.endsWith(u8, ns, "ns"));
249
250 var buf2: [16]u8 = undefined;
251 const us = fmtTime(&buf2, 5_000);
252 try std.testing.expect(std.mem.endsWith(u8, us, "us"));
253
254 var buf3: [16]u8 = undefined;
255 const ms = fmtTime(&buf3, 5_000_000);
256 try std.testing.expect(std.mem.endsWith(u8, ms, "ms"));
257 }
258
259 test "streaming document writes preserve regular file contents" {
260 var tmp = std.testing.tmpDir(.{});
261 defer tmp.cleanup();
262 const io = sys.stdio.debugIo();
263 const file = try tmp.dir.createFile(io, "output.txt", .{ .read = true });
264 defer file.close(io);
265
266 try writeDocToFile(file, .{ .text = "first" });
267 try writeDocToFile(file, .{ .text = " second" });
268
269 var contents: [32]u8 = undefined;
270 const length = try file.readPositionalAll(io, &contents, 0);
271 try std.testing.expectEqualStrings("first second", contents[0..length]);
272 }
273
274 test "result line renders timing and allocation columns" {
275 const row = .{
276 .name = "alloc",
277 .iterations = @as(u32, 2),
278 .evaluation = OutputTestEvaluation{ .automatic = .{ .evals = 4 } },
279 .mean_ns = @as(f64, 1_500),
280 .median_ns = @as(u64, 1_500),
281 .stddev_ns = @as(f64, 500),
282 .min_ns = @as(u64, 1_000),
283 .max_ns = @as(u64, 2_000),
284 .p75_ns = @as(u64, 1_750),
285 .p99_ns = @as(u64, 2_000),
286 .confidence_intervals = @as(?stats.BootstrapConfidenceIntervals, null),
287 .allocation_attribution = OutputTestAllocationAttribution.prepare_owner,
288 .alloc_count = @as(?u64, 2),
289 .free_count = @as(?u64, 2),
290 .alloc_bytes = @as(?u64, 64),
291 };
292 var line_buf: [512]u8 = undefined;
293 const line = try formatResultLine(&line_buf, row);
294
295 try std.testing.expect(std.mem.indexOf(u8, line, "alloc") != null);
296 try std.testing.expect(std.mem.indexOf(u8, line, "auto:4") != null);
297 try std.testing.expect(std.mem.indexOf(u8, line, "1.5us") != null);
298 try std.testing.expect(std.mem.indexOf(u8, line, "1.5us +/- 500ns") != null);
299 try std.testing.expect(std.mem.indexOf(u8, line, "(1.0us ... 2.0us)") != null);
300 try std.testing.expect(std.mem.indexOf(u8, line, "prepare-owner") != null);
301 try std.testing.expect(std.mem.indexOf(u8, line, "2/2") != null);
302 try std.testing.expect(std.mem.indexOf(u8, line, "64B") != null);
303 }
304
305 test "result line renders median confidence interval when present" {
306 const row = .{
307 .name = "uncertain",
308 .iterations = @as(u32, 4),
309 .mean_ns = @as(f64, 1_500),
310 .median_ns = @as(u64, 1_500),
311 .stddev_ns = @as(f64, 500),
312 .min_ns = @as(u64, 1_000),
313 .max_ns = @as(u64, 2_000),
314 .p75_ns = @as(u64, 1_750),
315 .p99_ns = @as(u64, 2_000),
316 .confidence_intervals = @as(?stats.BootstrapConfidenceIntervals, .{
317 .confidence = 0.95,
318 .iterations = 1000,
319 .seed = 7,
320 .mean_ns = .{ .low_ns = 1_000, .high_ns = 2_000 },
321 .median_ns = .{ .low_ns = 1_000, .high_ns = 2_000 },
322 .p75_ns = .{ .low_ns = 1_500, .high_ns = 2_000 },
323 .p95_ns = .{ .low_ns = 2_000, .high_ns = 2_000 },
324 .p99_ns = .{ .low_ns = 2_000, .high_ns = 2_000 },
325 }),
326 .alloc_count = @as(?u64, null),
327 .free_count = @as(?u64, null),
328 .alloc_bytes = @as(?u64, null),
329 };
330 var line_buf: [512]u8 = undefined;
331 const line = try formatResultLine(&line_buf, row);
332
333 try std.testing.expect(std.mem.indexOf(u8, line, "1.0us..2.0us") != null);
334 }
335
336 test "missing allocation columns render as dashes" {
337 const row = .{
338 .name = "noop",
339 .iterations = @as(u32, 1),
340 .mean_ns = @as(f64, 10),
341 .median_ns = @as(u64, 10),
342 .stddev_ns = @as(f64, 0),
343 .min_ns = @as(u64, 10),
344 .max_ns = @as(u64, 10),
345 .p75_ns = @as(u64, 10),
346 .p99_ns = @as(u64, 10),
347 .alloc_count = @as(?u64, null),
348 .free_count = @as(?u64, null),
349 .alloc_bytes = @as(?u64, null),
350 };
351 var line_buf: [512]u8 = undefined;
352 const line = try formatResultLine(&line_buf, row);
353
354 try std.testing.expect(std.mem.indexOf(u8, line, "noop") != null);
355 try std.testing.expect(std.mem.indexOf(u8, line, " - -") != null);
356 }