lib/choir/src/profiling/versus/runner.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const alloc_arena = @import("alloc_arena");
3 const builtin = @import("builtin");
4 const bench = @import("bench");
5 const choir = @import("choir");
6 const sys = @import("sys");
7
8 const abi = choir.versus.abi;
9 const cc = @import("cc.zig");
10 const compile_mod = choir.versus.compile;
11 const jsonl = choir.versus.jsonl;
12 const kernels = @import("kernels.zig");
13 const oracle = choir.versus.oracle;
14 const systems = @import("systems.zig");
15 const workload_mod = choir.versus.workload;
16
17 const coz = bench.coz;
18 const Allocator = std.mem.Allocator;
19 const Workload = workload_mod.Workload;
20 const Backend = choir.backends.x86_64.backend.Backend;
21 const AllocationSnapshot = choir.passes.PassAllocationSnapshot;
22 const PassRunSummary = choir.passes.PassRunTimingSummary;
23
24 const host = @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag);
25 const counter_selectors = [_]sys.perf.CounterSelector{
26 .{ .hardware = .instructions },
27 .{ .hardware = .cycles },
28 };
29 const PerfCounterSet = sys.perf.CounterSetRegion(counter_selectors.len);
30
31 const supports_choir_execution = sys.capabilities.current.supportsX86_64Execution();
32
33 const Timer = struct {
34 start_ns: i128,
35
36 fn start() Timer {
37 return .{ .start_ns = sys.time.nanoTimestamp() };
38 }
39
40 fn reset(self: *Timer) void {
41 self.start_ns = sys.time.nanoTimestamp();
42 }
43
44 fn read(self: *const Timer) u64 {
45 const end = sys.time.nanoTimestamp();
46 if (end <= self.start_ns) return 0;
47 const elapsed = end - self.start_ns;
48 if (elapsed > std.math.maxInt(u64)) return std.math.maxInt(u64);
49 return @intCast(elapsed);
50 }
51 };
52
53 const ChoirCompileTimes = struct {
54 ir_ns: u64,
55 pass_ns: u64,
56 jit_ns: u64,
57 size: IrSize,
58 pass_count: u64,
59 pass_runs: []const PassRunSummary = &.{},
60
61 fn total(self: ChoirCompileTimes) u64 {
62 return self.ir_ns + self.pass_ns + self.jit_ns;
63 }
64 };
65
66 const CompileStats = struct {
67 samples: u32,
68 median_ns: u64,
69 p10_ns: u64,
70 p90_ns: u64,
71 ir_ns: u64 = 0,
72 ir_p10_ns: u64 = 0,
73 ir_p90_ns: u64 = 0,
74 pass_ns: u64 = 0,
75 pass_p10_ns: u64 = 0,
76 pass_p90_ns: u64 = 0,
77 jit_ns: u64 = 0,
78 jit_p10_ns: u64 = 0,
79 jit_p90_ns: u64 = 0,
80 alloc_count: u64 = 0,
81 alloc_count_p10: u64 = 0,
82 alloc_count_p90: u64 = 0,
83 alloc_bytes: u64 = 0,
84 alloc_bytes_p10: u64 = 0,
85 alloc_bytes_p90: u64 = 0,
86 ir_ops: u64 = 0,
87 ir_blocks: u64 = 0,
88 ir_values: u64 = 0,
89 pass_count: u64 = 0,
90 };
91
92 pub const Options = struct {
93 samples: u32 = 20,
94 warmup: u32 = 3,
95 workload: ?[]const u8 = null,
96 verify: bool = true,
97 counters: bool = true,
98 raw_lane: bool = true,
99 external: bool = true,
100 compile_only: bool = false,
101 compile_matrix: bool = false,
102 trace_compile_threshold_ns: u64 = 0,
103 workdir: []const u8 = "/tmp/choir-versus-work",
104 };
105
106 const IrSize = struct {
107 ops: u64 = 0,
108 blocks: u64 = 0,
109 values: u64 = 0,
110 };
111
112 const CompileSubject = union(enum) {
113 kernel: Workload,
114 generated: compile_mod.Workload,
115
116 fn name(self: CompileSubject) []const u8 {
117 return switch (self) {
118 .kernel => |workload| workload.name,
119 .generated => |workload| workload.name,
120 };
121 }
122
123 fn flops(self: CompileSubject) u64 {
124 return switch (self) {
125 .kernel => |workload| workload.flops(),
126 .generated => 0,
127 };
128 }
129
130 fn movedBytes(self: CompileSubject) u64 {
131 return switch (self) {
132 .kernel => |workload| workload.movedBytes(),
133 .generated => 0,
134 };
135 }
136
137 fn build(self: CompileSubject, ctx: *choir.Context) !choir.dialects.BuiltinDialect.ModuleOp {
138 return switch (self) {
139 .kernel => |workload| try kernels.build(ctx, workload.kind),
140 .generated => |workload| try compile_mod.build(ctx, workload),
141 };
142 }
143 };
144
145 pub fn run(arena: Allocator, backing: Allocator, out: *std.Io.Writer, environ: std.process.Environ, options: Options) !u8 {
146 sys.fs.createDirPath(options.workdir) catch return error.WorkdirUnavailable;
147
148 try jsonl.writeMeta(out, "harness", host, "one process measures every lane through the same call loop");
149 if (!supports_choir_execution) {
150 try jsonl.writeMeta(out, systems.choir, host, "unavailable: host cannot execute x86_64 JIT code");
151 }
152
153 var counters_state = CountersState{ .enabled = options.counters };
154
155 for (workload_mod.battery) |workload| {
156 if (options.workload) |filter| {
157 if (!std.mem.eql(u8, filter, workload.name)) continue;
158 }
159
160 var buffers_storage: oracle.Buffers = undefined;
161 var buffers: ?*oracle.Buffers = null;
162 if (!options.compile_only) {
163 buffers_storage = try oracle.Buffers.alloc(backing, workload);
164 buffers = &buffers_storage;
165 }
166 defer if (buffers) |actual| actual.deinit(backing);
167
168 if (supports_choir_execution) {
169 runChoir(arena, backing, out, environ, workload, buffers, options, &counters_state, true) catch |err| {
170 try writeErrorRow(out, systems.choir, workload, err);
171 };
172 if (options.raw_lane) {
173 runChoir(arena, backing, out, environ, workload, buffers, options, &counters_state, false) catch |err| {
174 try writeErrorRow(out, systems.choir_raw, workload, err);
175 };
176 }
177 }
178
179 if (options.external) {
180 for (cc.default_lanes) |lane| {
181 runExternal(arena, out, environ, workload, buffers, options, &counters_state, lane) catch |err| switch (err) {
182 error.CompilerUnavailable => {
183 try jsonl.writeMeta(out, lane.name, host, "unavailable: compiler not on PATH");
184 },
185 else => try writeErrorRow(out, lane.name, workload, err),
186 };
187 }
188 }
189 try out.flush();
190 }
191 if (options.compile_matrix) {
192 try runCompileMatrix(arena, backing, out, options);
193 }
194 return 0;
195 }
196
197 fn writeErrorRow(out: *std.Io.Writer, system: []const u8, workload: Workload, err: anyerror) !void {
198 var note_buffer: [96]u8 = undefined;
199 const note = std.fmt.bufPrint(note_buffer[0..], "error: {t}", .{err}) catch "error";
200 try jsonl.writeRow(out, .{
201 .system = system,
202 .workload = workload.name,
203 .metric = .kernel_ns,
204 .samples = 0,
205 .median_ns = 0,
206 .p10_ns = 0,
207 .p90_ns = 0,
208 .flops = workload.flops(),
209 .moved_bytes = workload.movedBytes(),
210 .note = note,
211 });
212 }
213
214 fn writeCompileRow(
215 out: *std.Io.Writer,
216 system: []const u8,
217 subject: CompileSubject,
218 stats: CompileStats,
219 static_metrics: cc.StaticMetrics,
220 note: ?[]const u8,
221 ) !void {
222 const row = jsonl.Row{
223 .system = system,
224 .workload = subject.name(),
225 .metric = .compile_ns,
226 .samples = stats.samples,
227 .median_ns = stats.median_ns,
228 .p10_ns = stats.p10_ns,
229 .p90_ns = stats.p90_ns,
230 .flops = subject.flops(),
231 .moved_bytes = subject.movedBytes(),
232 .code_bytes = static_metrics.code_bytes,
233 .inst_count = static_metrics.inst_count,
234 .compile_ir_ns = stats.ir_ns,
235 .compile_ir_p10_ns = stats.ir_p10_ns,
236 .compile_ir_p90_ns = stats.ir_p90_ns,
237 .compile_pass_ns = stats.pass_ns,
238 .compile_pass_p10_ns = stats.pass_p10_ns,
239 .compile_pass_p90_ns = stats.pass_p90_ns,
240 .compile_jit_ns = stats.jit_ns,
241 .compile_jit_p10_ns = stats.jit_p10_ns,
242 .compile_jit_p90_ns = stats.jit_p90_ns,
243 .compile_alloc_count = stats.alloc_count,
244 .compile_alloc_count_p10 = stats.alloc_count_p10,
245 .compile_alloc_count_p90 = stats.alloc_count_p90,
246 .compile_alloc_bytes = stats.alloc_bytes,
247 .compile_alloc_bytes_p10 = stats.alloc_bytes_p10,
248 .compile_alloc_bytes_p90 = stats.alloc_bytes_p90,
249 .compile_ir_ops = stats.ir_ops,
250 .compile_ir_blocks = stats.ir_blocks,
251 .compile_ir_values = stats.ir_values,
252 .compile_pass_count = stats.pass_count,
253 .note = note,
254 };
255 try jsonl.writeRow(out, row);
256 try jsonl.writeProfilingMetrics(out, row);
257 }
258
259 /// Returns the kernel for `kind`, or `null` when `module` has no function of that name.
260 /// Fails with `error.SignatureMismatch` when the function's signature differs from the kernel's.
261 pub fn choirKernel(
262 runtime: *const choir.backends.x86_64.JitRuntime,
263 module: choir.backends.x86_64.jit.ModuleHandle,
264 kind: workload_mod.Kind,
265 ) choir.backends.x86_64.jit.CallError!?abi.Kernel {
266 switch (kind) {
267 inline else => |tag| {
268 const Function = @FieldType(abi.Kernel, @tagName(tag));
269 const function = runtime.getFunction(module, tag.symbol(), Function) catch |err|
270 switch (err) {
271 error.FunctionNotFound => return null,
272 else => |other| return other,
273 };
274 return @unionInit(abi.Kernel, @tagName(tag), function);
275 },
276 }
277 }
278
279 fn runChoir(
280 arena: Allocator,
281 backing: Allocator,
282 out: *std.Io.Writer,
283 environ: std.process.Environ,
284 workload: Workload,
285 buffers: ?*oracle.Buffers,
286 options: Options,
287 counters_state: *CountersState,
288 use_pipeline: bool,
289 ) !void {
290 const system: []const u8 = if (use_pipeline) systems.choir else systems.choir_raw;
291 const subject = CompileSubject{ .kernel = workload };
292
293 const compile_stats = try sampleChoirCompile(arena, backing, out, system, subject, options, use_pipeline);
294 var compile_note_buffer: [64]u8 = undefined;
295 const compile_note = std.fmt.bufPrint(compile_note_buffer[0..], "pipeline={s}", .{
296 if (use_pipeline) choir.passes.default_optimization_pipeline_name else "none",
297 }) catch null;
298
299 var lane_arena = alloc_arena.Arena.init(backing);
300 defer lane_arena.deinit();
301 const lane_allocator = lane_arena.allocator();
302
303 var ctx = try choir.Context.init(lane_allocator, choir.Context.Limits.testing);
304 defer ctx.deinit(lane_allocator);
305 try choir.dialects.registerAllDialects(&ctx);
306 const module = try kernels.build(&ctx, workload.kind);
307
308 if (use_pipeline) {
309 var pass_manager = choir.PassManager.init(lane_allocator);
310 defer pass_manager.deinit();
311 try choir.passes.addDefaultOptimizationPipeline(&pass_manager);
312 const result = pass_manager.run(module.op, &ctx);
313 if (result != .success) return error.PipelineFailed;
314 }
315
316 var backend = try Backend.init(lane_allocator, &ctx, .standard);
317 defer backend.deinit();
318 const compiled = try backend.compile(module.op);
319
320 var static_metrics = cc.StaticMetrics{};
321 if (backend.compileFunctionToObjectFile(module.op, workload.kind.symbol())) |object_artifact| {
322 var artifact = object_artifact;
323 defer artifact.deinit();
324 const object_path = try std.fmt.allocPrint(arena, "{s}/{s}-{s}.o", .{ options.workdir, workload.name, system });
325 if (artifact.payload.buffers.items.len > 0) {
326 sys.fs.writeFile(object_path, artifact.payload.buffers.items[0].bytes) catch {};
327 static_metrics = cc.staticMetrics(arena, environ, object_path, workload.kind.symbol());
328 }
329 } else |_| {}
330
331 try writeCompileRow(out, system, subject, compile_stats, static_metrics, compile_note);
332 if (options.compile_only) return;
333
334 const actual_buffers = buffers orelse return error.BuffersMissing;
335 const kernel = (try choirKernel(&backend.runtime, compiled, workload.kind)) orelse
336 return error.KernelMissing;
337
338 try measure(arena, out, system, workload, actual_buffers, kernel, options, counters_state, static_metrics);
339 }
340
341 fn runCompileMatrix(
342 arena: Allocator,
343 backing: Allocator,
344 out: *std.Io.Writer,
345 options: Options,
346 ) !void {
347 if (!supports_choir_execution) return;
348 for (compile_mod.matrix) |workload| {
349 if (options.workload) |filter| {
350 if (!std.mem.eql(u8, filter, workload.name)) continue;
351 }
352
353 const subject = CompileSubject{ .generated = workload };
354 const stats = try sampleChoirCompile(arena, backing, out, systems.choir, subject, options, true);
355 var note_buffer: [96]u8 = undefined;
356 const note = std.fmt.bufPrint(note_buffer[0..], "pipeline={s};shape={s}", .{
357 choir.passes.default_optimization_pipeline_name,
358 workload.shape.name(),
359 }) catch null;
360 try writeCompileRow(out, systems.choir, subject, stats, .{}, note);
361
362 if (options.raw_lane) {
363 const raw_stats = try sampleChoirCompile(arena, backing, out, systems.choir_raw, subject, options, false);
364 var raw_note_buffer: [96]u8 = undefined;
365 const raw_note = std.fmt.bufPrint(raw_note_buffer[0..], "pipeline=none;shape={s}", .{workload.shape.name()}) catch null;
366 try writeCompileRow(out, systems.choir_raw, subject, raw_stats, .{}, raw_note);
367 }
368 try out.flush();
369 }
370 }
371
372 fn compileChoirOnce(
373 allocator: Allocator,
374 subject: CompileSubject,
375 use_pipeline: bool,
376 trace_passes: bool,
377 allocation_tracker: ?*const bench.CountingAllocator,
378 ) !ChoirCompileTimes {
379 var timer = Timer.start();
380 var ctx = try choir.Context.init(allocator, choir.Context.Limits.testing);
381 defer ctx.deinit(allocator);
382 try choir.dialects.registerAllDialects(&ctx);
383 const module = try subject.build(&ctx);
384 const ir_ns = timer.read();
385
386 timer.reset();
387 var pass_count: u64 = 0;
388 var pass_runs: []const PassRunSummary = &.{};
389 if (use_pipeline) {
390 var pass_manager = choir.PassManager.init(allocator);
391 defer pass_manager.deinit();
392 try choir.passes.addDefaultOptimizationPipeline(&pass_manager);
393
394 var instrumentation = choir.passes.TimingInstrumentation.initWithOptions(allocator, .{
395 .collect_pass_ir_sizes = trace_passes,
396 });
397 defer instrumentation.deinit();
398 if (trace_passes) {
399 if (allocation_tracker) |tracker| {
400 instrumentation.setAllocationSnapshotProvider(.{
401 .context = tracker,
402 .snapshot = allocationSnapshot,
403 });
404 }
405 try pass_manager.addInstrumentation(instrumentation.instrumentation());
406 }
407
408 const result = pass_manager.run(module.op, &ctx);
409 if (result != .success) return error.PipelineFailed;
410 pass_count = pass_manager.stats.pass_runs;
411 if (trace_passes) pass_runs = try instrumentation.passRunSummariesAlloc(allocator);
412 }
413 const pass_ns = timer.read();
414 const size = countIr(module.op);
415
416 timer.reset();
417 var backend = try Backend.init(allocator, &ctx, .standard);
418 defer backend.deinit();
419 _ = try backend.compile(module.op);
420 const jit_ns = timer.read();
421
422 return .{
423 .ir_ns = ir_ns,
424 .pass_ns = pass_ns,
425 .jit_ns = jit_ns,
426 .size = size,
427 .pass_count = pass_count,
428 .pass_runs = pass_runs,
429 };
430 }
431
432 fn sampleChoirCompile(
433 arena: Allocator,
434 backing: Allocator,
435 out: *std.Io.Writer,
436 system: []const u8,
437 subject: CompileSubject,
438 options: Options,
439 use_pipeline: bool,
440 ) !CompileStats {
441 var warmup_index: u32 = 0;
442 while (warmup_index < options.warmup) : (warmup_index += 1) {
443 var sample_arena_state = alloc_arena.Arena.init(backing);
444 defer sample_arena_state.deinit();
445 _ = try compileChoirOnce(sample_arena_state.allocator(), subject, use_pipeline, false, null);
446 }
447
448 const totals = try arena.alloc(u64, options.samples);
449 const irs = try arena.alloc(u64, options.samples);
450 const passes = try arena.alloc(u64, options.samples);
451 const jits = try arena.alloc(u64, options.samples);
452 const alloc_counts = try arena.alloc(u64, options.samples);
453 const alloc_bytes = try arena.alloc(u64, options.samples);
454 const ir_ops = try arena.alloc(u64, options.samples);
455 const ir_blocks = try arena.alloc(u64, options.samples);
456 const ir_values = try arena.alloc(u64, options.samples);
457 const pass_counts = try arena.alloc(u64, options.samples);
458
459 for (totals, 0..) |*total, index| {
460 var sample_arena_state = alloc_arena.Arena.init(backing);
461 defer sample_arena_state.deinit();
462 var allocation_tracker = bench.CountingAllocator.init(sample_arena_state.allocator());
463 const trace_passes = options.trace_compile_threshold_ns != 0 and use_pipeline;
464 const times = try compileChoirOnce(allocation_tracker.allocator(), subject, use_pipeline, trace_passes, &allocation_tracker);
465 total.* = times.total();
466 irs[index] = times.ir_ns;
467 passes[index] = times.pass_ns;
468 jits[index] = times.jit_ns;
469 alloc_counts[index] = allocation_tracker.counts.alloc_count;
470 alloc_bytes[index] = allocation_tracker.counts.alloc_bytes;
471 ir_ops[index] = times.size.ops;
472 ir_blocks[index] = times.size.blocks;
473 ir_values[index] = times.size.values;
474 pass_counts[index] = times.pass_count;
475 if (trace_passes and total.* >= options.trace_compile_threshold_ns) {
476 try writeCompileTraceRows(out, system, subject, index, total.*, times.pass_runs);
477 }
478 coz.progressNamed("choir.versus.compile_sample");
479 }
480
481 const total_stats = jsonl.Samples.init(totals);
482 const ir_stats = jsonl.Samples.init(irs);
483 const pass_stats = jsonl.Samples.init(passes);
484 const jit_stats = jsonl.Samples.init(jits);
485 const alloc_count_stats = jsonl.Samples.init(alloc_counts);
486 const alloc_byte_stats = jsonl.Samples.init(alloc_bytes);
487 const ir_op_stats = jsonl.Samples.init(ir_ops);
488 const ir_block_stats = jsonl.Samples.init(ir_blocks);
489 const ir_value_stats = jsonl.Samples.init(ir_values);
490 const pass_count_stats = jsonl.Samples.init(pass_counts);
491
492 return .{
493 .samples = options.samples,
494 .median_ns = total_stats.median(),
495 .p10_ns = total_stats.p10(),
496 .p90_ns = total_stats.p90(),
497 .ir_ns = ir_stats.median(),
498 .ir_p10_ns = ir_stats.p10(),
499 .ir_p90_ns = ir_stats.p90(),
500 .pass_ns = pass_stats.median(),
501 .pass_p10_ns = pass_stats.p10(),
502 .pass_p90_ns = pass_stats.p90(),
503 .jit_ns = jit_stats.median(),
504 .jit_p10_ns = jit_stats.p10(),
505 .jit_p90_ns = jit_stats.p90(),
506 .alloc_count = alloc_count_stats.median(),
507 .alloc_count_p10 = alloc_count_stats.p10(),
508 .alloc_count_p90 = alloc_count_stats.p90(),
509 .alloc_bytes = alloc_byte_stats.median(),
510 .alloc_bytes_p10 = alloc_byte_stats.p10(),
511 .alloc_bytes_p90 = alloc_byte_stats.p90(),
512 .ir_ops = ir_op_stats.median(),
513 .ir_blocks = ir_block_stats.median(),
514 .ir_values = ir_value_stats.median(),
515 .pass_count = pass_count_stats.median(),
516 };
517 }
518
519 fn writeCompileTraceRows(
520 out: *std.Io.Writer,
521 system: []const u8,
522 subject: CompileSubject,
523 sample_index: usize,
524 total_ns: u64,
525 pass_runs: []const PassRunSummary,
526 ) !void {
527 for (pass_runs) |pass_run| {
528 try jsonl.writeCompileTrace(out, .{
529 .system = system,
530 .workload = subject.name(),
531 .sample = sample_index,
532 .total_ns = total_ns,
533 .pass_ordinal = pass_run.ordinal,
534 .pass_name = pass_run.name,
535 .pass_ns = nsOrZero(pass_run.elapsed_ns),
536 .modified = pass_run.modified,
537 .op_count_before = pass_run.op_count_before orelse 0,
538 .op_count_after = pass_run.op_count_after orelse 0,
539 .op_count_delta = pass_run.op_count_delta orelse 0,
540 .alloc_count = pass_run.alloc_count orelse 0,
541 .free_count = pass_run.free_count orelse 0,
542 .alloc_bytes = pass_run.alloc_bytes orelse 0,
543 });
544 }
545 }
546
547 fn nsOrZero(value: i128) u64 {
548 if (value <= 0) return 0;
549 if (value > std.math.maxInt(u64)) return std.math.maxInt(u64);
550 return @intCast(value);
551 }
552
553 fn allocationSnapshot(context: ?*const anyopaque) AllocationSnapshot {
554 const tracker: *const bench.CountingAllocator = @ptrCast(@alignCast(context.?));
555 return .{
556 .alloc_count = tracker.counts.alloc_count,
557 .free_count = tracker.counts.free_count,
558 .alloc_bytes = tracker.counts.alloc_bytes,
559 };
560 }
561
562 fn countIr(op: *choir.Operation) IrSize {
563 var size = IrSize{
564 .ops = 1,
565 .blocks = 0,
566 .values = op.results.items.len,
567 };
568 for (op.regions.items) |*region| {
569 var block_iter = region.getBlocks();
570 while (block_iter.next()) |block| {
571 size.blocks +|= 1;
572 size.values +|= block.arguments.items.len;
573 var op_node = block.operations.head;
574 while (op_node) |node| {
575 const child: *choir.Operation = @ptrCast(@alignCast(node));
576 const child_size = countIr(child);
577 size.ops +|= child_size.ops;
578 size.blocks +|= child_size.blocks;
579 size.values +|= child_size.values;
580 op_node = child.next_op;
581 }
582 }
583 }
584 return size;
585 }
586
587 fn runExternal(
588 arena: Allocator,
589 out: *std.Io.Writer,
590 environ: std.process.Environ,
591 workload: Workload,
592 buffers: ?*oracle.Buffers,
593 options: Options,
594 counters_state: *CountersState,
595 lane: cc.Lane,
596 ) !void {
597 const source_path = try std.fmt.allocPrint(arena, "{s}/{s}.c", .{ options.workdir, @tagName(workload.kind) });
598 try sys.fs.writeFile(source_path, workload.kind.source());
599 const shared_path = try std.fmt.allocPrint(arena, "{s}/{s}-{s}.so", .{ options.workdir, workload.name, lane.name });
600
601 const compile_stats = try sampleExternalCompile(arena, environ, lane, source_path, shared_path, options);
602 const static_metrics = cc.staticMetrics(arena, environ, shared_path, workload.kind.symbol());
603 const subject = CompileSubject{ .kernel = workload };
604 try writeCompileRow(out, lane.name, subject, compile_stats, static_metrics, "sampled subprocess wall including driver");
605 if (options.compile_only) return;
606
607 var library = sys.dynamic.Library.open(shared_path) catch return error.LibraryOpenFailed;
608 defer library.close();
609 const kernel = cc.openKernel(&library, workload.kind) orelse return error.KernelMissing;
610
611 const actual_buffers = buffers orelse return error.BuffersMissing;
612 try measure(arena, out, lane.name, workload, actual_buffers, kernel, options, counters_state, static_metrics);
613 }
614
615 fn sampleExternalCompile(
616 arena: Allocator,
617 environ: std.process.Environ,
618 lane: cc.Lane,
619 source_path: []const u8,
620 shared_path: []const u8,
621 options: Options,
622 ) !CompileStats {
623 var warmup_index: u32 = 0;
624 while (warmup_index < options.warmup) : (warmup_index += 1) {
625 _ = try cc.compileShared(arena, environ, lane, source_path, shared_path);
626 }
627
628 const totals = try arena.alloc(u64, options.samples);
629 for (totals) |*sample| {
630 sample.* = try cc.compileShared(arena, environ, lane, source_path, shared_path);
631 coz.progressNamed("choir.versus.compile_sample");
632 }
633
634 const total_stats = jsonl.Samples.init(totals);
635 return .{
636 .samples = options.samples,
637 .median_ns = total_stats.median(),
638 .p10_ns = total_stats.p10(),
639 .p90_ns = total_stats.p90(),
640 };
641 }
642
643 const CountersState = struct {
644 enabled: bool,
645
646 fn read(
647 self: *CountersState,
648 kernel: abi.Kernel,
649 workload: Workload,
650 buffers: *oracle.Buffers,
651 ) CounterPair {
652 if (!self.enabled) return .{};
653 buffers.reset();
654 var region = PerfCounterSet.start(&counter_selectors, .{}) catch {
655 self.enabled = false;
656 return .{};
657 };
658 defer region.deinit();
659 kernel.call(workload, buffers);
660 const result = region.stop() catch {
661 self.enabled = false;
662 return .{};
663 };
664 const instructions = result.find(counter_selectors[0]) orelse {
665 self.enabled = false;
666 return .{};
667 };
668 const cycles = result.find(counter_selectors[1]) orelse {
669 self.enabled = false;
670 return .{};
671 };
672 return .{
673 .instructions = instructions.scaledValue() orelse 0,
674 .cycles = cycles.scaledValue() orelse 0,
675 };
676 }
677 };
678
679 const CounterPair = struct {
680 instructions: u64 = 0,
681 cycles: u64 = 0,
682 };
683
684 const CounterSamples = struct {
685 instructions: []u64,
686 cycles: []u64,
687 };
688
689 fn warmupKernel(
690 kernel: abi.Kernel,
691 workload: Workload,
692 buffers: *oracle.Buffers,
693 count: u32,
694 ) void {
695 var index: u32 = 0;
696 while (index < count) : (index += 1) {
697 buffers.reset();
698 kernel.call(workload, buffers);
699 }
700 }
701
702 fn sampleWall(
703 arena: Allocator,
704 kernel: abi.Kernel,
705 workload: Workload,
706 buffers: *oracle.Buffers,
707 sample_count: u32,
708 ) ![]u64 {
709 const samples = try arena.alloc(u64, sample_count);
710 var timer = Timer.start();
711 for (samples) |*sample| {
712 buffers.reset();
713 timer.reset();
714 kernel.call(workload, buffers);
715 sample.* = timer.read();
716 coz.progressNamed("choir.versus.sample");
717 }
718 return samples;
719 }
720
721 fn sampleCounters(
722 arena: Allocator,
723 state: *CountersState,
724 kernel: abi.Kernel,
725 workload: Workload,
726 buffers: *oracle.Buffers,
727 sample_count: u32,
728 ) !CounterSamples {
729 const instructions = try arena.alloc(u64, sample_count);
730 const cycles = try arena.alloc(u64, sample_count);
731 for (instructions, cycles) |*instruction_sample, *cycle_sample| {
732 const pair = state.read(kernel, workload, buffers);
733 instruction_sample.* = pair.instructions;
734 cycle_sample.* = pair.cycles;
735 }
736 return .{ .instructions = instructions, .cycles = cycles };
737 }
738
739 fn measure(
740 arena: Allocator,
741 out: *std.Io.Writer,
742 system: []const u8,
743 workload: Workload,
744 buffers: *oracle.Buffers,
745 kernel: abi.Kernel,
746 options: Options,
747 counters_state: *CountersState,
748 static_metrics: cc.StaticMetrics,
749 ) !void {
750 if (options.verify) {
751 buffers.reset();
752 kernel.call(workload, buffers);
753 oracle.verify(workload, buffers) catch {
754 try jsonl.writeRow(out, .{
755 .system = system,
756 .workload = workload.name,
757 .metric = .kernel_ns,
758 .samples = 0,
759 .median_ns = 0,
760 .p10_ns = 0,
761 .p90_ns = 0,
762 .flops = workload.flops(),
763 .moved_bytes = workload.movedBytes(),
764 .code_bytes = static_metrics.code_bytes,
765 .inst_count = static_metrics.inst_count,
766 .mca_rthroughput = static_metrics.mca_rthroughput,
767 .note = "oracle mismatch: timing skipped",
768 });
769 return;
770 };
771 }
772
773 warmupKernel(kernel, workload, buffers, options.warmup);
774
775 const wall = try sampleWall(arena, kernel, workload, buffers, options.samples);
776 const counters = try sampleCounters(
777 arena,
778 counters_state,
779 kernel,
780 workload,
781 buffers,
782 options.samples,
783 );
784
785 const wall_stats = jsonl.Samples.init(wall);
786 const instruction_stats = jsonl.Samples.init(counters.instructions);
787 const cycle_stats = jsonl.Samples.init(counters.cycles);
788
789 try jsonl.writeRow(out, .{
790 .system = system,
791 .workload = workload.name,
792 .metric = .kernel_ns,
793 .samples = options.samples,
794 .median_ns = wall_stats.median(),
795 .p10_ns = wall_stats.p10(),
796 .p90_ns = wall_stats.p90(),
797 .flops = workload.flops(),
798 .moved_bytes = workload.movedBytes(),
799 .instructions = instruction_stats.median(),
800 .cycles = cycle_stats.median(),
801 .code_bytes = static_metrics.code_bytes,
802 .inst_count = static_metrics.inst_count,
803 .mca_rthroughput = static_metrics.mca_rthroughput,
804 });
805 }
806
807 const test_battery = [_]Workload{
808 .{ .name = "saxpy_smoke", .kind = .saxpy, .n = 64 },
809 .{ .name = "dot_smoke", .kind = .dot, .n = 64 },
810 .{ .name = "sum_smoke", .kind = .sum, .n = 64 },
811 .{ .name = "matmul_smoke", .kind = .matmul, .n = 8 },
812 .{ .name = "polybench_gemm_smoke", .kind = .polybench_gemm, .n = 8 },
813 .{ .name = "stencil3_smoke", .kind = .stencil3, .n = 64 },
814 .{ .name = "clampsum_smoke", .kind = .clampsum, .n = 64 },
815 };
816
817 fn smokeLane(use_pipeline: bool) !void {
818 var arena = alloc_arena.Arena.init(std.testing.allocator);
819 defer arena.deinit();
820 const allocator = arena.allocator();
821
822 for (test_battery) |workload| {
823 var ctx = try choir.Context.init(allocator, choir.Context.Limits.testing);
824 defer ctx.deinit(allocator);
825 try choir.dialects.registerAllDialects(&ctx);
826 const module = try kernels.build(&ctx, workload.kind);
827
828 if (use_pipeline) {
829 var pass_manager = choir.PassManager.init(allocator);
830 defer pass_manager.deinit();
831 try choir.passes.addDefaultOptimizationPipeline(&pass_manager);
832 try std.testing.expectEqual(choir.passes.PassResult.success, pass_manager.run(module.op, &ctx));
833 }
834
835 var backend = try Backend.init(allocator, &ctx, .testing);
836 defer backend.deinit();
837 const compiled = try backend.compile(module.op);
838 const kernel = (try choirKernel(&backend.runtime, compiled, workload.kind)) orelse
839 return error.KernelMissing;
840
841 var buffers = try oracle.Buffers.alloc(std.testing.allocator, workload);
842 defer buffers.deinit(std.testing.allocator);
843 buffers.reset();
844 kernel.call(workload, &buffers);
845 try oracle.verify(workload, &buffers);
846 }
847 }
848
849 test "choir lane compiles, runs, and conforms on every kernel" {
850 if (!supports_choir_execution) return;
851 try smokeLane(false);
852 }
853
854 test "choir pipeline lane compiles, runs, and conforms on every kernel" {
855 if (!supports_choir_execution) return;
856 try smokeLane(true);
857 }
858
859 test "choir compile sampling reports phase medians" {
860 if (!supports_choir_execution) return;
861
862 var arena_state = alloc_arena.Arena.init(std.testing.allocator);
863 defer arena_state.deinit();
864
865 var buffer: [4096]u8 = undefined;
866 var writer = std.Io.Writer.fixed(buffer[0..]);
867 const stats = try sampleChoirCompile(arena_state.allocator(), std.testing.allocator, &writer, systems.choir, .{ .kernel = test_battery[0] }, .{
868 .samples = 3,
869 .warmup = 0,
870 }, true);
871
872 try std.testing.expectEqual(@as(u32, 3), stats.samples);
873 try std.testing.expect(stats.median_ns > 0);
874 try std.testing.expect(stats.ir_ns > 0);
875 try std.testing.expect(stats.pass_ns > 0);
876 try std.testing.expect(stats.jit_ns > 0);
877 try std.testing.expect(stats.alloc_count > 0);
878 try std.testing.expect(stats.alloc_bytes > 0);
879 try std.testing.expect(stats.ir_ops > 0);
880 try std.testing.expect(stats.ir_blocks > 0);
881 try std.testing.expect(stats.ir_values > 0);
882 try std.testing.expect(stats.pass_count > 0);
883 }
884
885 test "generated compile workload reports IR size" {
886 if (!supports_choir_execution) return;
887
888 var arena_state = alloc_arena.Arena.init(std.testing.allocator);
889 defer arena_state.deinit();
890
891 var buffer: [4096]u8 = undefined;
892 var writer = std.Io.Writer.fixed(buffer[0..]);
893 const stats = try sampleChoirCompile(arena_state.allocator(), std.testing.allocator, &writer, systems.choir, .{ .generated = compile_mod.matrix[0] }, .{
894 .samples = 2,
895 .warmup = 0,
896 }, true);
897
898 try std.testing.expectEqual(@as(u32, 2), stats.samples);
899 try std.testing.expect(stats.median_ns > 0);
900 try std.testing.expect(stats.ir_ops > compile_mod.matrix[0].sourceOps());
901 }