lib/accy/src/profiling/choir/suite.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

   1 const std = @import("std");
   2 const gpu = @import("gpu");
   3 const choir_abi = @import("choir_abi");
   4 const accy = @import("accy");
   5 const bench = @import("bench");
   6 const sys = @import("sys");
   7 
   8 const config = @import("config.zig");
   9 const jsonl = @import("jsonl.zig");
  10 const stats_mod = @import("stats.zig");
  11 const timing_mod = @import("timing.zig");
  12 const workload_mod = @import("workload/root.zig");
  13 
  14 const Allocator = std.mem.Allocator;
  15 const cuda_backend = gpu.cuda;
  16 const family_tuning = accy.kernel.library.tuning;
  17 const schedule_tuning = accy.executable;
  18 const coz = bench.coz;
  19 
  20 const InitCudaResult = union(enum) {
  21     state: cuda_backend.State,
  22     skipped: []const u8,
  23 };
  24 
  25 const SuiteCase = struct {
  26     workload_kind: config.WorkloadKind,
  27     elements: u32,
  28     chain: u32,
  29 };
  30 
  31 const quick_cases = [_]SuiteCase{
  32     .{ .workload_kind = .elementwise_chain, .elements = 256, .chain = 8 },
  33     .{ .workload_kind = .indexing_mix, .elements = 128, .chain = 2 },
  34     .{ .workload_kind = .reduction_row_norm, .elements = 32, .chain = 128 },
  35     .{ .workload_kind = .coordinate_mesh, .elements = 16, .chain = 16 },
  36     .{ .workload_kind = .nbody_all_pairs, .elements = 16, .chain = 3 },
  37     .{ .workload_kind = .mlp_two_layer, .elements = 32, .chain = 64 },
  38     .{ .workload_kind = .residual_block, .elements = 32, .chain = 64 },
  39     .{ .workload_kind = .halo_stencil, .elements = 16, .chain = 16 },
  40     .{ .workload_kind = .attention_softmax, .elements = 16, .chain = 32 },
  41     .{ .workload_kind = .transformer_block, .elements = 16, .chain = 32 },
  42     .{ .workload_kind = .decoder_block, .elements = 16, .chain = 32 },
  43 };
  44 
  45 const standard_cases = [_]SuiteCase{
  46     .{ .workload_kind = .elementwise_chain, .elements = 4096, .chain = 64 },
  47     .{ .workload_kind = .elementwise_chain, .elements = 16384, .chain = 16 },
  48     .{ .workload_kind = .indexing_mix, .elements = 2048, .chain = 8 },
  49     .{ .workload_kind = .indexing_mix, .elements = 8192, .chain = 4 },
  50     .{ .workload_kind = .reduction_row_norm, .elements = 64, .chain = 256 },
  51     .{ .workload_kind = .coordinate_mesh, .elements = 64, .chain = 64 },
  52     .{ .workload_kind = .nbody_all_pairs, .elements = 32, .chain = 3 },
  53     .{ .workload_kind = .mlp_two_layer, .elements = 64, .chain = 128 },
  54     .{ .workload_kind = .residual_block, .elements = 64, .chain = 128 },
  55     .{ .workload_kind = .halo_stencil, .elements = 64, .chain = 64 },
  56     .{ .workload_kind = .attention_softmax, .elements = 32, .chain = 64 },
  57     .{ .workload_kind = .transformer_block, .elements = 32, .chain = 64 },
  58     .{ .workload_kind = .decoder_block, .elements = 32, .chain = 64 },
  59 };
  60 
  61 const scaling_cases = [_]SuiteCase{
  62     .{ .workload_kind = .elementwise_chain, .elements = 256, .chain = 32 },
  63     .{ .workload_kind = .elementwise_chain, .elements = 1024, .chain = 32 },
  64     .{ .workload_kind = .elementwise_chain, .elements = 4096, .chain = 32 },
  65     .{ .workload_kind = .indexing_mix, .elements = 128, .chain = 4 },
  66     .{ .workload_kind = .indexing_mix, .elements = 512, .chain = 4 },
  67     .{ .workload_kind = .indexing_mix, .elements = 2048, .chain = 4 },
  68     .{ .workload_kind = .reduction_row_norm, .elements = 16, .chain = 128 },
  69     .{ .workload_kind = .reduction_row_norm, .elements = 32, .chain = 128 },
  70     .{ .workload_kind = .reduction_row_norm, .elements = 64, .chain = 128 },
  71     .{ .workload_kind = .nbody_all_pairs, .elements = 8, .chain = 3 },
  72     .{ .workload_kind = .nbody_all_pairs, .elements = 16, .chain = 3 },
  73     .{ .workload_kind = .nbody_all_pairs, .elements = 32, .chain = 3 },
  74     .{ .workload_kind = .transformer_block, .elements = 8, .chain = 32 },
  75     .{ .workload_kind = .transformer_block, .elements = 16, .chain = 32 },
  76     .{ .workload_kind = .transformer_block, .elements = 32, .chain = 32 },
  77     .{ .workload_kind = .decoder_block, .elements = 8, .chain = 32 },
  78     .{ .workload_kind = .decoder_block, .elements = 16, .chain = 32 },
  79     .{ .workload_kind = .decoder_block, .elements = 32, .chain = 32 },
  80 };
  81 
  82 pub fn runSuite(_: Allocator, backing_allocator: Allocator, out: *std.Io.Writer, options: config.Options) !void {
  83     const suite_kind = options.suite_kind orelse return error.InvalidArguments;
  84     const cases = suiteCases(suite_kind);
  85     for (cases) |case| {
  86         var case_arena_state = std.heap.ArenaAllocator.init(backing_allocator);
  87         defer case_arena_state.deinit();
  88         var case_options = options;
  89         case_options.suite_kind = null;
  90         case_options.workload_kind = case.workload_kind;
  91         case_options.elements = case.elements;
  92         case_options.chain = case.chain;
  93         case_options.element_value_count = 0;
  94         case_options.chain_value_count = 0;
  95         try runWorkload(case_arena_state.allocator(), backing_allocator, out, case_options);
  96     }
  97 }
  98 
  99 fn suiteCases(kind: config.SuiteKind) []const SuiteCase {
 100     return switch (kind) {
 101         .quick => &quick_cases,
 102         .standard => &standard_cases,
 103         .scaling => &scaling_cases,
 104     };
 105 }
 106 
 107 pub fn runWorkload(arena: Allocator, backing_allocator: Allocator, out: *std.Io.Writer, options: config.Options) !void {
 108     const workload_latency = bench.phaseAt("accy.choir.workload", @src());
 109     defer workload_latency.end();
 110 
 111     const workload = workload_mod.Workload.init(options);
 112     const launch_measurement_inputs = if (options.measure_cuda_launch_candidates)
 113         try workload_mod.buildLaunchInputs(arena, options)
 114     else
 115         null;
 116 
 117     try jsonl.writeRunStart(out, options, workload);
 118 
 119     for (0..options.warmup) |_| {
 120         _ = try runPipeline(backing_allocator, options, null, null);
 121         coz.progressNamed("accy.choir.warmup");
 122     }
 123 
 124     const sample_set_count = config.phase_names.len + config.pass_count +
 125         config.analysis_count + config.memory_names.len;
 126     const sample_value_count = try stats_mod.sampleSetValueCapacity(
 127         sample_set_count,
 128         options.samples,
 129     );
 130     const sample_values = try arena.alloc(u64, sample_value_count);
 131     var sample_sets = stats_mod.SampleSetCursor.init(sample_values);
 132     var phase_samples = try sample_sets.take(config.phase_names.len, options.samples);
 133     var pass_samples = try sample_sets.take(config.pass_count, options.samples);
 134     var analysis_samples = try sample_sets.take(config.analysis_count, options.samples);
 135     var memory_samples = try sample_sets.take(config.memory_names.len, options.samples);
 136     std.debug.assert(sample_sets.complete());
 137     var pass_ir_totals = stats_mod.PassIrTotals{};
 138     var pass_memory_totals = stats_mod.PassMemoryTotals{};
 139     var initial_choir_ops: u64 = 0;
 140     var final_choir_ops: u64 = 0;
 141 
 142     for (0..options.samples) |sample| {
 143         const sample_latency = bench.phaseAt("accy.choir.sample", @src());
 144         defer sample_latency.end();
 145 
 146         var sample_arena_state = std.heap.ArenaAllocator.init(backing_allocator);
 147         defer sample_arena_state.deinit();
 148         const sample_arena = sample_arena_state.allocator();
 149 
 150         var timing = accy.preparation.BackendPreparationTiming.initWithOptions(sample_arena, .{
 151             .collect_pass_ir_sizes = options.pass_ir_sizes,
 152         });
 153         defer timing.deinit();
 154         var stats: accy.preparation.BackendPreparationStats = .{};
 155 
 156         const timings = try runPipeline(backing_allocator, options, &timing, &stats);
 157         initial_choir_ops = timings.initial_choir_ops;
 158         final_choir_ops = timings.final_choir_ops;
 159         timing_mod.recordPhaseSamples(&phase_samples, sample, timings);
 160         timing_mod.recordMemorySamples(&memory_samples, sample, timings);
 161         try jsonl.writePhaseRecord(out, options, workload, sample, "total", timings.total_ns, timings, stats);
 162         try jsonl.writePhaseRecord(out, options, workload, sample, "semantic", timings.semantic_ns, timings, .{});
 163         try jsonl.writePhaseRecord(out, options, workload, sample, "contract", timings.contract_ns, timings, .{});
 164         try jsonl.writePhaseRecord(out, options, workload, sample, "tensor", timings.tensor_ns, timings, .{});
 165         try jsonl.writePhaseRecord(out, options, workload, sample, "dispatch", timings.dispatch_ns, timings, .{});
 166         try jsonl.writePhaseRecord(out, options, workload, sample, "memory", timings.memory_ns, timings, .{});
 167         try jsonl.writePhaseRecord(out, options, workload, sample, "kernel", timings.kernel_ns, timings, .{});
 168         try jsonl.writePhaseRecord(out, options, workload, sample, "target", timings.target_ns, timings, stats);
 169         try jsonl.writeMemoryRecord(out, options, workload, sample, timings);
 170         try jsonl.writePassRecords(out, options, workload, sample, timings, &timing, &pass_samples, &pass_ir_totals, &pass_memory_totals);
 171         try jsonl.writeAnalysisRecords(out, options, workload, sample, timings, &timing, &analysis_samples);
 172         coz.progressNamed("accy.choir.sample");
 173     }
 174 
 175     var statistics = try stats_mod.StatisticsStorage.init(arena, .{ .samples = options.samples });
 176     defer statistics.deinit(arena);
 177     statistics.activate();
 178     try jsonl.writePhaseSummaries(out, &statistics, options, workload, initial_choir_ops, final_choir_ops, phase_samples);
 179     try jsonl.writeMemorySummaries(out, &statistics, options, workload, initial_choir_ops, final_choir_ops, memory_samples);
 180     try jsonl.writePassSummaries(out, &statistics, options, workload, initial_choir_ops, final_choir_ops, pass_samples, pass_ir_totals, pass_memory_totals);
 181     try jsonl.writeAnalysisSummaries(out, &statistics, options, workload, initial_choir_ops, final_choir_ops, analysis_samples);
 182     if (options.measure_cuda_launch_candidates) {
 183         try runCudaLaunchCandidateMeasurements(
 184             arena,
 185             backing_allocator,
 186             out,
 187             options,
 188             workload,
 189             launch_measurement_inputs.?,
 190         );
 191     }
 192     try jsonl.writeRunEnd(out, options, workload);
 193     coz.progressNamed("accy.choir.workload.complete");
 194 }
 195 
 196 const BackendPhaseEmitter = struct {
 197     out: *std.Io.Writer,
 198     options: config.Options,
 199     workload: workload_mod.Workload,
 200 };
 201 
 202 fn recordFragmentPhase(context: ?*anyopaque, phase_name: []const u8, ns: u64) anyerror!void {
 203     const emitter: *BackendPhaseEmitter = @ptrCast(@alignCast(context.?));
 204     try jsonl.writeBackendPhaseRecord(emitter.out, emitter.options, emitter.workload, phase_name, ns);
 205 }
 206 
 207 fn compileAndLoadProfileModule(
 208     allocator: Allocator,
 209     handle: gpu.BackendHandle,
 210     module: *accy.choir.SemanticModule,
 211     options: accy.executable.FragmentCompilerOptions,
 212 ) !*accy.executable.LoadedFragment {
 213     const compiled = try accy.executable.compileFragmentFromSemanticModule(allocator, handle, module, options);
 214     return try accy.executable.loadFragment(allocator, handle, compiled, options);
 215 }
 216 
 217 fn runCudaLaunchCandidateMeasurements(
 218     arena: Allocator,
 219     backing_allocator: Allocator,
 220     out: *std.Io.Writer,
 221     options: config.Options,
 222     workload: workload_mod.Workload,
 223     inputs: workload_mod.LaunchInputs,
 224 ) !void {
 225     var cuda_state = switch (try initCudaBackendState(backing_allocator)) {
 226         .state => |state| state,
 227         .skipped => |reason| {
 228             try jsonl.writeLaunchCandidateSkip(out, options, workload, reason);
 229             return;
 230         },
 231     };
 232     defer cuda_state.deinit();
 233     const handle = cuda_state.handle();
 234 
 235     var phase_emitter = BackendPhaseEmitter{ .out = out, .options = options, .workload = workload };
 236     const stream = try handle.createStream(.{});
 237     defer handle.destroyObject(stream.id);
 238 
 239     const capabilities = try handle.queryCapabilities();
 240     var family_sink = FamilyTuningSink{
 241         .device_fingerprint = family_tuning.deviceFingerprint(capabilities),
 242         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(backing_allocator),
 243     };
 244     defer family_sink.deinit();
 245     var matrix_product_schedule_sink = MatrixProductFamilyScheduleTuningSink.init(backing_allocator, capabilities);
 246     defer matrix_product_schedule_sink.deinit();
 247     var scan_schedule_sink = GeneratedScanScheduleTuningSink.init(backing_allocator, capabilities);
 248     defer scan_schedule_sink.deinit();
 249     var row_pipeline_schedule_sink = GeneratedRowPipelineScheduleTuningSink.init(backing_allocator, capabilities);
 250     defer row_pipeline_schedule_sink.deinit();
 251 
 252     {
 253         const semantic_start = timing_mod.nowNanos();
 254         const module = try workload_mod.buildSemanticModule(backing_allocator, options);
 255         var module_owned = true;
 256         errdefer if (module_owned) module.deinit();
 257         try jsonl.writeBackendPhaseRecord(out, options, workload, "build_semantic_choir", stats_mod.nsBetween(semantic_start, timing_mod.nowNanos()));
 258 
 259         const fragment_start = timing_mod.nowNanos();
 260         module_owned = false;
 261         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
 262             .instrumentation = .{
 263                 .context = &phase_emitter,
 264                 .observe = recordFragmentPhase,
 265             },
 266         });
 267         const fragment_ns = stats_mod.nsBetween(fragment_start, timing_mod.nowNanos());
 268         defer fragment.deinit();
 269         try jsonl.writeBackendPhaseRecord(out, options, workload, "create_backend_loaded_fragment", fragment_ns);
 270 
 271         const bindings_start = timing_mod.nowNanos();
 272         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
 273         const bindings_ns = stats_mod.nsBetween(bindings_start, timing_mod.nowNanos());
 274         defer bindings.deinit();
 275         try jsonl.writeBackendPhaseRecord(out, options, workload, "prepare_backend_bindings", bindings_ns);
 276 
 277         const measurement_start = timing_mod.nowNanos();
 278         const measurement_options = accy.executable.LaunchCandidateBenchmarkOptions{
 279             .warmup = options.warmup,
 280             .samples = options.samples,
 281             .base_options = .{ .stream = stream },
 282             .synchronize = .stream,
 283         };
 284         const records = try bindings.measureAndRecordLaunchCandidateRecords(
 285             arena,
 286             arena,
 287             measurement_options,
 288         );
 289         var best_record: ?accy.executable.LaunchCandidateRecord = null;
 290         for (records) |record| {
 291             try jsonl.writeLaunchCandidateMeasurementRecord(out, options, workload, record);
 292             if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
 293                 if (best_record) |best| try jsonl.writeLaunchCandidateBestRecord(out, options, workload, best);
 294                 best_record = record;
 295             } else if (launchCandidateRecordBeats(record, best_record.?)) {
 296                 best_record = record;
 297             }
 298         }
 299         if (best_record) |record| try jsonl.writeLaunchCandidateBestRecord(out, options, workload, record);
 300         var kernel_index: usize = 0;
 301         while (kernel_index < fragment.kernelCount()) : (kernel_index += 1) {
 302             const summary = try fragment.kernelSummary(kernel_index);
 303             if (summary.launch_candidate_count != 1) continue;
 304             const fixed_records = try bindings.measureLaunchCandidateRecords(
 305                 arena,
 306                 arena,
 307                 kernel_index,
 308                 measurement_options,
 309             );
 310             defer arena.free(fixed_records);
 311             for (fixed_records) |record| {
 312                 try jsonl.writeLaunchCandidateMeasurementRecord(out, options, workload, record);
 313                 try jsonl.writeLaunchCandidateBestRecord(out, options, workload, record);
 314             }
 315         }
 316         const launch_measurement_ns = stats_mod.nsBetween(measurement_start, timing_mod.nowNanos());
 317         try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_launch_candidates", launch_measurement_ns);
 318 
 319         const sync_start = timing_mod.nowNanos();
 320         try handle.synchronize(.{ .scope = .stream, .stream = stream });
 321         try jsonl.writeBackendPhaseRecord(out, options, workload, "synchronize", stats_mod.nsBetween(sync_start, timing_mod.nowNanos()));
 322 
 323         try runCudaMatrixProductFamilyCandidateMeasurements(
 324             arena,
 325             backing_allocator,
 326             out,
 327             options,
 328             workload,
 329             inputs,
 330             handle,
 331             stream,
 332             &phase_emitter,
 333             &family_sink,
 334             &matrix_product_schedule_sink,
 335         );
 336 
 337         try runCudaStencilWindowFamilyCandidateMeasurements(
 338             arena,
 339             backing_allocator,
 340             out,
 341             options,
 342             workload,
 343             handle,
 344             stream,
 345             &phase_emitter,
 346             &family_sink,
 347         );
 348 
 349         try runCudaGeneratedScanScheduleMeasurements(
 350             arena,
 351             backing_allocator,
 352             out,
 353             options,
 354             workload,
 355             handle,
 356             stream,
 357             &phase_emitter,
 358             &scan_schedule_sink,
 359         );
 360 
 361         try runCudaGeneratedRowPipelineScheduleMeasurements(
 362             arena,
 363             backing_allocator,
 364             out,
 365             options,
 366             workload,
 367             handle,
 368             stream,
 369             &phase_emitter,
 370             &row_pipeline_schedule_sink,
 371         );
 372     }
 373 
 374     try runCudaGatherFamilyCandidateMeasurements(
 375         arena,
 376         backing_allocator,
 377         out,
 378         options,
 379         workload,
 380         handle,
 381         stream,
 382         &phase_emitter,
 383         &family_sink,
 384     );
 385 
 386     try runCudaPrefixSumFamilyCandidateMeasurements(
 387         arena,
 388         backing_allocator,
 389         out,
 390         options,
 391         workload,
 392         handle,
 393         stream,
 394         &phase_emitter,
 395         &family_sink,
 396     );
 397 
 398     try runCudaSortStructureFamilyMeasurements(
 399         arena,
 400         backing_allocator,
 401         out,
 402         options,
 403         workload,
 404         handle,
 405         &family_sink,
 406     );
 407 
 408     try runCudaScatterFamilyCandidateMeasurements(
 409         arena,
 410         backing_allocator,
 411         out,
 412         options,
 413         workload,
 414         handle,
 415         stream,
 416         &phase_emitter,
 417         &family_sink,
 418     );
 419 
 420     try runCudaScatterAddFamilyCandidateMeasurements(
 421         arena,
 422         backing_allocator,
 423         out,
 424         options,
 425         workload,
 426         handle,
 427         stream,
 428         &family_sink,
 429     );
 430 
 431     try runCudaSpmvCsrFamilyCandidateMeasurements(
 432         arena,
 433         backing_allocator,
 434         out,
 435         options,
 436         workload,
 437         handle,
 438         stream,
 439         &phase_emitter,
 440         &family_sink,
 441     );
 442 
 443     try runCudaSpmmCsrFamilyCandidateMeasurements(
 444         arena,
 445         backing_allocator,
 446         out,
 447         options,
 448         workload,
 449         handle,
 450         stream,
 451         &phase_emitter,
 452         &family_sink,
 453     );
 454 
 455     try runCudaSpmvCooFamilyCandidateMeasurements(
 456         arena,
 457         backing_allocator,
 458         out,
 459         options,
 460         workload,
 461         handle,
 462         stream,
 463         &phase_emitter,
 464         &family_sink,
 465     );
 466 
 467     try runCudaSpmvEllFamilyCandidateMeasurements(
 468         arena,
 469         backing_allocator,
 470         out,
 471         options,
 472         workload,
 473         handle,
 474         stream,
 475         &phase_emitter,
 476         &family_sink,
 477     );
 478 
 479     try runCudaSpmvSellFamilyCandidateMeasurements(
 480         arena,
 481         backing_allocator,
 482         out,
 483         options,
 484         workload,
 485         handle,
 486         stream,
 487         &phase_emitter,
 488         &family_sink,
 489     );
 490 
 491     try runCudaSegmentSumFamilyCandidateMeasurements(
 492         arena,
 493         backing_allocator,
 494         out,
 495         options,
 496         workload,
 497         handle,
 498         stream,
 499         &phase_emitter,
 500         &family_sink,
 501     );
 502 
 503     try runCudaRandomFamilyCandidateMeasurements(
 504         arena,
 505         backing_allocator,
 506         out,
 507         options,
 508         workload,
 509         handle,
 510         stream,
 511         &phase_emitter,
 512         &family_sink,
 513     );
 514 
 515     try runCudaRandomFoldMeasurements(
 516         arena,
 517         backing_allocator,
 518         out,
 519         options,
 520         workload,
 521         handle,
 522         stream,
 523         &phase_emitter,
 524     );
 525 
 526     try runCudaFilterFamilyCandidateMeasurements(
 527         arena,
 528         backing_allocator,
 529         out,
 530         options,
 531         workload,
 532         handle,
 533         stream,
 534         &phase_emitter,
 535         &family_sink,
 536     );
 537 
 538     try emitFamilyTuningRecords(arena, out, options, workload, &family_sink);
 539     try emitMatrixProductFamilyScheduleTuningRecords(arena, out, options, workload, &matrix_product_schedule_sink);
 540     try emitGeneratedScanScheduleTuningRecords(arena, out, options, workload, &scan_schedule_sink);
 541     try emitGeneratedRowPipelineScheduleTuningRecords(arena, out, options, workload, &row_pipeline_schedule_sink);
 542 }
 543 
 544 fn runCudaFilterFamilyCandidateMeasurements(
 545     arena: Allocator,
 546     backing_allocator: Allocator,
 547     out: *std.Io.Writer,
 548     options: config.Options,
 549     workload: workload_mod.Workload,
 550     handle: gpu.BackendHandle,
 551     stream: ?gpu.StreamHandle,
 552     phase_emitter: *BackendPhaseEmitter,
 553     family_sink: *FamilyTuningSink,
 554 ) !void {
 555     const family_shape = workload_mod.filterFamilyShape(options) orelse return;
 556 
 557     var candidates = try accy.kernel.library.selectOwnedFilterCandidates(backing_allocator, .{
 558         .dtype = .f32,
 559         .extent = family_shape.extent,
 560     });
 561     defer candidates.deinit();
 562     if (candidates.count == 0) return;
 563 
 564     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
 565         backing_allocator,
 566         handle,
 567         candidates.slice(),
 568         .{ .limits = .standard },
 569     );
 570     defer registry.deinit();
 571     const registry_value = registry.registry();
 572 
 573     const inputs = try workload_mod.buildFilterKernelCallInputs(arena, family_shape.extent);
 574 
 575     const measurement_start = timing_mod.nowNanos();
 576     for (candidates.slice()) |candidate| {
 577         const instance = accy.kernel.library.compaction.filterInstanceFromSpecialization(
 578             candidate.descriptor.metadata.specialization,
 579         ) orelse return error.InvalidKernelLibraryEntry;
 580         const module = try workload_mod.buildFilterKernelCallModule(
 581             backing_allocator,
 582             instance.extent,
 583             instance.padded(),
 584             candidate.descriptor.metadata.target,
 585             candidate.descriptor.metadata.version,
 586         );
 587         var module_owned = true;
 588         errdefer if (module_owned) module.deinit();
 589         module_owned = false;
 590         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
 591             .kernel_call_registry = &registry_value,
 592             .instrumentation = .{
 593                 .context = phase_emitter,
 594                 .observe = recordFragmentPhase,
 595             },
 596         });
 597         defer fragment.deinit();
 598 
 599         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
 600         defer bindings.deinit();
 601 
 602         const runtime_arguments = try accy.kernel.library.compaction.filterRuntimeArguments(instance);
 603         const records = try measureAndRecordFamilyCandidateRecords(
 604             arena,
 605             arena,
 606             fragment,
 607             bindings,
 608             candidate.descriptor.name,
 609             .{
 610                 .warmup = options.warmup,
 611                 .samples = options.samples,
 612                 .base_options = .{
 613                     .stream = stream,
 614                     .runtime_scalar_arguments = runtime_arguments[0..],
 615                 },
 616                 .synchronize = .stream,
 617             },
 618         );
 619         defer arena.free(records);
 620         try writeFilterFamilyCandidateRecords(out, options, workload, records);
 621         const tuning_key = try accy.kernel.library.compaction.filterFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
 622         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
 623     }
 624     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_filter_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
 625 }
 626 
 627 fn writeFilterFamilyCandidateRecords(
 628     out: *std.Io.Writer,
 629     options: config.Options,
 630     workload: workload_mod.Workload,
 631     records: []const accy.executable.LaunchCandidateRecord,
 632 ) !void {
 633     var best_record: ?accy.executable.LaunchCandidateRecord = null;
 634     for (records) |record| {
 635         try jsonl.writeFilterFamilyCandidateMeasurementRecord(out, options, workload, record);
 636         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
 637             if (best_record) |best| try jsonl.writeFilterFamilyCandidateBestRecord(out, options, workload, best);
 638             best_record = record;
 639         } else if (launchCandidateRecordBeats(record, best_record.?)) {
 640             best_record = record;
 641         }
 642     }
 643     if (best_record) |record| try jsonl.writeFilterFamilyCandidateBestRecord(out, options, workload, record);
 644 }
 645 
 646 const RandomMeasurementConfig = struct {
 647     algorithm: accy.kernel.library.RandomAlgorithm,
 648     rounds: u32,
 649 };
 650 
 651 const random_measurement_configs = [_]RandomMeasurementConfig{
 652     .{ .algorithm = .philox, .rounds = 10 },
 653     .{ .algorithm = .philox, .rounds = 7 },
 654     .{ .algorithm = .threefry, .rounds = 20 },
 655     .{ .algorithm = .threefry, .rounds = 13 },
 656     .{ .algorithm = .squares, .rounds = 0 },
 657 };
 658 
 659 fn runCudaRandomFamilyCandidateMeasurements(
 660     arena: Allocator,
 661     backing_allocator: Allocator,
 662     out: *std.Io.Writer,
 663     options: config.Options,
 664     workload: workload_mod.Workload,
 665     handle: gpu.BackendHandle,
 666     stream: gpu.StreamHandle,
 667     phase_emitter: *BackendPhaseEmitter,
 668     family_sink: *FamilyTuningSink,
 669 ) !void {
 670     const family_shape = workload_mod.randomFamilyShape(options) orelse return;
 671 
 672     const inputs = try workload_mod.buildRandomKernelCallInputs(arena);
 673 
 674     const measurement_start = timing_mod.nowNanos();
 675     for (random_measurement_configs) |measurement_config| {
 676         var candidates = try accy.kernel.library.selectOwnedRandomCandidates(backing_allocator, .{
 677             .dtype = .f32,
 678             .algorithm = measurement_config.algorithm,
 679             .count = family_shape.count,
 680             .rounds = measurement_config.rounds,
 681         });
 682         defer candidates.deinit();
 683         if (candidates.count == 0) continue;
 684 
 685         var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
 686             backing_allocator,
 687             handle,
 688             candidates.slice(),
 689             .{ .limits = .standard },
 690         );
 691         defer registry.deinit();
 692         const registry_value = registry.registry();
 693 
 694         for (candidates.slice()) |candidate| {
 695             const metadata = candidate.descriptor.metadata;
 696             const facts: RandomCandidateFacts = switch (measurement_config.algorithm) {
 697                 .philox => blk: {
 698                     const instance = accy.kernel.library.random.philoxInstanceFromSpecialization(
 699                         metadata.specialization,
 700                     ) orelse return error.InvalidKernelLibraryEntry;
 701                     break :blk .{
 702                         .runtime_arguments = try accy.kernel.library.random.philoxRuntimeArguments(instance),
 703                         .tuning_key = try accy.kernel.library.random.philoxFamilyTuningKey(arena, family_sink.device_fingerprint, instance),
 704                     };
 705                 },
 706                 .threefry => blk: {
 707                     const instance = accy.kernel.library.random.threefryInstanceFromSpecialization(
 708                         metadata.specialization,
 709                     ) orelse return error.InvalidKernelLibraryEntry;
 710                     break :blk .{
 711                         .runtime_arguments = try accy.kernel.library.random.threefryRuntimeArguments(instance),
 712                         .tuning_key = try accy.kernel.library.random.threefryFamilyTuningKey(arena, family_sink.device_fingerprint, instance),
 713                     };
 714                 },
 715                 .squares => blk: {
 716                     const instance = accy.kernel.library.random.squaresInstanceFromSpecialization(
 717                         metadata.specialization,
 718                     ) orelse return error.InvalidKernelLibraryEntry;
 719                     break :blk .{
 720                         .runtime_arguments = try accy.kernel.library.random.squaresRuntimeArguments(instance),
 721                         .tuning_key = try accy.kernel.library.random.squaresFamilyTuningKey(arena, family_sink.device_fingerprint, instance),
 722                     };
 723                 },
 724             };
 725             const module = try workload_mod.buildRandomKernelCallModule(
 726                 backing_allocator,
 727                 family_shape,
 728                 metadata.target,
 729                 metadata.version,
 730             );
 731             var module_owned = true;
 732             errdefer if (module_owned) module.deinit();
 733             module_owned = false;
 734             var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
 735                 .kernel_call_registry = &registry_value,
 736                 .instrumentation = .{
 737                     .context = phase_emitter,
 738                     .observe = recordFragmentPhase,
 739                 },
 740             });
 741             defer fragment.deinit();
 742 
 743             const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
 744             defer bindings.deinit();
 745 
 746             const records = try measureAndRecordFamilyCandidateRecords(
 747                 arena,
 748                 arena,
 749                 fragment,
 750                 bindings,
 751                 candidate.descriptor.name,
 752                 .{
 753                     .warmup = options.warmup,
 754                     .samples = options.samples,
 755                     .base_options = .{
 756                         .stream = stream,
 757                         .runtime_scalar_arguments = facts.runtime_arguments[0..],
 758                     },
 759                     .synchronize = .stream,
 760                 },
 761             );
 762             defer arena.free(records);
 763             try writeRandomFamilyCandidateRecords(out, options, workload, records);
 764             try family_sink.appendBest(facts.tuning_key, metadata.target, records);
 765         }
 766     }
 767     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_random_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
 768 }
 769 
 770 fn writeRandomFamilyCandidateRecords(
 771     out: *std.Io.Writer,
 772     options: config.Options,
 773     workload: workload_mod.Workload,
 774     records: []const accy.executable.LaunchCandidateRecord,
 775 ) !void {
 776     var best_record: ?accy.executable.LaunchCandidateRecord = null;
 777     for (records) |record| {
 778         try jsonl.writeRandomFamilyCandidateMeasurementRecord(out, options, workload, record);
 779         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
 780             if (best_record) |best| try jsonl.writeRandomFamilyCandidateBestRecord(out, options, workload, best);
 781             best_record = record;
 782         } else if (launchCandidateRecordBeats(record, best_record.?)) {
 783             best_record = record;
 784         }
 785     }
 786     if (best_record) |record| try jsonl.writeRandomFamilyCandidateBestRecord(out, options, workload, record);
 787 }
 788 
 789 const random_fold_sample_depths = [_]u32{ 1, 4, 16, 64, 256 };
 790 const random_fold_threads: u32 = 256;
 791 
 792 fn runCudaRandomFoldMeasurements(
 793     arena: Allocator,
 794     backing_allocator: Allocator,
 795     out: *std.Io.Writer,
 796     options: config.Options,
 797     workload: workload_mod.Workload,
 798     handle: gpu.BackendHandle,
 799     stream: gpu.StreamHandle,
 800     phase_emitter: *BackendPhaseEmitter,
 801 ) !void {
 802     const family_shape = workload_mod.randomFamilyShape(options) orelse return;
 803 
 804     const inputs = try workload_mod.buildRandomKernelCallInputs(arena);
 805     const random_library = accy.kernel.library.random;
 806 
 807     const measurement_start = timing_mod.nowNanos();
 808     for (random_measurement_configs) |measurement_config| {
 809         const lanes: u64 = switch (measurement_config.algorithm) {
 810             .philox => random_library.philox_lanes,
 811             .threefry => random_library.threefry_lanes,
 812             .squares => random_library.squares_lanes,
 813         };
 814         const outputs = family_shape.count / lanes;
 815         if (outputs == 0) continue;
 816 
 817         var owned_artifact = switch (measurement_config.algorithm) {
 818             .philox => try random_library.createPhiloxFoldFamilyArtifact(backing_allocator, handle, .{
 819                 .count = outputs,
 820                 .rounds = measurement_config.rounds,
 821                 .threads = random_fold_threads,
 822             }, .{ .limits = .standard }),
 823             .threefry => try random_library.createThreefryFoldFamilyArtifact(backing_allocator, handle, .{
 824                 .count = outputs,
 825                 .rounds = measurement_config.rounds,
 826                 .threads = random_fold_threads,
 827             }, .{ .limits = .standard }),
 828             .squares => try random_library.createSquaresFoldFamilyArtifact(backing_allocator, handle, .{
 829                 .count = outputs,
 830                 .threads = random_fold_threads,
 831             }, .{ .limits = .standard }),
 832         };
 833         defer owned_artifact.deinit();
 834         const registry_entries = [_]accy.artifact.KernelCallArtifact{owned_artifact.entry()};
 835         const registry_value = accy.artifact.KernelCallRegistry{ .entries = registry_entries[0..] };
 836 
 837         const family_entry = owned_artifact.entry();
 838         for (random_fold_sample_depths) |depth| {
 839             const module = try workload_mod.buildRandomKernelCallModule(
 840                 backing_allocator,
 841                 .{ .count = outputs },
 842                 family_entry.target,
 843                 family_entry.version,
 844             );
 845             var module_owned = true;
 846             errdefer if (module_owned) module.deinit();
 847             module_owned = false;
 848             var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
 849                 .kernel_call_registry = &registry_value,
 850                 .instrumentation = .{
 851                     .context = phase_emitter,
 852                     .observe = recordFragmentPhase,
 853                 },
 854             });
 855             defer fragment.deinit();
 856 
 857             const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
 858             defer bindings.deinit();
 859 
 860             const runtime_arguments = switch (measurement_config.algorithm) {
 861                 .philox => try random_library.philoxFoldRuntimeArguments(.{
 862                     .count = outputs,
 863                     .samples = depth,
 864                     .rounds = measurement_config.rounds,
 865                     .threads = random_fold_threads,
 866                 }),
 867                 .threefry => try random_library.threefryFoldRuntimeArguments(.{
 868                     .count = outputs,
 869                     .samples = depth,
 870                     .rounds = measurement_config.rounds,
 871                     .threads = random_fold_threads,
 872                 }),
 873                 .squares => try random_library.squaresFoldRuntimeArguments(.{
 874                     .count = outputs,
 875                     .samples = depth,
 876                     .threads = random_fold_threads,
 877                 }),
 878             };
 879             const records = try measureAndRecordFamilyCandidateRecords(
 880                 arena,
 881                 arena,
 882                 fragment,
 883                 bindings,
 884                 family_entry.entry_name,
 885                 .{
 886                     .warmup = options.warmup,
 887                     .samples = options.samples,
 888                     .base_options = .{
 889                         .stream = stream,
 890                         .runtime_scalar_arguments = runtime_arguments[0..],
 891                     },
 892                     .synchronize = .stream,
 893                 },
 894             );
 895             defer arena.free(records);
 896             const entry_label = try std.fmt.allocPrint(arena, "{s}_k{d}", .{ family_entry.entry_name, depth });
 897             for (records) |*record| record.kernel.entry_name = entry_label;
 898             try writeRandomFoldCandidateRecords(out, options, workload, records);
 899         }
 900     }
 901     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_random_fold_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
 902 }
 903 
 904 fn writeRandomFoldCandidateRecords(
 905     out: *std.Io.Writer,
 906     options: config.Options,
 907     workload: workload_mod.Workload,
 908     records: []const accy.executable.LaunchCandidateRecord,
 909 ) !void {
 910     var best_record: ?accy.executable.LaunchCandidateRecord = null;
 911     for (records) |record| {
 912         try jsonl.writeRandomFoldCandidateMeasurementRecord(out, options, workload, record);
 913         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
 914             if (best_record) |best| try jsonl.writeRandomFoldCandidateBestRecord(out, options, workload, best);
 915             best_record = record;
 916         } else if (launchCandidateRecordBeats(record, best_record.?)) {
 917             best_record = record;
 918         }
 919     }
 920     if (best_record) |record| try jsonl.writeRandomFoldCandidateBestRecord(out, options, workload, record);
 921 }
 922 
 923 fn runCudaMatrixProductFamilyCandidateMeasurements(
 924     arena: Allocator,
 925     backing_allocator: Allocator,
 926     out: *std.Io.Writer,
 927     options: config.Options,
 928     workload: workload_mod.Workload,
 929     inputs: workload_mod.LaunchInputs,
 930     handle: gpu.BackendHandle,
 931     stream: gpu.StreamHandle,
 932     phase_emitter: *BackendPhaseEmitter,
 933     family_sink: *FamilyTuningSink,
 934     matrix_product_schedule_sink: *MatrixProductFamilyScheduleTuningSink,
 935 ) !void {
 936     const family_shape = workload_mod.matrixProductFamilyShape(options) orelse return;
 937     const lhs_dims = [_]i64{ try i64Extent(family_shape.m), try i64Extent(family_shape.k) };
 938     const rhs_dims = [_]i64{ try i64Extent(family_shape.k), try i64Extent(family_shape.n) };
 939     const output_dims = [_]i64{ try i64Extent(family_shape.m), try i64Extent(family_shape.n) };
 940 
 941     var candidates = try accy.kernel.library.selectOwnedMatrixProductCandidates(backing_allocator, .{
 942         .dtype = .f32,
 943         .lhs_indices = "mk",
 944         .rhs_indices = "kn",
 945         .output_indices = "mn",
 946         .lhs_dims = &lhs_dims,
 947         .rhs_dims = &rhs_dims,
 948         .output_dims = &output_dims,
 949     });
 950     defer candidates.deinit();
 951     if (candidates.count <= 1) return;
 952 
 953     const capacity = family_tuning.matrix_product_family_schedule_tuning_max_candidates;
 954     var candidate_instances: [capacity]accy.kernel.library.linalg.MatrixProduct = undefined;
 955     var schedule_candidates: [capacity]family_tuning.MatrixProductFamilyScheduleThreads = undefined;
 956     if (candidates.count > candidate_instances.len) return error.InvalidArtifact;
 957     for (candidates.slice(), 0..) |candidate, index| {
 958         const instance = accy.kernel.library.linalg.matrixProductInstanceFromSpecialization(
 959             candidate.descriptor.metadata.specialization,
 960         ) orelse return error.InvalidKernelLibraryEntry;
 961         candidate_instances[index] = instance;
 962         schedule_candidates[index] = .{ .x = instance.threads.x, .y = instance.threads.y };
 963     }
 964     const schedule_problem = family_tuning.MatrixProductFamilyScheduleTuningProblem{
 965         .format = .cuda_ptx,
 966         .m = family_shape.m,
 967         .n = family_shape.n,
 968         .k = family_shape.k,
 969         .dtype = candidate_instances[0].dtype,
 970         .accumulation_dtype = candidate_instances[0].accumulation_dtype,
 971         .family_version = accy.kernel.library.linalg.matrix_product_family_version,
 972         .candidates = schedule_candidates[0..candidates.count],
 973     };
 974     var schedule_measurements: [capacity]MatrixProductFamilyScheduleMeasurement = undefined;
 975     var schedule_measurement_count: usize = 0;
 976 
 977     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
 978         backing_allocator,
 979         handle,
 980         candidates.slice(),
 981         .{ .limits = .standard },
 982     );
 983     defer registry.deinit();
 984     const registry_value = registry.registry();
 985 
 986     const measurement_start = timing_mod.nowNanos();
 987     for (candidates.slice(), 0..) |candidate, candidate_index| {
 988         const instance = candidate_instances[candidate_index];
 989         const schedule = accy.kernel.library.MatrixProductSchedule{ .thread_blocks = instance.threads };
 990         const module = try workload_mod.buildSemanticModule(backing_allocator, options);
 991         var module_owned = true;
 992         errdefer if (module_owned) module.deinit();
 993         module_owned = false;
 994         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
 995             .kernel_call_registry = &registry_value,
 996             .matrix_product_schedule = schedule,
 997             .instrumentation = .{
 998                 .context = phase_emitter,
 999                 .observe = recordFragmentPhase,
1000             },
1001         });
1002         defer fragment.deinit();
1003 
1004         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
1005         defer bindings.deinit();
1006 
1007         const runtime_arguments = try accy.kernel.library.linalg.matrixProductRuntimeArguments(instance);
1008         const records = try measureAndRecordFamilyCandidateRecords(
1009             arena,
1010             arena,
1011             fragment,
1012             bindings,
1013             candidate.descriptor.name,
1014             .{
1015                 .warmup = options.warmup,
1016                 .samples = options.samples,
1017                 .base_options = .{
1018                     .stream = stream,
1019                     .runtime_scalar_arguments = runtime_arguments[0..],
1020                 },
1021                 .synchronize = .stream,
1022             },
1023         );
1024         defer arena.free(records);
1025         try writeMatrixProductFamilyCandidateRecords(out, options, workload, records);
1026         const tuning_key = try accy.kernel.library.linalg.matrixProductFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
1027         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
1028         if (bestFamilyCandidateRecord(records)) |best| {
1029             if (best.median_ns != 0) {
1030                 schedule_measurements[schedule_measurement_count] = .{
1031                     .threads = schedule_candidates[candidate_index],
1032                     .median_ns = best.median_ns,
1033                     .sample_count = best.sample_count,
1034                 };
1035                 schedule_measurement_count += 1;
1036             }
1037         }
1038     }
1039     try matrix_product_schedule_sink.recordCandidateSet(
1040         schedule_problem,
1041         schedule_measurements[0..schedule_measurement_count],
1042     );
1043     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_matrix_product_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
1044 }
1045 
1046 fn measureAndRecordFamilyCandidateRecords(
1047     result_allocator: Allocator,
1048     scratch: Allocator,
1049     fragment: *accy.executable.LoadedFragment,
1050     bindings: *const accy.executable.Invocation,
1051     entry_name: []const u8,
1052     options: accy.executable.LaunchCandidateBenchmarkOptions,
1053 ) ![]accy.executable.LaunchCandidateRecord {
1054     var records = std.ArrayListUnmanaged(accy.executable.LaunchCandidateRecord).empty;
1055     errdefer records.deinit(result_allocator);
1056 
1057     var kernel_index: usize = 0;
1058     while (kernel_index < fragment.kernelCount()) : (kernel_index += 1) {
1059         const summary = try fragment.kernelSummary(kernel_index);
1060         if (!std.mem.eql(u8, summary.entry_name, entry_name)) continue;
1061         if (summary.launch_candidate_count == 0) continue;
1062         const measured = try bindings.measureLaunchCandidateRecords(
1063             scratch,
1064             scratch,
1065             kernel_index,
1066             options,
1067         );
1068         defer scratch.free(measured);
1069         records.appendSlice(result_allocator, measured) catch return error.OutOfMemory;
1070     }
1071 
1072     const owned_records = records.toOwnedSlice(result_allocator) catch return error.OutOfMemory;
1073     errdefer result_allocator.free(owned_records);
1074     try fragment.recordLaunchCandidateRecords(owned_records);
1075     return owned_records;
1076 }
1077 
1078 const RandomCandidateFacts = struct {
1079     runtime_arguments: [3]choir_abi.ScalarArgument,
1080     tuning_key: family_tuning.FamilyTuningKey,
1081 };
1082 
1083 fn runCudaSortStructureFamilyMeasurements(
1084     arena: Allocator,
1085     backing_allocator: Allocator,
1086     out: *std.Io.Writer,
1087     options: config.Options,
1088     workload: workload_mod.Workload,
1089     handle: gpu.BackendHandle,
1090     family_sink: *FamilyTuningSink,
1091 ) !void {
1092     _ = out;
1093     _ = workload;
1094     const family_shape = workload_mod.sortFamilyShape(options) orelse return;
1095     const library = accy.kernel.library;
1096     const extent = family_shape.extent;
1097     const threads = library.sort.radixSplitThreadsForExtent(extent) orelse return;
1098     const instance = library.sort.RadixSplit{ .extent = extent, .threads = threads };
1099     if (!library.sort.radixSplitInstanceValid(instance)) return;
1100     const key = try library.sort.radixSplitFamilyTuningKey(backing_allocator, family_sink.device_fingerprint, instance);
1101 
1102     const keys = try arena.alloc(i32, extent);
1103     var seed: u32 = 0x2545f491;
1104     for (keys) |*key_value| {
1105         seed ^= seed << 13;
1106         seed ^= seed >> 17;
1107         seed ^= seed << 5;
1108         const magnitude: i32 = @intCast(seed % 1000000);
1109         key_value.* = if (seed & 1 == 1) -magnitude else magnitude;
1110     }
1111 
1112     var buffers: [2]gpu.BufferHandle = undefined;
1113     var buffer_count: usize = 0;
1114     defer destroyBufferHandles(handle, buffers[0..buffer_count]);
1115     for (&buffers) |*buffer| {
1116         buffer.* = try handle.allocateBuffer(.{
1117             .byte_size = extent * @sizeOf(i32),
1118             .alignment = 256,
1119             .dtype = .i32,
1120             .element_count = extent,
1121         });
1122         buffer_count += 1;
1123     }
1124     var bindings: [2]gpu.BufferBinding = undefined;
1125     for (buffers, 0..) |buffer, index| {
1126         bindings[index] = .{
1127             .handle = buffer,
1128             .access = .read_write,
1129             .ownership = buffer.ownership,
1130             .byte_size = buffer.byte_size,
1131         };
1132     }
1133 
1134     const structures = [_]library.catalog.SortStructure{ .radix_split, .radix_digit };
1135     inline for (structures) |structure| {
1136         var descriptor = (try library.catalog.selectOwned(backing_allocator, .{ .sort = .{
1137             .dtype = .i32,
1138             .kind = .radix_ascending,
1139             .extent = extent,
1140             .structure = structure,
1141             .schedule = .{ .thread_blocks = threads },
1142         } })) orelse return;
1143         defer descriptor.deinit();
1144         var package = (try library.createOwnedKernelCallPipelinePackage(backing_allocator, handle, descriptor, .{ .limits = .standard })) orelse {
1145             return;
1146         };
1147         defer package.deinit();
1148 
1149         const pipeline_value = package.pipeline.value;
1150         const pass_args = [_]choir_abi.ScalarArgument{ .{ .u32 = @intCast(extent) }, .{ .u32 = 0 }, .{ .u32 = 0 } };
1151         const scalar_count = pipeline_value.runtime_scalar_argument_count;
1152         const pooled = try accy.executable.allocatePipelineIntermediates(
1153             backing_allocator,
1154             handle,
1155             pipeline_value,
1156             pass_args[0..scalar_count],
1157         );
1158         defer accy.executable.deinitPipelineIntermediates(backing_allocator, handle, pooled);
1159         var artifact_pool = try accy.executable.loadPipelineArtifacts(
1160             backing_allocator,
1161             handle,
1162             pipeline_value,
1163             package.registry(),
1164             .cuda_ptx,
1165         );
1166         defer artifact_pool.deinit();
1167 
1168         const pass_step: u32 = if (structure == .radix_split) 1 else library.sort.radix_digit_bits;
1169         const reps: usize = 20;
1170         var durations: [reps]u64 = undefined;
1171         for (&durations) |*duration| {
1172             try handle.writeBuffer(.{ .handle = buffers[0], .bytes = std.mem.sliceAsBytes(keys) });
1173             const start = timing_mod.nowNanos();
1174             var source: usize = 0;
1175             var shift: u32 = 0;
1176             while (shift < library.sort.radix_split_key_bits) : (shift += pass_step) {
1177                 const runtime_arguments = if (structure == .radix_split)
1178                     try library.sort.radixSplitPipelineRuntimeArguments(instance, shift)
1179                 else
1180                     try library.sort.radixDigitHistogramRuntimeArguments(instance, shift);
1181                 try accy.executable.launchPipeline(backing_allocator, handle, .{
1182                     .pipeline = pipeline_value,
1183                     .registry = package.registry(),
1184                     .format = .cuda_ptx,
1185                     .operands = bindings[source .. source + 1],
1186                     .results = bindings[1 - source .. 2 - source],
1187                     .runtime_scalar_arguments = runtime_arguments[0..],
1188                     .intermediates = pooled,
1189                     .artifacts = &artifact_pool,
1190                 });
1191                 source = 1 - source;
1192             }
1193             try handle.synchronize(.{ .scope = .device });
1194             duration.* = @intCast(timing_mod.nowNanos() - start);
1195         }
1196         std.mem.sort(u64, durations[0..], {}, std.sort.asc(u64));
1197         const median = durations[reps / 2];
1198         try family_sink.accumulator.append(key, descriptor.descriptor.metadata.target, median, reps);
1199     }
1200 }
1201 
1202 fn destroyBufferHandles(handle: gpu.BackendHandle, buffers: []const gpu.BufferHandle) void {
1203     for (buffers) |buffer| handle.destroyObject(buffer.id);
1204 }
1205 
1206 const FamilyTuningSink = struct {
1207     device_fingerprint: u64,
1208     accumulator: family_tuning.FamilyMeasurementAccumulator,
1209 
1210     fn appendBest(
1211         self: *FamilyTuningSink,
1212         key: family_tuning.FamilyTuningKey,
1213         target: []const u8,
1214         records: []const accy.executable.LaunchCandidateRecord,
1215     ) !void {
1216         const best = bestFamilyCandidateRecord(records) orelse return;
1217         if (best.median_ns == 0) return;
1218         try self.accumulator.append(key, target, best.median_ns, best.sample_count);
1219     }
1220 
1221     fn deinit(self: *FamilyTuningSink) void {
1222         self.accumulator.deinit();
1223     }
1224 };
1225 
1226 fn bestFamilyCandidateRecord(records: []const accy.executable.LaunchCandidateRecord) ?accy.executable.LaunchCandidateRecord {
1227     var best: ?accy.executable.LaunchCandidateRecord = null;
1228     for (records) |record| {
1229         if (best == null or launchCandidateRecordBeats(record, best.?)) best = record;
1230     }
1231     return best;
1232 }
1233 
1234 const MatrixProductFamilyScheduleMeasurement = struct {
1235     threads: family_tuning.MatrixProductFamilyScheduleThreads,
1236     median_ns: u64,
1237     sample_count: u32,
1238 };
1239 
1240 const MatrixProductFamilyScheduleTuningSink = struct {
1241     caps: gpu.BackendCapabilities,
1242     measurement_count: usize = 0,
1243     cache: schedule_tuning.MatrixProductFamilyScheduleTuningCache,
1244 
1245     fn init(
1246         allocator: std.mem.Allocator,
1247         caps: gpu.BackendCapabilities,
1248     ) MatrixProductFamilyScheduleTuningSink {
1249         return .{
1250             .caps = caps,
1251             .cache = schedule_tuning.MatrixProductFamilyScheduleTuningCache.init(allocator),
1252         };
1253     }
1254 
1255     fn recordCandidateSet(
1256         self: *MatrixProductFamilyScheduleTuningSink,
1257         problem: family_tuning.MatrixProductFamilyScheduleTuningProblem,
1258         measurements: []const MatrixProductFamilyScheduleMeasurement,
1259     ) !void {
1260         self.measurement_count += measurements.len;
1261         if (measurements.len < 2) return;
1262         const selected = selectMatrixProductFamilyScheduleMeasurement(measurements) orelse return;
1263         const margin_floor = selected.winner_median_ns + selected.winner_median_ns * family_tuning.family_tuning_default_margin_percent / 100;
1264         if (selected.runner_up_median_ns < margin_floor) return;
1265         try self.cache.recordSelection(self.caps, problem, selected);
1266     }
1267 
1268     fn deinit(self: *MatrixProductFamilyScheduleTuningSink) void {
1269         self.cache.deinit();
1270     }
1271 };
1272 
1273 fn selectMatrixProductFamilyScheduleMeasurement(
1274     measurements: []const MatrixProductFamilyScheduleMeasurement,
1275 ) ?family_tuning.MatrixProductFamilyScheduleTuningSelection {
1276     var winner: ?MatrixProductFamilyScheduleMeasurement = null;
1277     var runner_up_ns: u64 = 0;
1278     for (measurements) |measurement| {
1279         if (measurement.median_ns == 0 or measurement.sample_count == 0) continue;
1280         if (winner == null or matrixProductFamilyScheduleMeasurementBeats(measurement, winner.?)) {
1281             if (winner) |previous| {
1282                 if (runner_up_ns == 0 or previous.median_ns < runner_up_ns) runner_up_ns = previous.median_ns;
1283             }
1284             winner = measurement;
1285         } else if (runner_up_ns == 0 or measurement.median_ns < runner_up_ns) {
1286             runner_up_ns = measurement.median_ns;
1287         }
1288     }
1289     const selected = winner orelse return null;
1290     if (runner_up_ns == 0) return null;
1291     return .{
1292         .threads = selected.threads,
1293         .winner_median_ns = selected.median_ns,
1294         .runner_up_median_ns = runner_up_ns,
1295         .sample_count = selected.sample_count,
1296     };
1297 }
1298 
1299 fn matrixProductFamilyScheduleMeasurementBeats(
1300     lhs: MatrixProductFamilyScheduleMeasurement,
1301     rhs: MatrixProductFamilyScheduleMeasurement,
1302 ) bool {
1303     if (lhs.median_ns != rhs.median_ns) return lhs.median_ns < rhs.median_ns;
1304     if (lhs.sample_count != rhs.sample_count) return lhs.sample_count > rhs.sample_count;
1305     if (lhs.threads.x != rhs.threads.x) return lhs.threads.x < rhs.threads.x;
1306     return lhs.threads.y < rhs.threads.y;
1307 }
1308 
1309 fn emitFamilyTuningRecords(
1310     arena: Allocator,
1311     out: *std.Io.Writer,
1312     options: config.Options,
1313     workload: workload_mod.Workload,
1314     sink: *FamilyTuningSink,
1315 ) !void {
1316     var winners = try sink.accumulator.selectWinners(arena, family_tuning.family_tuning_default_margin_percent);
1317     defer winners.deinit();
1318     for (winners.records) |record| {
1319         try jsonl.writeFamilyTuningWinnerRecord(out, options, workload, record);
1320     }
1321     const artifact = try family_tuning.encodeFamilyTuningArtifact(arena, winners.records);
1322     defer arena.free(artifact);
1323     try jsonl.writeFamilyTuningArtifactRecord(
1324         out,
1325         options,
1326         workload,
1327         sink.accumulator.count(),
1328         winners.records.len,
1329         artifact.len,
1330         family_tuning.artifactFingerprint(artifact),
1331     );
1332     if (options.family_tuning_out) |artifact_path| {
1333         try sys.fs.writeFile(artifact_path, artifact);
1334     }
1335 }
1336 
1337 fn emitMatrixProductFamilyScheduleTuningRecords(
1338     arena: Allocator,
1339     out: *std.Io.Writer,
1340     options: config.Options,
1341     workload: workload_mod.Workload,
1342     sink: *MatrixProductFamilyScheduleTuningSink,
1343 ) !void {
1344     if (sink.measurement_count == 0 and options.matrix_product_schedule_tuning_out == null) return;
1345     const records = try sink.cache.exportRecords(arena);
1346     defer arena.free(records);
1347     for (records) |record| {
1348         try jsonl.writeMatrixProductFamilyScheduleTuningWinnerRecord(out, options, workload, record);
1349     }
1350     const artifact = try schedule_tuning.encodeMatrixProductFamilyScheduleTuningArtifact(arena, records);
1351     defer arena.free(artifact);
1352     try jsonl.writeMatrixProductFamilyScheduleTuningArtifactRecord(
1353         out,
1354         options,
1355         workload,
1356         sink.measurement_count,
1357         records.len,
1358         artifact.len,
1359         schedule_tuning.matrixProductFamilyScheduleTuningArtifactFingerprint(artifact),
1360     );
1361     if (options.matrix_product_schedule_tuning_out) |artifact_path| {
1362         try sys.fs.writeFile(artifact_path, artifact);
1363     }
1364 }
1365 
1366 const GeneratedRowPipelineScheduleMeasurement = struct {
1367     schedule: accy.preparation.target.GeneratedRowPipelineSchedule,
1368     median_ns: u64,
1369     sample_count: u32,
1370 };
1371 
1372 const GeneratedRowPipelineScheduleTuningSink = struct {
1373     caps: gpu.BackendCapabilities,
1374     measurement_count: usize = 0,
1375     cache: schedule_tuning.GeneratedRowPipelineScheduleTuningCache,
1376 
1377     fn init(
1378         allocator: std.mem.Allocator,
1379         caps: gpu.BackendCapabilities,
1380     ) GeneratedRowPipelineScheduleTuningSink {
1381         return .{
1382             .caps = caps,
1383             .cache = schedule_tuning.GeneratedRowPipelineScheduleTuningCache.init(allocator),
1384         };
1385     }
1386 
1387     fn recordCandidateSet(
1388         self: *GeneratedRowPipelineScheduleTuningSink,
1389         problem: schedule_tuning.GeneratedRowPipelineScheduleTuningProblem,
1390         measurements: []const GeneratedRowPipelineScheduleMeasurement,
1391     ) !void {
1392         self.measurement_count += measurements.len;
1393         if (measurements.len < 2) return;
1394         const selected = selectGeneratedRowPipelineScheduleMeasurement(measurements) orelse return;
1395         const margin_floor = selected.winner_median_ns + selected.winner_median_ns * family_tuning.family_tuning_default_margin_percent / 100;
1396         if (selected.runner_up_median_ns < margin_floor) return;
1397         try self.cache.recordSelection(self.caps, problem, selected);
1398     }
1399 
1400     fn deinit(self: *GeneratedRowPipelineScheduleTuningSink) void {
1401         self.cache.deinit();
1402     }
1403 };
1404 
1405 fn selectGeneratedRowPipelineScheduleMeasurement(
1406     measurements: []const GeneratedRowPipelineScheduleMeasurement,
1407 ) ?schedule_tuning.GeneratedRowPipelineScheduleTuningSelection {
1408     var winner: ?GeneratedRowPipelineScheduleMeasurement = null;
1409     var runner_up_ns: u64 = 0;
1410     for (measurements) |measurement| {
1411         if (measurement.median_ns == 0 or measurement.sample_count == 0) continue;
1412         if (winner == null or generatedRowPipelineScheduleMeasurementBeats(measurement, winner.?)) {
1413             if (winner) |previous| {
1414                 if (runner_up_ns == 0 or previous.median_ns < runner_up_ns) runner_up_ns = previous.median_ns;
1415             }
1416             winner = measurement;
1417         } else if (runner_up_ns == 0 or measurement.median_ns < runner_up_ns) {
1418             runner_up_ns = measurement.median_ns;
1419         }
1420     }
1421     const selected = winner orelse return null;
1422     if (runner_up_ns == 0) return null;
1423     return .{
1424         .schedule = selected.schedule,
1425         .winner_median_ns = selected.median_ns,
1426         .runner_up_median_ns = runner_up_ns,
1427         .sample_count = selected.sample_count,
1428     };
1429 }
1430 
1431 fn generatedRowPipelineScheduleMeasurementBeats(
1432     lhs: GeneratedRowPipelineScheduleMeasurement,
1433     rhs: GeneratedRowPipelineScheduleMeasurement,
1434 ) bool {
1435     if (lhs.median_ns != rhs.median_ns) return lhs.median_ns < rhs.median_ns;
1436     if (lhs.sample_count != rhs.sample_count) return lhs.sample_count > rhs.sample_count;
1437     return lhs.schedule.threads < rhs.schedule.threads;
1438 }
1439 
1440 fn runCudaGeneratedRowPipelineScheduleMeasurements(
1441     arena: Allocator,
1442     backing_allocator: Allocator,
1443     out: *std.Io.Writer,
1444     options: config.Options,
1445     workload: workload_mod.Workload,
1446     handle: gpu.BackendHandle,
1447     stream: gpu.StreamHandle,
1448     phase_emitter: *BackendPhaseEmitter,
1449     row_pipeline_schedule_sink: *GeneratedRowPipelineScheduleTuningSink,
1450 ) !void {
1451     if (options.row_pipeline_schedule_tuning_out == null) return;
1452 
1453     const rows: u64 = options.elements;
1454     const cols: u64 = options.elements;
1455     var candidate_buffer: [accy.preparation.kernelization.max_row_pipeline_schedule_candidates]accy.preparation.target.GeneratedRowPipelineSchedule = undefined;
1456     const candidates = accy.preparation.kernelization.rowPipelineScheduleCandidates(cols, .cuda_ptx, &candidate_buffer);
1457     if (candidates.len < 2) return;
1458 
1459     const problem = schedule_tuning.GeneratedRowPipelineScheduleTuningProblem{
1460         .format = .cuda_ptx,
1461         .rows = rows,
1462         .cols = cols,
1463         .dtype = .f32,
1464         .schedule_version = accy.preparation.kernelization.generated_row_pipeline_schedule_version,
1465         .candidates = candidates,
1466     };
1467 
1468     var measurements: [accy.preparation.kernelization.max_row_pipeline_schedule_candidates]GeneratedRowPipelineScheduleMeasurement = undefined;
1469     var measurement_count: usize = 0;
1470 
1471     const inputs = try workload_mod.buildRowPipelineLaunchInputs(arena, rows, cols);
1472 
1473     const measurement_start = timing_mod.nowNanos();
1474     for (candidates) |candidate| {
1475         const module = try workload_mod.buildRowPipelineSemanticModule(backing_allocator, rows, cols);
1476         var module_owned = true;
1477         errdefer if (module_owned) module.deinit();
1478         module_owned = false;
1479         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
1480             .generated_row_pipeline_schedule = candidate,
1481             .instrumentation = .{
1482                 .context = phase_emitter,
1483                 .observe = recordFragmentPhase,
1484             },
1485         });
1486         defer fragment.deinit();
1487 
1488         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
1489         defer bindings.deinit();
1490 
1491         var best_median: u64 = 0;
1492         var best_samples: u32 = 0;
1493         var kernel_index: usize = 0;
1494         while (kernel_index < fragment.kernelCount()) : (kernel_index += 1) {
1495             const summary = try fragment.kernelSummary(kernel_index);
1496             if (std.mem.indexOf(u8, summary.entry_name, "row_pipeline") == null) continue;
1497             if (summary.launch_geometry.threadgroup[0] != candidate.threads) return error.LaunchArgumentMismatch;
1498             const records = try bindings.measureLaunchCandidateRecords(
1499                 arena,
1500                 arena,
1501                 kernel_index,
1502                 .{
1503                     .warmup = options.warmup,
1504                     .samples = options.samples,
1505                     .base_options = .{ .stream = stream },
1506                     .synchronize = .stream,
1507                 },
1508             );
1509             defer arena.free(records);
1510             for (records) |record| {
1511                 try jsonl.writeGeneratedRowPipelineScheduleCandidateMeasurementRecord(out, options, workload, record);
1512                 if (best_median == 0 or record.median_ns < best_median) {
1513                     best_median = record.median_ns;
1514                     best_samples = record.sample_count;
1515                 }
1516             }
1517         }
1518         if (best_median != 0) {
1519             measurements[measurement_count] = .{
1520                 .schedule = candidate,
1521                 .median_ns = best_median,
1522                 .sample_count = best_samples,
1523             };
1524             measurement_count += 1;
1525         }
1526     }
1527     try row_pipeline_schedule_sink.recordCandidateSet(problem, measurements[0..measurement_count]);
1528     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_generated_row_pipeline_schedule_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
1529 }
1530 
1531 fn emitGeneratedRowPipelineScheduleTuningRecords(
1532     arena: Allocator,
1533     out: *std.Io.Writer,
1534     options: config.Options,
1535     workload: workload_mod.Workload,
1536     sink: *GeneratedRowPipelineScheduleTuningSink,
1537 ) !void {
1538     if (sink.measurement_count == 0 and options.row_pipeline_schedule_tuning_out == null) return;
1539     const records = try sink.cache.exportRecords(arena);
1540     defer arena.free(records);
1541     for (records) |record| {
1542         try jsonl.writeGeneratedRowPipelineScheduleTuningWinnerRecord(out, options, workload, record);
1543     }
1544     const artifact = try schedule_tuning.encodeGeneratedRowPipelineScheduleTuningArtifact(arena, records);
1545     defer arena.free(artifact);
1546     try jsonl.writeGeneratedRowPipelineScheduleTuningArtifactRecord(
1547         out,
1548         options,
1549         workload,
1550         sink.measurement_count,
1551         records.len,
1552         artifact.len,
1553         schedule_tuning.generatedRowPipelineScheduleTuningArtifactFingerprint(artifact),
1554     );
1555     if (options.row_pipeline_schedule_tuning_out) |artifact_path| {
1556         try sys.fs.writeFile(artifact_path, artifact);
1557     }
1558 }
1559 
1560 const GeneratedScanScheduleMeasurement = struct {
1561     schedule: accy.preparation.target.GeneratedScanSchedule,
1562     median_ns: u64,
1563     sample_count: u32,
1564 };
1565 
1566 const GeneratedScanScheduleTuningSink = struct {
1567     caps: gpu.BackendCapabilities,
1568     measurement_count: usize = 0,
1569     cache: schedule_tuning.GeneratedScanScheduleTuningCache,
1570 
1571     fn init(
1572         allocator: std.mem.Allocator,
1573         caps: gpu.BackendCapabilities,
1574     ) GeneratedScanScheduleTuningSink {
1575         return .{
1576             .caps = caps,
1577             .cache = schedule_tuning.GeneratedScanScheduleTuningCache.init(allocator),
1578         };
1579     }
1580 
1581     fn recordCandidateSet(
1582         self: *GeneratedScanScheduleTuningSink,
1583         problem: schedule_tuning.GeneratedScanScheduleTuningProblem,
1584         measurements: []const GeneratedScanScheduleMeasurement,
1585     ) !void {
1586         self.measurement_count += measurements.len;
1587         if (measurements.len < 2) return;
1588         const selected = selectGeneratedScanScheduleMeasurement(measurements) orelse return;
1589         const margin_floor = selected.winner_median_ns + selected.winner_median_ns * family_tuning.family_tuning_default_margin_percent / 100;
1590         if (selected.runner_up_median_ns < margin_floor) return;
1591         try self.cache.recordSelection(self.caps, problem, selected);
1592     }
1593 
1594     fn deinit(self: *GeneratedScanScheduleTuningSink) void {
1595         self.cache.deinit();
1596     }
1597 };
1598 
1599 fn selectGeneratedScanScheduleMeasurement(
1600     measurements: []const GeneratedScanScheduleMeasurement,
1601 ) ?schedule_tuning.GeneratedScanScheduleTuningSelection {
1602     var winner: ?GeneratedScanScheduleMeasurement = null;
1603     var runner_up_ns: u64 = 0;
1604     for (measurements) |measurement| {
1605         if (measurement.median_ns == 0 or measurement.sample_count == 0) continue;
1606         if (winner == null or generatedScanScheduleMeasurementBeats(measurement, winner.?)) {
1607             if (winner) |previous| {
1608                 if (runner_up_ns == 0 or previous.median_ns < runner_up_ns) runner_up_ns = previous.median_ns;
1609             }
1610             winner = measurement;
1611         } else if (runner_up_ns == 0 or measurement.median_ns < runner_up_ns) {
1612             runner_up_ns = measurement.median_ns;
1613         }
1614     }
1615     const selected = winner orelse return null;
1616     if (runner_up_ns == 0) return null;
1617     return .{
1618         .schedule = selected.schedule,
1619         .winner_median_ns = selected.median_ns,
1620         .runner_up_median_ns = runner_up_ns,
1621         .sample_count = selected.sample_count,
1622     };
1623 }
1624 
1625 fn generatedScanScheduleMeasurementBeats(
1626     lhs: GeneratedScanScheduleMeasurement,
1627     rhs: GeneratedScanScheduleMeasurement,
1628 ) bool {
1629     if (lhs.median_ns != rhs.median_ns) return lhs.median_ns < rhs.median_ns;
1630     if (lhs.sample_count != rhs.sample_count) return lhs.sample_count > rhs.sample_count;
1631     if (lhs.schedule.threads != rhs.schedule.threads) return lhs.schedule.threads < rhs.schedule.threads;
1632     return lhs.schedule.items < rhs.schedule.items;
1633 }
1634 
1635 fn runCudaGeneratedScanScheduleMeasurements(
1636     arena: Allocator,
1637     backing_allocator: Allocator,
1638     out: *std.Io.Writer,
1639     options: config.Options,
1640     workload: workload_mod.Workload,
1641     handle: gpu.BackendHandle,
1642     stream: gpu.StreamHandle,
1643     phase_emitter: *BackendPhaseEmitter,
1644     scan_schedule_sink: *GeneratedScanScheduleTuningSink,
1645 ) !void {
1646     if (options.scan_schedule_tuning_out == null) return;
1647 
1648     const total: u64 = options.elements;
1649     var candidate_buffer: [accy.preparation.kernelization.max_scan_schedule_candidates]accy.preparation.target.GeneratedScanSchedule = undefined;
1650     const candidates = accy.preparation.kernelization.scanScheduleCandidates(total, .cuda_ptx, &candidate_buffer);
1651     if (candidates.len < 2) return;
1652 
1653     const problem = schedule_tuning.GeneratedScanScheduleTuningProblem{
1654         .format = .cuda_ptx,
1655         .total = total,
1656         .dtype = .f32,
1657         .schedule_version = accy.preparation.kernelization.generated_scan_schedule_version,
1658         .candidates = candidates,
1659     };
1660 
1661     var measurements: [accy.preparation.kernelization.max_scan_schedule_candidates]GeneratedScanScheduleMeasurement = undefined;
1662     var measurement_count: usize = 0;
1663 
1664     const inputs = try workload_mod.buildCumsumLaunchInputs(arena, total);
1665 
1666     const measurement_start = timing_mod.nowNanos();
1667     for (candidates) |candidate| {
1668         const module = try workload_mod.buildCumsumSemanticModule(backing_allocator, total);
1669         var module_owned = true;
1670         errdefer if (module_owned) module.deinit();
1671         module_owned = false;
1672         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
1673             .generated_scan_schedule = candidate,
1674             .instrumentation = .{
1675                 .context = phase_emitter,
1676                 .observe = recordFragmentPhase,
1677             },
1678         });
1679         defer fragment.deinit();
1680 
1681         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
1682         defer bindings.deinit();
1683 
1684         var best_median: u64 = 0;
1685         var best_samples: u32 = 0;
1686         var kernel_index: usize = 0;
1687         while (kernel_index < fragment.kernelCount()) : (kernel_index += 1) {
1688             const summary = try fragment.kernelSummary(kernel_index);
1689             if (std.mem.indexOf(u8, summary.entry_name, "scan_lookback") == null) continue;
1690             if (summary.launch_geometry.threadgroup[0] != candidate.threads) return error.LaunchArgumentMismatch;
1691             const records = try bindings.measureLaunchCandidateRecords(
1692                 arena,
1693                 arena,
1694                 kernel_index,
1695                 .{
1696                     .warmup = options.warmup,
1697                     .samples = options.samples,
1698                     .base_options = .{ .stream = stream },
1699                     .synchronize = .stream,
1700                 },
1701             );
1702             defer arena.free(records);
1703             for (records) |record| {
1704                 try jsonl.writeGeneratedScanScheduleCandidateMeasurementRecord(out, options, workload, record);
1705                 if (best_median == 0 or record.median_ns < best_median) {
1706                     best_median = record.median_ns;
1707                     best_samples = record.sample_count;
1708                 }
1709             }
1710         }
1711         if (best_median != 0) {
1712             measurements[measurement_count] = .{
1713                 .schedule = candidate,
1714                 .median_ns = best_median,
1715                 .sample_count = best_samples,
1716             };
1717             measurement_count += 1;
1718         }
1719     }
1720     try scan_schedule_sink.recordCandidateSet(problem, measurements[0..measurement_count]);
1721     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_generated_scan_schedule_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
1722 }
1723 
1724 fn emitGeneratedScanScheduleTuningRecords(
1725     arena: Allocator,
1726     out: *std.Io.Writer,
1727     options: config.Options,
1728     workload: workload_mod.Workload,
1729     sink: *GeneratedScanScheduleTuningSink,
1730 ) !void {
1731     if (sink.measurement_count == 0 and options.scan_schedule_tuning_out == null) return;
1732     const records = try sink.cache.exportRecords(arena);
1733     defer arena.free(records);
1734     for (records) |record| {
1735         try jsonl.writeGeneratedScanScheduleTuningWinnerRecord(out, options, workload, record);
1736     }
1737     const artifact = try schedule_tuning.encodeGeneratedScanScheduleTuningArtifact(arena, records);
1738     defer arena.free(artifact);
1739     try jsonl.writeGeneratedScanScheduleTuningArtifactRecord(
1740         out,
1741         options,
1742         workload,
1743         sink.measurement_count,
1744         records.len,
1745         artifact.len,
1746         schedule_tuning.generatedScanScheduleTuningArtifactFingerprint(artifact),
1747     );
1748     if (options.scan_schedule_tuning_out) |artifact_path| {
1749         try sys.fs.writeFile(artifact_path, artifact);
1750     }
1751 }
1752 
1753 fn runCudaStencilWindowFamilyCandidateMeasurements(
1754     arena: Allocator,
1755     backing_allocator: Allocator,
1756     out: *std.Io.Writer,
1757     options: config.Options,
1758     workload: workload_mod.Workload,
1759     handle: gpu.BackendHandle,
1760     stream: gpu.StreamHandle,
1761     phase_emitter: *BackendPhaseEmitter,
1762     family_sink: *FamilyTuningSink,
1763 ) !void {
1764     const family_shape = workload_mod.stencilWindowFamilyShape(options) orelse return;
1765 
1766     var candidates = try accy.kernel.library.selectOwnedStencilWindowCandidates(backing_allocator, .{
1767         .dtype = .f32,
1768         .kind = .window,
1769         .rows = family_shape.rows,
1770         .cols = family_shape.cols,
1771         .radius = family_shape.radius,
1772     });
1773     defer candidates.deinit();
1774     if (candidates.count <= 1) return;
1775 
1776     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
1777         backing_allocator,
1778         handle,
1779         candidates.slice(),
1780         .{ .limits = .standard },
1781     );
1782     defer registry.deinit();
1783     const registry_value = registry.registry();
1784 
1785     const inputs = try workload_mod.buildStencilWindowKernelCallInputs(
1786         arena,
1787         family_shape,
1788         workload_mod.stencil_window_kernel_call_weights[0..],
1789     );
1790 
1791     const measurement_start = timing_mod.nowNanos();
1792     for (candidates.slice()) |candidate| {
1793         const instance = accy.kernel.library.stencil.windowInstanceFromSpecialization(
1794             candidate.descriptor.metadata.specialization,
1795         ) orelse return error.InvalidKernelLibraryEntry;
1796         const module = try workload_mod.buildStencilWindowKernelCallModule(
1797             backing_allocator,
1798             family_shape,
1799             candidate.descriptor.metadata.target,
1800             candidate.descriptor.metadata.version,
1801         );
1802         var module_owned = true;
1803         errdefer if (module_owned) module.deinit();
1804         module_owned = false;
1805         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
1806             .kernel_call_registry = &registry_value,
1807             .instrumentation = .{
1808                 .context = phase_emitter,
1809                 .observe = recordFragmentPhase,
1810             },
1811         });
1812         defer fragment.deinit();
1813 
1814         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
1815         defer bindings.deinit();
1816 
1817         const runtime_arguments = try accy.kernel.library.stencil.windowRuntimeArguments(instance);
1818         const records = try measureAndRecordFamilyCandidateRecords(
1819             arena,
1820             arena,
1821             fragment,
1822             bindings,
1823             candidate.descriptor.name,
1824             .{
1825                 .warmup = options.warmup,
1826                 .samples = options.samples,
1827                 .base_options = .{
1828                     .stream = stream,
1829                     .runtime_scalar_arguments = runtime_arguments[0..],
1830                 },
1831                 .synchronize = .stream,
1832             },
1833         );
1834         defer arena.free(records);
1835         try writeStencilWindowFamilyCandidateRecords(out, options, workload, records);
1836         const tuning_key = try accy.kernel.library.stencil.windowFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
1837         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
1838     }
1839     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_stencil_window_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
1840 }
1841 
1842 fn writeStencilWindowFamilyCandidateRecords(
1843     out: *std.Io.Writer,
1844     options: config.Options,
1845     workload: workload_mod.Workload,
1846     records: []const accy.executable.LaunchCandidateRecord,
1847 ) !void {
1848     var best_record: ?accy.executable.LaunchCandidateRecord = null;
1849     for (records) |record| {
1850         try jsonl.writeStencilWindowFamilyCandidateMeasurementRecord(out, options, workload, record);
1851         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
1852             if (best_record) |best| try jsonl.writeStencilWindowFamilyCandidateBestRecord(out, options, workload, best);
1853             best_record = record;
1854         } else if (launchCandidateRecordBeats(record, best_record.?)) {
1855             best_record = record;
1856         }
1857     }
1858     if (best_record) |record| try jsonl.writeStencilWindowFamilyCandidateBestRecord(out, options, workload, record);
1859 }
1860 
1861 fn runCudaGatherFamilyCandidateMeasurements(
1862     arena: Allocator,
1863     backing_allocator: Allocator,
1864     out: *std.Io.Writer,
1865     options: config.Options,
1866     workload: workload_mod.Workload,
1867     handle: gpu.BackendHandle,
1868     stream: gpu.StreamHandle,
1869     phase_emitter: *BackendPhaseEmitter,
1870     family_sink: *FamilyTuningSink,
1871 ) !void {
1872     const family_shape = workload_mod.gatherFamilyShape(options) orelse return;
1873 
1874     var candidates = try accy.kernel.library.selectOwnedGatherCandidates(backing_allocator, .{
1875         .dtype = .f32,
1876         .outer = family_shape.outer,
1877         .axis_size = family_shape.axis_size,
1878         .gathered = family_shape.gathered,
1879         .inner = family_shape.inner,
1880     });
1881     defer candidates.deinit();
1882     if (candidates.count <= 1) return;
1883 
1884     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
1885         backing_allocator,
1886         handle,
1887         candidates.slice(),
1888         .{ .limits = .standard },
1889     );
1890     defer registry.deinit();
1891     const registry_value = registry.registry();
1892 
1893     const inputs = try workload_mod.buildGatherKernelCallInputs(arena, family_shape);
1894 
1895     const measurement_start = timing_mod.nowNanos();
1896     for (candidates.slice()) |candidate| {
1897         const instance = accy.kernel.library.indexing.gatherInstanceFromSpecialization(
1898             candidate.descriptor.metadata.specialization,
1899         ) orelse return error.InvalidKernelLibraryEntry;
1900         const module = try workload_mod.buildGatherKernelCallModule(
1901             backing_allocator,
1902             family_shape,
1903             candidate.descriptor.metadata.target,
1904             candidate.descriptor.metadata.version,
1905         );
1906         var module_owned = true;
1907         errdefer if (module_owned) module.deinit();
1908         module_owned = false;
1909         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
1910             .kernel_call_registry = &registry_value,
1911             .instrumentation = .{
1912                 .context = phase_emitter,
1913                 .observe = recordFragmentPhase,
1914             },
1915         });
1916         defer fragment.deinit();
1917 
1918         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
1919         defer bindings.deinit();
1920 
1921         const runtime_arguments = try accy.kernel.library.indexing.gatherRuntimeArguments(instance);
1922         const records = try measureAndRecordFamilyCandidateRecords(
1923             arena,
1924             arena,
1925             fragment,
1926             bindings,
1927             candidate.descriptor.name,
1928             .{
1929                 .warmup = options.warmup,
1930                 .samples = options.samples,
1931                 .base_options = .{
1932                     .stream = stream,
1933                     .runtime_scalar_arguments = runtime_arguments[0..],
1934                 },
1935                 .synchronize = .stream,
1936             },
1937         );
1938         defer arena.free(records);
1939         try writeGatherFamilyCandidateRecords(out, options, workload, records);
1940         const tuning_key = try accy.kernel.library.indexing.gatherFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
1941         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
1942     }
1943     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_gather_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
1944 }
1945 
1946 fn writeGatherFamilyCandidateRecords(
1947     out: *std.Io.Writer,
1948     options: config.Options,
1949     workload: workload_mod.Workload,
1950     records: []const accy.executable.LaunchCandidateRecord,
1951 ) !void {
1952     var best_record: ?accy.executable.LaunchCandidateRecord = null;
1953     for (records) |record| {
1954         try jsonl.writeGatherFamilyCandidateMeasurementRecord(out, options, workload, record);
1955         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
1956             if (best_record) |best| try jsonl.writeGatherFamilyCandidateBestRecord(out, options, workload, best);
1957             best_record = record;
1958         } else if (launchCandidateRecordBeats(record, best_record.?)) {
1959             best_record = record;
1960         }
1961     }
1962     if (best_record) |record| try jsonl.writeGatherFamilyCandidateBestRecord(out, options, workload, record);
1963 }
1964 
1965 fn runCudaPrefixSumFamilyCandidateMeasurements(
1966     arena: Allocator,
1967     backing_allocator: Allocator,
1968     out: *std.Io.Writer,
1969     options: config.Options,
1970     workload: workload_mod.Workload,
1971     handle: gpu.BackendHandle,
1972     stream: gpu.StreamHandle,
1973     phase_emitter: *BackendPhaseEmitter,
1974     family_sink: *FamilyTuningSink,
1975 ) !void {
1976     const family_shape = workload_mod.prefixSumFamilyShape(options) orelse return;
1977     if (family_shape.extent > accy.kernel.library.scan.prefix_sum_max_threads) return;
1978 
1979     var candidates = try accy.kernel.library.selectOwnedPrefixSumCandidates(backing_allocator, .{
1980         .dtype = .f32,
1981         .kind = .prefix_sum,
1982         .extent = family_shape.extent,
1983     });
1984     defer candidates.deinit();
1985     if (candidates.count <= 1) return;
1986 
1987     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
1988         backing_allocator,
1989         handle,
1990         candidates.slice(),
1991         .{ .limits = .standard },
1992     );
1993     defer registry.deinit();
1994     const registry_value = registry.registry();
1995 
1996     const inputs = try workload_mod.buildPrefixSumKernelCallInputs(arena, family_shape);
1997 
1998     const measurement_start = timing_mod.nowNanos();
1999     for (candidates.slice()) |candidate| {
2000         const instance = accy.kernel.library.scan.prefixSumInstanceFromSpecialization(
2001             candidate.descriptor.metadata.specialization,
2002         ) orelse return error.InvalidKernelLibraryEntry;
2003         const module = try workload_mod.buildPrefixSumKernelCallModule(
2004             backing_allocator,
2005             family_shape,
2006             candidate.descriptor.metadata.target,
2007             candidate.descriptor.metadata.version,
2008         );
2009         var module_owned = true;
2010         errdefer if (module_owned) module.deinit();
2011         module_owned = false;
2012         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
2013             .kernel_call_registry = &registry_value,
2014             .instrumentation = .{
2015                 .context = phase_emitter,
2016                 .observe = recordFragmentPhase,
2017             },
2018         });
2019         defer fragment.deinit();
2020 
2021         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
2022         defer bindings.deinit();
2023 
2024         const runtime_arguments = try accy.kernel.library.scan.prefixSumRuntimeArguments(instance);
2025         const records = try measureAndRecordFamilyCandidateRecords(
2026             arena,
2027             arena,
2028             fragment,
2029             bindings,
2030             candidate.descriptor.name,
2031             .{
2032                 .warmup = options.warmup,
2033                 .samples = options.samples,
2034                 .base_options = .{
2035                     .stream = stream,
2036                     .runtime_scalar_arguments = runtime_arguments[0..],
2037                 },
2038                 .synchronize = .stream,
2039             },
2040         );
2041         defer arena.free(records);
2042         try writePrefixSumFamilyCandidateRecords(out, options, workload, records);
2043         const tuning_key = try accy.kernel.library.scan.prefixSumFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2044         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
2045     }
2046     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_prefix_sum_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2047 }
2048 
2049 fn writePrefixSumFamilyCandidateRecords(
2050     out: *std.Io.Writer,
2051     options: config.Options,
2052     workload: workload_mod.Workload,
2053     records: []const accy.executable.LaunchCandidateRecord,
2054 ) !void {
2055     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2056     for (records) |record| {
2057         try jsonl.writePrefixSumFamilyCandidateMeasurementRecord(out, options, workload, record);
2058         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2059             if (best_record) |best| try jsonl.writePrefixSumFamilyCandidateBestRecord(out, options, workload, best);
2060             best_record = record;
2061         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2062             best_record = record;
2063         }
2064     }
2065     if (best_record) |record| try jsonl.writePrefixSumFamilyCandidateBestRecord(out, options, workload, record);
2066 }
2067 
2068 fn runCudaScatterFamilyCandidateMeasurements(
2069     arena: Allocator,
2070     backing_allocator: Allocator,
2071     out: *std.Io.Writer,
2072     options: config.Options,
2073     workload: workload_mod.Workload,
2074     handle: gpu.BackendHandle,
2075     stream: gpu.StreamHandle,
2076     phase_emitter: *BackendPhaseEmitter,
2077     family_sink: *FamilyTuningSink,
2078 ) !void {
2079     const family_shape = workload_mod.scatterFamilyShape(options) orelse return;
2080 
2081     var candidates = try accy.kernel.library.selectOwnedScatterCandidates(backing_allocator, .{
2082         .dtype = .f32,
2083         .outer = family_shape.outer,
2084         .axis_size = family_shape.axis_size,
2085         .updates = family_shape.updates,
2086         .inner = family_shape.inner,
2087     });
2088     defer candidates.deinit();
2089     if (candidates.count <= 1) return;
2090 
2091     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
2092         backing_allocator,
2093         handle,
2094         candidates.slice(),
2095         .{ .limits = .standard },
2096     );
2097     defer registry.deinit();
2098     const registry_value = registry.registry();
2099 
2100     const inputs = try workload_mod.buildScatterKernelCallInputs(arena, family_shape);
2101 
2102     const measurement_start = timing_mod.nowNanos();
2103     for (candidates.slice()) |candidate| {
2104         const instance = accy.kernel.library.indexing.scatterInstanceFromSpecialization(
2105             candidate.descriptor.metadata.specialization,
2106         ) orelse return error.InvalidKernelLibraryEntry;
2107         const module = try workload_mod.buildScatterKernelCallModule(
2108             backing_allocator,
2109             family_shape,
2110             candidate.descriptor.metadata.target,
2111             candidate.descriptor.metadata.version,
2112         );
2113         var module_owned = true;
2114         errdefer if (module_owned) module.deinit();
2115         module_owned = false;
2116         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
2117             .kernel_call_registry = &registry_value,
2118             .instrumentation = .{
2119                 .context = phase_emitter,
2120                 .observe = recordFragmentPhase,
2121             },
2122         });
2123         defer fragment.deinit();
2124 
2125         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
2126         defer bindings.deinit();
2127 
2128         const runtime_arguments = try accy.kernel.library.indexing.scatterRuntimeArguments(instance);
2129         const records = try measureAndRecordFamilyCandidateRecords(
2130             arena,
2131             arena,
2132             fragment,
2133             bindings,
2134             candidate.descriptor.name,
2135             .{
2136                 .warmup = options.warmup,
2137                 .samples = options.samples,
2138                 .base_options = .{
2139                     .stream = stream,
2140                     .runtime_scalar_arguments = runtime_arguments[0..],
2141                 },
2142                 .synchronize = .stream,
2143             },
2144         );
2145         defer arena.free(records);
2146         try writeScatterFamilyCandidateRecords(out, options, workload, records);
2147         const tuning_key = try accy.kernel.library.indexing.scatterFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2148         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
2149     }
2150     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_scatter_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2151 }
2152 
2153 fn writeScatterFamilyCandidateRecords(
2154     out: *std.Io.Writer,
2155     options: config.Options,
2156     workload: workload_mod.Workload,
2157     records: []const accy.executable.LaunchCandidateRecord,
2158 ) !void {
2159     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2160     for (records) |record| {
2161         try jsonl.writeScatterFamilyCandidateMeasurementRecord(out, options, workload, record);
2162         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2163             if (best_record) |best| try jsonl.writeScatterFamilyCandidateBestRecord(out, options, workload, best);
2164             best_record = record;
2165         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2166             best_record = record;
2167         }
2168     }
2169     if (best_record) |record| try jsonl.writeScatterFamilyCandidateBestRecord(out, options, workload, record);
2170 }
2171 
2172 const scatter_add_measurement_dtypes = [_]choir_abi.DType{ .i32, .f32 };
2173 
2174 const ScatterAddMeasurementInputs = struct {
2175     dst_seed: []const u8,
2176     indices: []const u8,
2177     updates: []const u8,
2178 };
2179 
2180 fn runCudaScatterAddFamilyCandidateMeasurements(
2181     arena: Allocator,
2182     backing_allocator: Allocator,
2183     out: *std.Io.Writer,
2184     options: config.Options,
2185     workload: workload_mod.Workload,
2186     handle: gpu.BackendHandle,
2187     stream: ?gpu.StreamHandle,
2188     family_sink: *FamilyTuningSink,
2189 ) !void {
2190     const family_shape = workload_mod.scatterAddFamilyShape(options) orelse return;
2191     const measurement_start = timing_mod.nowNanos();
2192 
2193     for (scatter_add_measurement_dtypes) |dtype| {
2194         var candidates = try accy.kernel.library.selectOwnedScatterAddCandidates(backing_allocator, .{
2195             .dtype = dtype,
2196             .outer = family_shape.outer,
2197             .axis_size = family_shape.axis_size,
2198             .updates = family_shape.updates,
2199             .inner = family_shape.inner,
2200         });
2201         defer candidates.deinit();
2202         if (candidates.count <= 1) continue;
2203 
2204         const inputs = try buildScatterAddMeasurementInputs(arena, family_shape, dtype);
2205         for (candidates.slice()) |candidate| {
2206             const instance = accy.kernel.library.indexing.scatterAddInstanceFromSpecialization(
2207                 candidate.descriptor.metadata.specialization,
2208             ) orelse return error.InvalidKernelLibraryEntry;
2209             const record = try measureDirectScatterAddFamilyCandidateRecord(
2210                 arena,
2211                 backing_allocator,
2212                 handle,
2213                 stream,
2214                 options,
2215                 candidate,
2216                 instance,
2217                 inputs,
2218             );
2219             const records = [_]accy.executable.LaunchCandidateRecord{record};
2220             try writeScatterAddFamilyCandidateRecords(out, options, workload, records[0..]);
2221             const tuning_key = try accy.kernel.library.indexing.scatterAddFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2222             try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records[0..]);
2223         }
2224     }
2225 
2226     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_scatter_add_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2227 }
2228 
2229 fn writeScatterAddFamilyCandidateRecords(
2230     out: *std.Io.Writer,
2231     options: config.Options,
2232     workload: workload_mod.Workload,
2233     records: []const accy.executable.LaunchCandidateRecord,
2234 ) !void {
2235     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2236     for (records) |record| {
2237         try jsonl.writeScatterAddFamilyCandidateMeasurementRecord(out, options, workload, record);
2238         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2239             if (best_record) |best| try jsonl.writeScatterAddFamilyCandidateBestRecord(out, options, workload, best);
2240             best_record = record;
2241         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2242             best_record = record;
2243         }
2244     }
2245     if (best_record) |record| try jsonl.writeScatterAddFamilyCandidateBestRecord(out, options, workload, record);
2246 }
2247 
2248 fn measureDirectScatterAddFamilyCandidateRecord(
2249     scratch: Allocator,
2250     lifetime_allocator: Allocator,
2251     handle: gpu.BackendHandle,
2252     stream: ?gpu.StreamHandle,
2253     options: config.Options,
2254     candidate: accy.kernel.library.catalog.OwnedDescriptor,
2255     instance: accy.kernel.library.indexing.ScatterAdd,
2256     inputs: ScatterAddMeasurementInputs,
2257 ) !accy.executable.LaunchCandidateRecord {
2258     if (options.samples == 0) return error.LaunchArgumentMismatch;
2259 
2260     var owned_artifact = try accy.kernel.library.createOwnedKernelCallArtifact(
2261         lifetime_allocator,
2262         handle,
2263         candidate,
2264         .{ .limits = .standard },
2265     );
2266     defer owned_artifact.deinit();
2267     const entry = owned_artifact.entry();
2268 
2269     var artifact = try kernelArtifactFromKernelCallEntry(lifetime_allocator, handle, entry, "accy-choir-bench-scatter-add");
2270     defer artifact.deinit();
2271     const loaded = try handle.loadArtifact(&artifact);
2272     defer handle.destroyObject(loaded.id);
2273 
2274     const runtime_arguments = try accy.kernel.library.indexing.scatterAddRuntimeArguments(instance);
2275     const geometry = try kernelCallLaunchGeometry(entry.launch, runtime_arguments[0..]);
2276 
2277     var buffers: [4]gpu.BufferHandle = undefined;
2278     var buffer_count: usize = 0;
2279     defer destroyBufferHandles(handle, buffers[0..buffer_count]);
2280 
2281     var bindings: [4]gpu.BufferBinding = undefined;
2282     const host_views = [_][]const u8{ inputs.dst_seed, inputs.dst_seed, inputs.indices, inputs.updates };
2283     const dtypes = [_]choir_abi.DType{ instance.dtype, instance.dtype, .i32, instance.dtype };
2284     const accesses = [_]gpu.BufferAccess{ .read_write, .read_only, .read_only, .read_only };
2285     for (host_views, dtypes, accesses, 0..) |bytes, dtype, access, index| {
2286         buffers[index] = try handle.allocateBuffer(.{
2287             .byte_size = bytes.len,
2288             .alignment = 256,
2289             .dtype = dtype,
2290             .element_count = bytes.len / @as(usize, dtype.sizeOf()),
2291         });
2292         buffer_count += 1;
2293         bindings[index] = .{
2294             .handle = buffers[index],
2295             .access = access,
2296             .ownership = buffers[index].ownership,
2297             .byte_size = buffers[index].byte_size,
2298         };
2299     }
2300 
2301     try handle.writeBuffer(.{ .handle = bindings[1].handle, .bytes = inputs.dst_seed });
2302     try handle.writeBuffer(.{ .handle = bindings[2].handle, .bytes = inputs.indices });
2303     try handle.writeBuffer(.{ .handle = bindings[3].handle, .bytes = inputs.updates });
2304 
2305     var warmup_index: u32 = 0;
2306     while (warmup_index < options.warmup) : (warmup_index += 1) {
2307         try launchScatterAddMeasurement(handle, stream, &artifact, loaded, bindings[0..], runtime_arguments[0..], geometry, inputs.dst_seed);
2308     }
2309 
2310     const sample_times = try scratch.alloc(u64, options.samples);
2311     defer scratch.free(sample_times);
2312     for (sample_times) |*sample| {
2313         try handle.writeBuffer(.{ .handle = bindings[0].handle, .bytes = inputs.dst_seed });
2314         const start = timing_mod.nowNanos();
2315         try handle.launch(.{
2316             .artifact = &artifact,
2317             .loaded_artifact = loaded,
2318             .buffers = bindings[0..],
2319             .scalar_arguments = runtime_arguments[0..],
2320             .geometry = geometry,
2321             .stream = stream,
2322         });
2323         try synchronizeScatterAddMeasurement(handle, stream);
2324         sample.* = @intCast(timing_mod.nowNanos() - start);
2325     }
2326     std.mem.sort(u64, sample_times, {}, std.sort.asc(u64));
2327 
2328     return try directScatterAddLaunchCandidateRecord(
2329         scratch,
2330         entry,
2331         artifact,
2332         instance,
2333         geometry,
2334         sample_times[sample_times.len / 2],
2335         options.samples,
2336     );
2337 }
2338 
2339 fn launchScatterAddMeasurement(
2340     handle: gpu.BackendHandle,
2341     stream: ?gpu.StreamHandle,
2342     artifact: *const gpu.KernelArtifact,
2343     loaded: gpu.LoadedArtifact,
2344     bindings: []const gpu.BufferBinding,
2345     runtime_arguments: []const choir_abi.ScalarArgument,
2346     geometry: choir_abi.LaunchGeometry,
2347     dst_seed: []const u8,
2348 ) !void {
2349     try handle.writeBuffer(.{ .handle = bindings[0].handle, .bytes = dst_seed });
2350     try handle.launch(.{
2351         .artifact = artifact,
2352         .loaded_artifact = loaded,
2353         .buffers = bindings,
2354         .scalar_arguments = runtime_arguments,
2355         .geometry = geometry,
2356         .stream = stream,
2357     });
2358     try synchronizeScatterAddMeasurement(handle, stream);
2359 }
2360 
2361 fn synchronizeScatterAddMeasurement(
2362     handle: gpu.BackendHandle,
2363     stream: ?gpu.StreamHandle,
2364 ) !void {
2365     if (stream) |stream_handle| {
2366         try handle.synchronize(.{ .scope = .stream, .stream = stream_handle });
2367     } else {
2368         try handle.synchronize(.{ .scope = .device });
2369     }
2370 }
2371 
2372 fn buildScatterAddMeasurementInputs(
2373     allocator: Allocator,
2374     family_shape: workload_mod.ScatterAddFamilyShape,
2375     dtype: choir_abi.DType,
2376 ) !ScatterAddMeasurementInputs {
2377     const dst_count = try std.math.mul(u64, try std.math.mul(u64, family_shape.outer, family_shape.axis_size), family_shape.inner);
2378     const update_count = try std.math.mul(u64, try std.math.mul(u64, family_shape.outer, family_shape.updates), family_shape.inner);
2379     const dst_seed = try allocator.alloc(u8, try typedByteCount(dst_count, dtype));
2380     const indices = try allocator.alloc(u8, try typedByteCount(family_shape.updates, .i32));
2381     const updates = try allocator.alloc(u8, try typedByteCount(update_count, dtype));
2382     try fillScatterAddValues(dst_seed, dtype, .dst);
2383     try fillScatterAddIndices(indices, family_shape.axis_size);
2384     try fillScatterAddValues(updates, dtype, .updates);
2385     return .{ .dst_seed = dst_seed, .indices = indices, .updates = updates };
2386 }
2387 
2388 const ScatterAddValueRole = enum {
2389     dst,
2390     updates,
2391 };
2392 
2393 fn fillScatterAddValues(bytes: []u8, dtype: choir_abi.DType, role: ScatterAddValueRole) !void {
2394     var offset: usize = 0;
2395     var index: usize = 0;
2396     while (offset < bytes.len) : ({
2397         offset += dtype.sizeOf();
2398         index += 1;
2399     }) {
2400         switch (dtype) {
2401             .i32 => {
2402                 const base: i32 = if (role == .dst) 7 else 1;
2403                 const value = base + @as(i32, @intCast(index % 17));
2404                 std.mem.writeInt(i32, bytes[offset..][0..4], value, .little);
2405             },
2406             .f32 => {
2407                 const base: f32 = if (role == .dst) 0.25 else 0.5;
2408                 const step: f32 = if (role == .dst) 0.03125 else 0.01;
2409                 const value = base + step * @as(f32, @floatFromInt(index % 101));
2410                 std.mem.writeInt(u32, bytes[offset..][0..4], @bitCast(value), .little);
2411             },
2412             else => return error.UnsupportedDType,
2413         }
2414     }
2415 }
2416 
2417 fn fillScatterAddIndices(bytes: []u8, axis_size: u64) !void {
2418     const bounded_axis = std.math.cast(i32, axis_size) orelse return error.InvalidDimension;
2419     if (bounded_axis <= 0) return error.InvalidDimension;
2420     var offset: usize = 0;
2421     var index: usize = 0;
2422     while (offset < bytes.len) : ({
2423         offset += @sizeOf(i32);
2424         index += 1;
2425     }) {
2426         const value: i32 = @intCast((@as(u64, index) * 7 + 3) % axis_size);
2427         std.mem.writeInt(i32, bytes[offset..][0..4], value, .little);
2428     }
2429 }
2430 
2431 fn typedByteCount(element_count: u64, dtype: choir_abi.DType) !usize {
2432     const count = std.math.cast(usize, element_count) orelse return error.InvalidDimension;
2433     return std.math.mul(usize, count, dtype.sizeOf()) catch return error.InvalidDimension;
2434 }
2435 
2436 fn kernelArtifactFromKernelCallEntry(
2437     allocator: Allocator,
2438     handle: gpu.BackendHandle,
2439     entry: accy.artifact.KernelCallArtifact,
2440     diagnostic_id: []const u8,
2441 ) !gpu.KernelArtifact {
2442     const caps = try handle.queryCapabilities();
2443     var artifact = try gpu.KernelArtifact.init(allocator, .{
2444         .backend = handle.backendKind() orelse caps.identity.backend,
2445         .format = entry.format,
2446         .entry_name = entry.entry_name,
2447         .argument_count = entry.argument_count,
2448         .diagnostic_id = diagnostic_id,
2449         .interface = .{
2450             .features = entry.required_features,
2451             .subgroup = entry.required_subgroup,
2452             .push_constants = entry.push_constants,
2453         },
2454     });
2455     errdefer artifact.deinit();
2456     switch (entry.payload) {
2457         .text => |text| artifact.setBorrowedText(text),
2458         .bytes => |bytes| artifact.setBorrowedBytes(bytes),
2459         .words_u32 => |words| artifact.setBorrowedWords(words),
2460         .none => return error.InvalidArtifact,
2461     }
2462     return artifact;
2463 }
2464 
2465 fn kernelCallLaunchGeometry(
2466     launch: accy.artifact.KernelCallLaunch,
2467     runtime_arguments: []const choir_abi.ScalarArgument,
2468 ) !choir_abi.LaunchGeometry {
2469     return switch (launch) {
2470         .fixed => |geometry| geometry,
2471         .derived => |derived| try derived.geometry(runtime_arguments),
2472     };
2473 }
2474 
2475 fn directScatterAddLaunchCandidateRecord(
2476     allocator: Allocator,
2477     entry: accy.artifact.KernelCallArtifact,
2478     artifact: gpu.KernelArtifact,
2479     instance: accy.kernel.library.indexing.ScatterAdd,
2480     geometry: choir_abi.LaunchGeometry,
2481     median_ns: u64,
2482     sample_count: u32,
2483 ) !accy.executable.LaunchCandidateRecord {
2484     const entry_name = try allocator.dupe(u8, artifact.entry_name);
2485     return .{
2486         .kernel = .{
2487             .source = .kernel_call,
2488             .kernel_id = 0,
2489             .work_item_id = 0,
2490             .output_layout_fingerprint = 0,
2491             .input_layout_fingerprint = entry.shape_family_fingerprint orelse 0,
2492             .element_count = instance.total(),
2493             .op_count = std.math.cast(usize, instance.total()) orelse return error.InvalidArtifact,
2494             .entry_name = entry_name,
2495             .artifact_format = artifact.format,
2496             .artifact_payload_bytes = try kernelArtifactPayloadBytes(artifact),
2497             .compile_argument_count = artifact.argument_count,
2498             .compile_required_dtype_bits = entry.required_dtypes.bits,
2499             .compile_required_features = entry.required_features,
2500             .compile_required_subgroup = entry.required_subgroup,
2501             .compile_payload = compilePayloadKind(entry.payload),
2502             .compile_payload_bytes = try compilePayloadBytes(entry.payload),
2503             .compile_launch = .kernel_call,
2504             .runtime_scalar_argument_count = entry.runtime_scalar_argument_count,
2505             .launch_geometry = geometry,
2506             .launch_candidate_count = 1,
2507             .launch_resource_class = "unknown",
2508             .fixed_threadgroup = true,
2509             .subgroup_aligned = geometry.threadgroup[0] % 32 == 0,
2510             .subgroup_size = if (geometry.threadgroup[0] % 32 == 0) 32 else null,
2511             .static_bytes_complete = true,
2512         },
2513         .candidate_index = 0,
2514         .geometry = geometry,
2515         .candidate_score = 0,
2516         .estimated_static_bytes_per_threadgroup = try scatterAddStaticBytes(instance),
2517         .estimated_element_ops_per_threadgroup = @as(u64, geometry.threadgroup[0]),
2518         .median_ns = median_ns,
2519         .sample_count = sample_count,
2520     };
2521 }
2522 
2523 fn scatterAddStaticBytes(instance: accy.kernel.library.indexing.ScatterAdd) !u64 {
2524     if (instance.variant != .shared_bins) return 0;
2525     return std.math.mul(u64, instance.axis_size, @as(u64, instance.dtype.sizeOf())) catch return error.InvalidArtifact;
2526 }
2527 
2528 fn kernelArtifactPayloadBytes(artifact: gpu.KernelArtifact) !usize {
2529     return switch (artifact.payload) {
2530         .text => |text| text.len,
2531         .bytes => |bytes| bytes.len,
2532         .words_u32 => |words| words.len * @sizeOf(u32),
2533         .none, .external => error.InvalidArtifact,
2534     };
2535 }
2536 
2537 fn compilePayloadBytes(payload: gpu.CompilePayload) !usize {
2538     return switch (payload) {
2539         .text => |text| text.len,
2540         .bytes => |bytes| bytes.len,
2541         .words_u32 => |words| words.len * @sizeOf(u32),
2542         .none => error.InvalidArtifact,
2543     };
2544 }
2545 
2546 fn compilePayloadKind(payload: gpu.CompilePayload) accy.artifact.PlannedKernelCompilePayload {
2547     return switch (payload) {
2548         .none => .none,
2549         .bytes => .bytes,
2550         .words_u32 => .words_u32,
2551         .text => .text,
2552     };
2553 }
2554 
2555 fn runCudaSpmvCsrFamilyCandidateMeasurements(
2556     arena: Allocator,
2557     backing_allocator: Allocator,
2558     out: *std.Io.Writer,
2559     options: config.Options,
2560     workload: workload_mod.Workload,
2561     handle: gpu.BackendHandle,
2562     stream: gpu.StreamHandle,
2563     phase_emitter: *BackendPhaseEmitter,
2564     family_sink: *FamilyTuningSink,
2565 ) !void {
2566     const family_shape = workload_mod.spmvCsrFamilyShape(options) orelse return;
2567 
2568     var candidates = try accy.kernel.library.selectOwnedSparseCandidates(backing_allocator, .{
2569         .dtype = .f32,
2570         .kind = .{ .csr_spmv = .{
2571             .rows = family_shape.rows,
2572             .nnz = family_shape.nnz,
2573             .x_extent = family_shape.x_extent,
2574         } },
2575     });
2576     defer candidates.deinit();
2577     if (candidates.count <= 1) return;
2578 
2579     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
2580         backing_allocator,
2581         handle,
2582         candidates.slice(),
2583         .{ .limits = .standard },
2584     );
2585     defer registry.deinit();
2586     const registry_value = registry.registry();
2587 
2588     const inputs = try workload_mod.buildSpmvCsrKernelCallInputs(arena, family_shape);
2589 
2590     const measurement_start = timing_mod.nowNanos();
2591     for (candidates.slice()) |candidate| {
2592         const instance = accy.kernel.library.sparse.spmvCsrInstanceFromSpecialization(
2593             candidate.descriptor.metadata.specialization,
2594         ) orelse return error.InvalidKernelLibraryEntry;
2595         const module = try workload_mod.buildSpmvCsrKernelCallModule(
2596             backing_allocator,
2597             family_shape,
2598             candidate.descriptor.metadata.target,
2599             candidate.descriptor.metadata.version,
2600         );
2601         var module_owned = true;
2602         errdefer if (module_owned) module.deinit();
2603         module_owned = false;
2604         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
2605             .kernel_call_registry = &registry_value,
2606             .instrumentation = .{
2607                 .context = phase_emitter,
2608                 .observe = recordFragmentPhase,
2609             },
2610         });
2611         defer fragment.deinit();
2612 
2613         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
2614         defer bindings.deinit();
2615 
2616         const runtime_arguments = try accy.kernel.library.sparse.spmvCsrRuntimeArguments(
2617             instance,
2618             family_shape.nnz,
2619             family_shape.x_extent,
2620         );
2621         const records = try measureAndRecordFamilyCandidateRecords(
2622             arena,
2623             arena,
2624             fragment,
2625             bindings,
2626             candidate.descriptor.name,
2627             .{
2628                 .warmup = options.warmup,
2629                 .samples = options.samples,
2630                 .base_options = .{
2631                     .stream = stream,
2632                     .runtime_scalar_arguments = runtime_arguments[0..],
2633                 },
2634                 .synchronize = .stream,
2635             },
2636         );
2637         defer arena.free(records);
2638         try writeSpmvCsrFamilyCandidateRecords(out, options, workload, records);
2639         const tuning_key = try accy.kernel.library.sparse.spmvCsrFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2640         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
2641     }
2642     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_spmv_csr_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2643 }
2644 
2645 fn writeSpmvCsrFamilyCandidateRecords(
2646     out: *std.Io.Writer,
2647     options: config.Options,
2648     workload: workload_mod.Workload,
2649     records: []const accy.executable.LaunchCandidateRecord,
2650 ) !void {
2651     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2652     for (records) |record| {
2653         try jsonl.writeSpmvCsrFamilyCandidateMeasurementRecord(out, options, workload, record);
2654         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2655             if (best_record) |best| try jsonl.writeSpmvCsrFamilyCandidateBestRecord(out, options, workload, best);
2656             best_record = record;
2657         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2658             best_record = record;
2659         }
2660     }
2661     if (best_record) |record| try jsonl.writeSpmvCsrFamilyCandidateBestRecord(out, options, workload, record);
2662 }
2663 
2664 fn runCudaSpmmCsrFamilyCandidateMeasurements(
2665     arena: Allocator,
2666     backing_allocator: Allocator,
2667     out: *std.Io.Writer,
2668     options: config.Options,
2669     workload: workload_mod.Workload,
2670     handle: gpu.BackendHandle,
2671     stream: ?gpu.StreamHandle,
2672     phase_emitter: *BackendPhaseEmitter,
2673     family_sink: *FamilyTuningSink,
2674 ) !void {
2675     const family_shape = workload_mod.spmmCsrFamilyShape(options) orelse return;
2676 
2677     var candidates = try accy.kernel.library.selectOwnedSparseCandidates(backing_allocator, .{
2678         .dtype = .f32,
2679         .kind = .{ .csr_spmm = .{
2680             .rows = family_shape.rows,
2681             .columns = family_shape.columns,
2682             .nnz = family_shape.nnz,
2683             .x_extent = family_shape.x_extent,
2684         } },
2685     });
2686     defer candidates.deinit();
2687     if (candidates.count <= 1) return;
2688 
2689     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
2690         backing_allocator,
2691         handle,
2692         candidates.slice(),
2693         .{ .limits = .standard },
2694     );
2695     defer registry.deinit();
2696     const registry_value = registry.registry();
2697 
2698     const inputs = try workload_mod.buildSpmmCsrKernelCallInputs(arena, family_shape);
2699 
2700     const measurement_start = timing_mod.nowNanos();
2701     for (candidates.slice()) |candidate| {
2702         const instance = accy.kernel.library.sparse.spmmCsrInstanceFromSpecialization(
2703             candidate.descriptor.metadata.specialization,
2704         ) orelse return error.InvalidKernelLibraryEntry;
2705         const module = try workload_mod.buildSpmmCsrKernelCallModule(
2706             backing_allocator,
2707             family_shape,
2708             candidate.descriptor.metadata.target,
2709             candidate.descriptor.metadata.version,
2710         );
2711         var module_owned = true;
2712         errdefer if (module_owned) module.deinit();
2713         module_owned = false;
2714         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
2715             .kernel_call_registry = &registry_value,
2716             .instrumentation = .{
2717                 .context = phase_emitter,
2718                 .observe = recordFragmentPhase,
2719             },
2720         });
2721         defer fragment.deinit();
2722 
2723         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
2724         defer bindings.deinit();
2725 
2726         const runtime_arguments = try accy.kernel.library.sparse.spmmCsrRuntimeArguments(
2727             instance,
2728             family_shape.nnz,
2729             family_shape.x_extent,
2730             family_shape.columns,
2731         );
2732         const synchronize: schedule_tuning.LaunchCandidateSynchronization = if (stream != null) .stream else .device;
2733         const records = try measureAndRecordFamilyCandidateRecords(
2734             arena,
2735             arena,
2736             fragment,
2737             bindings,
2738             candidate.descriptor.name,
2739             .{
2740                 .warmup = options.warmup,
2741                 .samples = options.samples,
2742                 .base_options = .{
2743                     .stream = stream,
2744                     .runtime_scalar_arguments = runtime_arguments[0..],
2745                 },
2746                 .synchronize = synchronize,
2747             },
2748         );
2749         defer arena.free(records);
2750         try writeSpmmCsrFamilyCandidateRecords(out, options, workload, records);
2751         const tuning_key = try accy.kernel.library.sparse.spmmCsrFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2752         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
2753     }
2754     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_spmm_csr_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2755 }
2756 
2757 fn writeSpmmCsrFamilyCandidateRecords(
2758     out: *std.Io.Writer,
2759     options: config.Options,
2760     workload: workload_mod.Workload,
2761     records: []const accy.executable.LaunchCandidateRecord,
2762 ) !void {
2763     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2764     for (records) |record| {
2765         try jsonl.writeSpmmCsrFamilyCandidateMeasurementRecord(out, options, workload, record);
2766         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2767             if (best_record) |best| try jsonl.writeSpmmCsrFamilyCandidateBestRecord(out, options, workload, best);
2768             best_record = record;
2769         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2770             best_record = record;
2771         }
2772     }
2773     if (best_record) |record| try jsonl.writeSpmmCsrFamilyCandidateBestRecord(out, options, workload, record);
2774 }
2775 
2776 fn runCudaSpmvCooFamilyCandidateMeasurements(
2777     arena: Allocator,
2778     backing_allocator: Allocator,
2779     out: *std.Io.Writer,
2780     options: config.Options,
2781     workload: workload_mod.Workload,
2782     handle: gpu.BackendHandle,
2783     stream: ?gpu.StreamHandle,
2784     phase_emitter: *BackendPhaseEmitter,
2785     family_sink: *FamilyTuningSink,
2786 ) !void {
2787     const family_shape = workload_mod.spmvCooFamilyShape(options) orelse return;
2788     _ = phase_emitter;
2789 
2790     var candidates = try accy.kernel.library.selectOwnedSparseCandidates(backing_allocator, .{
2791         .dtype = .f32,
2792         .kind = .{ .coo_spmv = .{
2793             .rows = family_shape.rows,
2794             .nnz = family_shape.nnz,
2795             .x_extent = family_shape.x_extent,
2796         } },
2797     });
2798     defer candidates.deinit();
2799     if (candidates.count <= 1) return;
2800 
2801     const inputs = try workload_mod.buildSpmvCooKernelCallInputs(arena, family_shape);
2802 
2803     const measurement_start = timing_mod.nowNanos();
2804     for (candidates.slice()) |candidate| {
2805         const instance = accy.kernel.library.sparse.spmvCooInstanceFromSpecialization(
2806             candidate.descriptor.metadata.specialization,
2807         ) orelse return error.InvalidKernelLibraryEntry;
2808         const record = try measureDirectSpmvCooFamilyCandidateRecord(
2809             arena,
2810             backing_allocator,
2811             handle,
2812             stream,
2813             options,
2814             candidate,
2815             instance,
2816             inputs,
2817         );
2818         const records = [_]accy.executable.LaunchCandidateRecord{record};
2819         try writeSpmvCooFamilyCandidateRecords(out, options, workload, records[0..]);
2820         const tuning_key = try accy.kernel.library.sparse.spmvCooFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2821         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records[0..]);
2822     }
2823     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_spmv_coo_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2824 }
2825 
2826 fn writeSpmvCooFamilyCandidateRecords(
2827     out: *std.Io.Writer,
2828     options: config.Options,
2829     workload: workload_mod.Workload,
2830     records: []const accy.executable.LaunchCandidateRecord,
2831 ) !void {
2832     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2833     for (records) |record| {
2834         try jsonl.writeSpmvCooFamilyCandidateMeasurementRecord(out, options, workload, record);
2835         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2836             if (best_record) |best| try jsonl.writeSpmvCooFamilyCandidateBestRecord(out, options, workload, best);
2837             best_record = record;
2838         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2839             best_record = record;
2840         }
2841     }
2842     if (best_record) |record| try jsonl.writeSpmvCooFamilyCandidateBestRecord(out, options, workload, record);
2843 }
2844 
2845 fn runCudaSpmvEllFamilyCandidateMeasurements(
2846     arena: Allocator,
2847     backing_allocator: Allocator,
2848     out: *std.Io.Writer,
2849     options: config.Options,
2850     workload: workload_mod.Workload,
2851     handle: gpu.BackendHandle,
2852     stream: ?gpu.StreamHandle,
2853     phase_emitter: *BackendPhaseEmitter,
2854     family_sink: *FamilyTuningSink,
2855 ) !void {
2856     const family_shape = workload_mod.spmvEllFamilyShape(options) orelse return;
2857 
2858     var candidates = try accy.kernel.library.selectOwnedSparseCandidates(backing_allocator, .{
2859         .dtype = .f32,
2860         .kind = .{ .ell_spmv = .{
2861             .rows = family_shape.rows,
2862             .slots = family_shape.slots,
2863             .x_extent = family_shape.x_extent,
2864         } },
2865     });
2866     defer candidates.deinit();
2867     if (candidates.count <= 1) return;
2868 
2869     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
2870         backing_allocator,
2871         handle,
2872         candidates.slice(),
2873         .{ .limits = .standard },
2874     );
2875     defer registry.deinit();
2876     const registry_value = registry.registry();
2877 
2878     const inputs = try workload_mod.buildSpmvEllKernelCallInputs(arena, family_shape);
2879 
2880     const measurement_start = timing_mod.nowNanos();
2881     for (candidates.slice()) |candidate| {
2882         const instance = accy.kernel.library.sparse.spmvEllInstanceFromSpecialization(
2883             candidate.descriptor.metadata.specialization,
2884         ) orelse return error.InvalidKernelLibraryEntry;
2885         const module = try workload_mod.buildSpmvEllKernelCallModule(
2886             backing_allocator,
2887             family_shape,
2888             candidate.descriptor.metadata.target,
2889             candidate.descriptor.metadata.version,
2890         );
2891         var module_owned = true;
2892         errdefer if (module_owned) module.deinit();
2893         module_owned = false;
2894         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
2895             .kernel_call_registry = &registry_value,
2896             .instrumentation = .{
2897                 .context = phase_emitter,
2898                 .observe = recordFragmentPhase,
2899             },
2900         });
2901         defer fragment.deinit();
2902 
2903         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
2904         defer bindings.deinit();
2905 
2906         const runtime_arguments = try accy.kernel.library.sparse.spmvEllRuntimeArguments(
2907             instance,
2908             family_shape.slots,
2909             family_shape.x_extent,
2910         );
2911         const synchronize: schedule_tuning.LaunchCandidateSynchronization = if (stream != null) .stream else .device;
2912         const records = try measureAndRecordFamilyCandidateRecords(
2913             arena,
2914             arena,
2915             fragment,
2916             bindings,
2917             candidate.descriptor.name,
2918             .{
2919                 .warmup = options.warmup,
2920                 .samples = options.samples,
2921                 .base_options = .{
2922                     .stream = stream,
2923                     .runtime_scalar_arguments = runtime_arguments[0..],
2924                 },
2925                 .synchronize = synchronize,
2926             },
2927         );
2928         defer arena.free(records);
2929         try writeSpmvEllFamilyCandidateRecords(out, options, workload, records);
2930         const tuning_key = try accy.kernel.library.sparse.spmvEllFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
2931         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
2932     }
2933     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_spmv_ell_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
2934 }
2935 
2936 fn writeSpmvEllFamilyCandidateRecords(
2937     out: *std.Io.Writer,
2938     options: config.Options,
2939     workload: workload_mod.Workload,
2940     records: []const accy.executable.LaunchCandidateRecord,
2941 ) !void {
2942     var best_record: ?accy.executable.LaunchCandidateRecord = null;
2943     for (records) |record| {
2944         try jsonl.writeSpmvEllFamilyCandidateMeasurementRecord(out, options, workload, record);
2945         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
2946             if (best_record) |best| try jsonl.writeSpmvEllFamilyCandidateBestRecord(out, options, workload, best);
2947             best_record = record;
2948         } else if (launchCandidateRecordBeats(record, best_record.?)) {
2949             best_record = record;
2950         }
2951     }
2952     if (best_record) |record| try jsonl.writeSpmvEllFamilyCandidateBestRecord(out, options, workload, record);
2953 }
2954 
2955 fn runCudaSpmvSellFamilyCandidateMeasurements(
2956     arena: Allocator,
2957     backing_allocator: Allocator,
2958     out: *std.Io.Writer,
2959     options: config.Options,
2960     workload: workload_mod.Workload,
2961     handle: gpu.BackendHandle,
2962     stream: ?gpu.StreamHandle,
2963     phase_emitter: *BackendPhaseEmitter,
2964     family_sink: *FamilyTuningSink,
2965 ) !void {
2966     const family_shape = workload_mod.spmvSellFamilyShape(options) orelse return;
2967 
2968     var candidates = try accy.kernel.library.selectOwnedSparseCandidates(backing_allocator, .{
2969         .dtype = .f32,
2970         .kind = .{ .sell_spmv = .{
2971             .rows = family_shape.rows,
2972             .slice_size = family_shape.slice_size,
2973             .values_size = family_shape.values_size,
2974             .x_extent = family_shape.x_extent,
2975         } },
2976     });
2977     defer candidates.deinit();
2978     if (candidates.count <= 1) return;
2979 
2980     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
2981         backing_allocator,
2982         handle,
2983         candidates.slice(),
2984         .{ .limits = .standard },
2985     );
2986     defer registry.deinit();
2987     const registry_value = registry.registry();
2988 
2989     const inputs = try workload_mod.buildSpmvSellKernelCallInputs(arena, family_shape);
2990 
2991     const measurement_start = timing_mod.nowNanos();
2992     for (candidates.slice()) |candidate| {
2993         const instance = accy.kernel.library.sparse.spmvSellInstanceFromSpecialization(
2994             candidate.descriptor.metadata.specialization,
2995         ) orelse return error.InvalidKernelLibraryEntry;
2996         const module = try workload_mod.buildSpmvSellKernelCallModule(
2997             backing_allocator,
2998             family_shape,
2999             candidate.descriptor.metadata.target,
3000             candidate.descriptor.metadata.version,
3001         );
3002         var module_owned = true;
3003         errdefer if (module_owned) module.deinit();
3004         module_owned = false;
3005         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
3006             .kernel_call_registry = &registry_value,
3007             .instrumentation = .{
3008                 .context = phase_emitter,
3009                 .observe = recordFragmentPhase,
3010             },
3011         });
3012         defer fragment.deinit();
3013 
3014         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
3015         defer bindings.deinit();
3016 
3017         const runtime_arguments = try accy.kernel.library.sparse.spmvSellRuntimeArguments(
3018             instance,
3019             family_shape.values_size,
3020             family_shape.x_extent,
3021         );
3022         const synchronize: schedule_tuning.LaunchCandidateSynchronization = if (stream != null) .stream else .device;
3023         const records = try measureAndRecordFamilyCandidateRecords(
3024             arena,
3025             arena,
3026             fragment,
3027             bindings,
3028             candidate.descriptor.name,
3029             .{
3030                 .warmup = options.warmup,
3031                 .samples = options.samples,
3032                 .base_options = .{
3033                     .stream = stream,
3034                     .runtime_scalar_arguments = runtime_arguments[0..],
3035                 },
3036                 .synchronize = synchronize,
3037             },
3038         );
3039         defer arena.free(records);
3040         try writeSpmvSellFamilyCandidateRecords(out, options, workload, records);
3041         const tuning_key = try accy.kernel.library.sparse.spmvSellFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
3042         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
3043     }
3044     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_spmv_sell_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
3045 }
3046 
3047 fn writeSpmvSellFamilyCandidateRecords(
3048     out: *std.Io.Writer,
3049     options: config.Options,
3050     workload: workload_mod.Workload,
3051     records: []const accy.executable.LaunchCandidateRecord,
3052 ) !void {
3053     var best_record: ?accy.executable.LaunchCandidateRecord = null;
3054     for (records) |record| {
3055         try jsonl.writeSpmvSellFamilyCandidateMeasurementRecord(out, options, workload, record);
3056         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
3057             if (best_record) |best| try jsonl.writeSpmvSellFamilyCandidateBestRecord(out, options, workload, best);
3058             best_record = record;
3059         } else if (launchCandidateRecordBeats(record, best_record.?)) {
3060             best_record = record;
3061         }
3062     }
3063     if (best_record) |record| try jsonl.writeSpmvSellFamilyCandidateBestRecord(out, options, workload, record);
3064 }
3065 
3066 fn measureDirectSpmvCooFamilyCandidateRecord(
3067     scratch: Allocator,
3068     lifetime_allocator: Allocator,
3069     handle: gpu.BackendHandle,
3070     stream: ?gpu.StreamHandle,
3071     options: config.Options,
3072     candidate: accy.kernel.library.catalog.OwnedDescriptor,
3073     instance: accy.kernel.library.sparse.SpmvCoo,
3074     inputs: workload_mod.LaunchInputs,
3075 ) !accy.executable.LaunchCandidateRecord {
3076     if (options.samples == 0) return error.LaunchArgumentMismatch;
3077     if (inputs.inputs.len != 5) return error.InvalidArtifact;
3078 
3079     var owned_artifact = try accy.kernel.library.createOwnedKernelCallArtifact(
3080         lifetime_allocator,
3081         handle,
3082         candidate,
3083         .{ .limits = .standard },
3084     );
3085     defer owned_artifact.deinit();
3086     const entry = owned_artifact.entry();
3087 
3088     var artifact = try kernelArtifactFromKernelCallEntry(lifetime_allocator, handle, entry, "accy-choir-bench-spmv-coo");
3089     defer artifact.deinit();
3090     const loaded = try handle.loadArtifact(&artifact);
3091     defer handle.destroyObject(loaded.id);
3092 
3093     const runtime_arguments = try accy.kernel.library.sparse.spmvCooRuntimeArguments(
3094         instance,
3095         instance.nnz,
3096         instance.x_extent,
3097     );
3098     const geometry = try kernelCallLaunchGeometry(entry.launch, runtime_arguments[0..]);
3099 
3100     var buffers: [5]gpu.BufferHandle = undefined;
3101     var buffer_count: usize = 0;
3102     defer destroyBufferHandles(handle, buffers[0..buffer_count]);
3103 
3104     var bindings: [5]gpu.BufferBinding = undefined;
3105     const dtypes = [_]choir_abi.DType{ instance.dtype, .i32, .i32, instance.dtype, instance.dtype };
3106     const accesses = [_]gpu.BufferAccess{ .read_write, .read_only, .read_only, .read_only, .read_only };
3107     for (inputs.inputs, dtypes, accesses, 0..) |bytes, dtype, access, index| {
3108         buffers[index] = try handle.allocateBuffer(.{
3109             .byte_size = bytes.len,
3110             .alignment = 256,
3111             .dtype = dtype,
3112             .element_count = bytes.len / @as(usize, dtype.sizeOf()),
3113         });
3114         buffer_count += 1;
3115         bindings[index] = .{
3116             .handle = buffers[index],
3117             .access = access,
3118             .ownership = buffers[index].ownership,
3119             .byte_size = buffers[index].byte_size,
3120         };
3121     }
3122 
3123     try handle.writeBuffer(.{ .handle = bindings[1].handle, .bytes = inputs.inputs[1] });
3124     try handle.writeBuffer(.{ .handle = bindings[2].handle, .bytes = inputs.inputs[2] });
3125     try handle.writeBuffer(.{ .handle = bindings[3].handle, .bytes = inputs.inputs[3] });
3126     try handle.writeBuffer(.{ .handle = bindings[4].handle, .bytes = inputs.inputs[4] });
3127 
3128     var warmup_index: u32 = 0;
3129     while (warmup_index < options.warmup) : (warmup_index += 1) {
3130         try launchSpmvCooMeasurement(handle, stream, &artifact, loaded, bindings[0..], runtime_arguments[0..], geometry, inputs.inputs[0]);
3131     }
3132 
3133     const sample_times = try scratch.alloc(u64, options.samples);
3134     defer scratch.free(sample_times);
3135     for (sample_times) |*sample| {
3136         try handle.writeBuffer(.{ .handle = bindings[0].handle, .bytes = inputs.inputs[0] });
3137         const start = timing_mod.nowNanos();
3138         try handle.launch(.{
3139             .artifact = &artifact,
3140             .loaded_artifact = loaded,
3141             .buffers = bindings[0..],
3142             .scalar_arguments = runtime_arguments[0..],
3143             .geometry = geometry,
3144             .stream = stream,
3145         });
3146         try synchronizeScatterAddMeasurement(handle, stream);
3147         sample.* = @intCast(timing_mod.nowNanos() - start);
3148     }
3149     std.mem.sort(u64, sample_times, {}, std.sort.asc(u64));
3150 
3151     return try directSpmvCooLaunchCandidateRecord(
3152         scratch,
3153         entry,
3154         artifact,
3155         instance,
3156         geometry,
3157         sample_times[sample_times.len / 2],
3158         options.samples,
3159     );
3160 }
3161 
3162 fn launchSpmvCooMeasurement(
3163     handle: gpu.BackendHandle,
3164     stream: ?gpu.StreamHandle,
3165     artifact: *const gpu.KernelArtifact,
3166     loaded: gpu.LoadedArtifact,
3167     bindings: []const gpu.BufferBinding,
3168     runtime_arguments: []const choir_abi.ScalarArgument,
3169     geometry: choir_abi.LaunchGeometry,
3170     y_seed: []const u8,
3171 ) !void {
3172     try handle.writeBuffer(.{ .handle = bindings[0].handle, .bytes = y_seed });
3173     try handle.launch(.{
3174         .artifact = artifact,
3175         .loaded_artifact = loaded,
3176         .buffers = bindings,
3177         .scalar_arguments = runtime_arguments,
3178         .geometry = geometry,
3179         .stream = stream,
3180     });
3181     try synchronizeScatterAddMeasurement(handle, stream);
3182 }
3183 
3184 fn directSpmvCooLaunchCandidateRecord(
3185     allocator: Allocator,
3186     entry: accy.artifact.KernelCallArtifact,
3187     artifact: gpu.KernelArtifact,
3188     instance: accy.kernel.library.sparse.SpmvCoo,
3189     geometry: choir_abi.LaunchGeometry,
3190     median_ns: u64,
3191     sample_count: u32,
3192 ) !accy.executable.LaunchCandidateRecord {
3193     const entry_name = try allocator.dupe(u8, artifact.entry_name);
3194     const launch_extent = accy.kernel.library.sparse.spmvCooLaunchExtent(instance);
3195     return .{
3196         .kernel = .{
3197             .source = .kernel_call,
3198             .kernel_id = 0,
3199             .work_item_id = 0,
3200             .output_layout_fingerprint = 0,
3201             .input_layout_fingerprint = entry.shape_family_fingerprint orelse 0,
3202             .element_count = launch_extent,
3203             .op_count = std.math.cast(usize, launch_extent) orelse return error.InvalidArtifact,
3204             .entry_name = entry_name,
3205             .artifact_format = artifact.format,
3206             .artifact_payload_bytes = try kernelArtifactPayloadBytes(artifact),
3207             .compile_argument_count = artifact.argument_count,
3208             .compile_required_dtype_bits = entry.required_dtypes.bits,
3209             .compile_required_features = entry.required_features,
3210             .compile_required_subgroup = entry.required_subgroup,
3211             .compile_payload = compilePayloadKind(entry.payload),
3212             .compile_payload_bytes = try compilePayloadBytes(entry.payload),
3213             .compile_launch = .kernel_call,
3214             .runtime_scalar_argument_count = entry.runtime_scalar_argument_count,
3215             .launch_geometry = geometry,
3216             .launch_candidate_count = 1,
3217             .launch_resource_class = "unknown",
3218             .fixed_threadgroup = true,
3219             .subgroup_aligned = geometry.threadgroup[0] % 32 == 0,
3220             .subgroup_size = if (geometry.threadgroup[0] % 32 == 0) 32 else null,
3221             .static_bytes_complete = true,
3222         },
3223         .candidate_index = 0,
3224         .geometry = geometry,
3225         .candidate_score = 0,
3226         .estimated_static_bytes_per_threadgroup = 0,
3227         .estimated_element_ops_per_threadgroup = @as(u64, geometry.threadgroup[0]),
3228         .median_ns = median_ns,
3229         .sample_count = sample_count,
3230     };
3231 }
3232 
3233 fn runCudaSegmentSumFamilyCandidateMeasurements(
3234     arena: Allocator,
3235     backing_allocator: Allocator,
3236     out: *std.Io.Writer,
3237     options: config.Options,
3238     workload: workload_mod.Workload,
3239     handle: gpu.BackendHandle,
3240     stream: gpu.StreamHandle,
3241     phase_emitter: *BackendPhaseEmitter,
3242     family_sink: *FamilyTuningSink,
3243 ) !void {
3244     const family_shape = workload_mod.segmentSumFamilyShape(options) orelse return;
3245 
3246     var candidates = try accy.kernel.library.selectOwnedSegmentSumCandidates(backing_allocator, .{
3247         .dtype = .f32,
3248         .kind = .segment_sum,
3249         .segments = family_shape.segments,
3250         .total = family_shape.total,
3251     });
3252     defer candidates.deinit();
3253     if (candidates.count <= 1) return;
3254 
3255     var registry = try accy.kernel.library.createOwnedKernelCallArtifactRegistry(
3256         backing_allocator,
3257         handle,
3258         candidates.slice(),
3259         .{ .limits = .standard },
3260     );
3261     defer registry.deinit();
3262     const registry_value = registry.registry();
3263 
3264     const inputs = try workload_mod.buildSegmentSumKernelCallInputs(arena, family_shape);
3265 
3266     const measurement_start = timing_mod.nowNanos();
3267     for (candidates.slice()) |candidate| {
3268         const instance = accy.kernel.library.segmented.segmentSumInstanceFromSpecialization(
3269             candidate.descriptor.metadata.specialization,
3270         ) orelse return error.InvalidKernelLibraryEntry;
3271         const module = try workload_mod.buildSegmentSumKernelCallModule(
3272             backing_allocator,
3273             family_shape,
3274             candidate.descriptor.metadata.target,
3275             candidate.descriptor.metadata.version,
3276         );
3277         var module_owned = true;
3278         errdefer if (module_owned) module.deinit();
3279         module_owned = false;
3280         var fragment = try compileAndLoadProfileModule(backing_allocator, handle, module, .{
3281             .kernel_call_registry = &registry_value,
3282             .instrumentation = .{
3283                 .context = phase_emitter,
3284                 .observe = recordFragmentPhase,
3285             },
3286         });
3287         defer fragment.deinit();
3288 
3289         const bindings = try accy.executable.prepareInvocation(fragment, backing_allocator, inputs.inputs);
3290         defer bindings.deinit();
3291 
3292         const runtime_arguments = try accy.kernel.library.segmented.segmentSumRuntimeArguments(instance);
3293         const records = try measureAndRecordFamilyCandidateRecords(
3294             arena,
3295             arena,
3296             fragment,
3297             bindings,
3298             candidate.descriptor.name,
3299             .{
3300                 .warmup = options.warmup,
3301                 .samples = options.samples,
3302                 .base_options = .{
3303                     .stream = stream,
3304                     .runtime_scalar_arguments = runtime_arguments[0..],
3305                 },
3306                 .synchronize = .stream,
3307             },
3308         );
3309         defer arena.free(records);
3310         try writeSegmentSumFamilyCandidateRecords(out, options, workload, records);
3311         const tuning_key = try accy.kernel.library.segmented.segmentSumFamilyTuningKey(arena, family_sink.device_fingerprint, instance);
3312         try family_sink.appendBest(tuning_key, candidate.descriptor.metadata.target, records);
3313     }
3314     try jsonl.writeBackendPhaseRecord(out, options, workload, "measure_segment_sum_family_candidates", stats_mod.nsBetween(measurement_start, timing_mod.nowNanos()));
3315 }
3316 
3317 fn writeSegmentSumFamilyCandidateRecords(
3318     out: *std.Io.Writer,
3319     options: config.Options,
3320     workload: workload_mod.Workload,
3321     records: []const accy.executable.LaunchCandidateRecord,
3322 ) !void {
3323     var best_record: ?accy.executable.LaunchCandidateRecord = null;
3324     for (records) |record| {
3325         try jsonl.writeSegmentSumFamilyCandidateMeasurementRecord(out, options, workload, record);
3326         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
3327             if (best_record) |best| try jsonl.writeSegmentSumFamilyCandidateBestRecord(out, options, workload, best);
3328             best_record = record;
3329         } else if (launchCandidateRecordBeats(record, best_record.?)) {
3330             best_record = record;
3331         }
3332     }
3333     if (best_record) |record| try jsonl.writeSegmentSumFamilyCandidateBestRecord(out, options, workload, record);
3334 }
3335 
3336 fn writeMatrixProductFamilyCandidateRecords(
3337     out: *std.Io.Writer,
3338     options: config.Options,
3339     workload: workload_mod.Workload,
3340     records: []const accy.executable.LaunchCandidateRecord,
3341 ) !void {
3342     var best_record: ?accy.executable.LaunchCandidateRecord = null;
3343     for (records) |record| {
3344         try jsonl.writeMatrixProductFamilyCandidateMeasurementRecord(out, options, workload, record);
3345         if (best_record == null or record.kernel.kernel_id != best_record.?.kernel.kernel_id) {
3346             if (best_record) |best| try jsonl.writeMatrixProductFamilyCandidateBestRecord(out, options, workload, best);
3347             best_record = record;
3348         } else if (launchCandidateRecordBeats(record, best_record.?)) {
3349             best_record = record;
3350         }
3351     }
3352     if (best_record) |record| try jsonl.writeMatrixProductFamilyCandidateBestRecord(out, options, workload, record);
3353 }
3354 
3355 fn i64Extent(extent: u64) !i64 {
3356     return std.math.cast(i64, extent) orelse error.InvalidDimension;
3357 }
3358 
3359 fn launchCandidateRecordBeats(
3360     lhs: accy.executable.LaunchCandidateRecord,
3361     rhs: accy.executable.LaunchCandidateRecord,
3362 ) bool {
3363     if (lhs.median_ns != rhs.median_ns) return lhs.median_ns < rhs.median_ns;
3364     if (lhs.sample_count != rhs.sample_count) return lhs.sample_count > rhs.sample_count;
3365     return lhs.candidate_index < rhs.candidate_index;
3366 }
3367 
3368 fn initCudaBackendState(allocator: Allocator) !InitCudaResult {
3369     if (!accy.validation.gating.enabledByBuildOptions(.cuda)) {
3370         return .{ .skipped = "build flag disabled; rerun with -Dgpu-tests=true" };
3371     }
3372     if (!cuda_backend.platformSupported()) {
3373         return .{ .skipped = "unsupported platform" };
3374     }
3375 
3376     const state = cuda_backend.State.initDevice(allocator, 0) catch |err| switch (err) {
3377         error.RuntimeUnavailable => return .{ .skipped = "CUDA driver or device unavailable" },
3378         else => return err,
3379     };
3380     return .{ .state = state };
3381 }
3382 
3383 pub fn runPipeline(
3384     allocator: Allocator,
3385     options: config.Options,
3386     timing: ?*accy.preparation.BackendPreparationTiming,
3387     stats_out: ?*accy.preparation.BackendPreparationStats,
3388 ) !timing_mod.RunTimings {
3389     var allocation_tracker = bench.CountingAllocator.init(allocator);
3390     const compiler_allocator = allocation_tracker.allocator();
3391     if (timing) |timing_inst| {
3392         timing_inst.setAllocationSnapshotProvider(.{
3393             .context = &allocation_tracker,
3394             .snapshot = timing_mod.allocationSnapshot,
3395         });
3396     }
3397 
3398     const semantic_start = timing_mod.nowNanos();
3399     const semantic_memory_start = allocation_tracker.counts;
3400     const module = try workload_mod.buildSemanticModule(compiler_allocator, options);
3401     const semantic_ns = stats_mod.nsBetween(semantic_start, timing_mod.nowNanos());
3402     const semantic_memory = timing_mod.deltaCounts(allocation_tracker.counts, semantic_memory_start);
3403 
3404     const preparation_memory_start = allocation_tracker.counts;
3405     const pipeline_run = try accy.preparation.runBackendPreparationPipelineFromSemanticModule(compiler_allocator, module, .{
3406         .timing = timing,
3407         .now = timing_mod.nowNanos,
3408     });
3409     const preparation_memory = timing_mod.deltaCounts(allocation_tracker.counts, preparation_memory_start);
3410     if (stats_out) |stats| stats.* = pipeline_run.target_stats;
3411     return .{
3412         .total_ns = semantic_ns +| pipeline_run.total_ns,
3413         .semantic_ns = semantic_ns,
3414         .contract_ns = pipeline_run.contract_ns,
3415         .tensor_ns = pipeline_run.tensor_ns,
3416         .dispatch_ns = pipeline_run.dispatch_ns,
3417         .memory_ns = pipeline_run.memory_ns,
3418         .kernel_ns = pipeline_run.kernel_ns,
3419         .target_ns = pipeline_run.target_ns,
3420         .initial_choir_ops = pipeline_run.initial_choir_ops,
3421         .final_choir_ops = pipeline_run.final_choir_ops,
3422         .total_memory = timing_mod.memoryCounts(allocation_tracker.counts),
3423         .semantic_memory = semantic_memory,
3424         .preparation_memory = preparation_memory,
3425     };
3426 }
3427 
3428 fn sortStructureMeasurementOptions() config.Options {
3429     return .{
3430         .workload_kind = .elementwise_chain,
3431         .elements = 2048,
3432         .chain = 64,
3433     };
3434 }
3435 
3436 fn recordingDestroyedId(state: *const gpu.recording.BackendState, id: gpu.BackendObjectId) bool {
3437     const count = @min(state.destroy_count, state.destroyed_ids.len);
3438     for (state.destroyed_ids[0..count]) |destroyed_id| {
3439         if (destroyed_id == id) return true;
3440     }
3441     return false;
3442 }
3443 
3444 test "sort structure measurements release backend buffers" {
3445     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3446     defer arena_state.deinit();
3447     const options = sortStructureMeasurementOptions();
3448     const workload = workload_mod.Workload.init(options);
3449     var state = gpu.recording.BackendState{
3450         .allocator = std.testing.allocator,
3451         .kind = .cuda,
3452         .format = .cuda_ptx,
3453     };
3454     const handle = state.handle();
3455     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3456     defer writer.deinit();
3457     var sink = FamilyTuningSink{
3458         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3459         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3460     };
3461     defer sink.deinit();
3462 
3463     try runCudaSortStructureFamilyMeasurements(
3464         arena_state.allocator(),
3465         std.testing.allocator,
3466         &writer.writer,
3467         options,
3468         workload,
3469         handle,
3470         &sink,
3471     );
3472 
3473     try std.testing.expectEqual(@as(usize, 2), sink.accumulator.count());
3474     try std.testing.expect(state.buffer_allocate_count > 2);
3475     for (state.allocated_buffer_ids[0..state.buffer_allocate_count]) |buffer_id| {
3476         try std.testing.expect(recordingDestroyedId(&state, buffer_id));
3477     }
3478 }
3479 
3480 test "sort structure measurements release buffers after partial allocation failure" {
3481     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3482     defer arena_state.deinit();
3483     const options = sortStructureMeasurementOptions();
3484     const workload = workload_mod.Workload.init(options);
3485     var state = gpu.recording.BackendState{
3486         .allocator = std.testing.allocator,
3487         .kind = .cuda,
3488         .format = .cuda_ptx,
3489         .fail_buffer_allocate_after_count = 1,
3490     };
3491     const handle = state.handle();
3492     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3493     defer writer.deinit();
3494     var sink = FamilyTuningSink{
3495         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3496         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3497     };
3498     defer sink.deinit();
3499 
3500     try std.testing.expectError(error.OutOfMemory, runCudaSortStructureFamilyMeasurements(
3501         arena_state.allocator(),
3502         std.testing.allocator,
3503         &writer.writer,
3504         options,
3505         workload,
3506         handle,
3507         &sink,
3508     ));
3509 
3510     try std.testing.expectEqual(@as(usize, 1), state.buffer_allocate_count);
3511     try std.testing.expect(recordingDestroyedId(&state, state.allocated_buffer_ids[0]));
3512 }
3513 
3514 test "scatter add family measurements launch candidates directly" {
3515     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3516     defer arena_state.deinit();
3517     const options = config.Options{
3518         .workload_kind = .indexing_mix,
3519         .elements = 64,
3520         .chain = 16,
3521         .warmup = 1,
3522         .samples = 2,
3523     };
3524     const workload = workload_mod.Workload.init(options);
3525     var state = gpu.recording.BackendState{
3526         .allocator = std.testing.allocator,
3527         .kind = .cuda,
3528         .format = .cuda_ptx,
3529     };
3530     const handle = state.handle();
3531     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3532     defer writer.deinit();
3533     var sink = FamilyTuningSink{
3534         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3535         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3536     };
3537     defer sink.deinit();
3538 
3539     try runCudaScatterAddFamilyCandidateMeasurements(
3540         arena_state.allocator(),
3541         std.testing.allocator,
3542         &writer.writer,
3543         options,
3544         workload,
3545         handle,
3546         null,
3547         &sink,
3548     );
3549 
3550     const output = writer.written();
3551     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"scatter_add_family_candidate_measurement\"") != null);
3552     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"scatter_add_family_candidate_best\"") != null);
3553     try std.testing.expect(std.mem.indexOf(u8, output, "\"phase\":\"measure_scatter_add_family_candidates\"") != null);
3554     try std.testing.expect(sink.accumulator.count() > 1);
3555     try std.testing.expect(state.launch_count > 0);
3556     try std.testing.expect(state.write_count >= state.launch_count);
3557     try std.testing.expectEqual(@as(usize, 4), state.last_launch_buffer_count);
3558     try std.testing.expectEqual(gpu.BufferAccess.read_write, state.last_buffer_access[0]);
3559     try std.testing.expectEqual(gpu.BufferAccess.read_only, state.last_buffer_access[1]);
3560     try std.testing.expectEqual(gpu.BufferAccess.read_only, state.last_buffer_access[2]);
3561     try std.testing.expectEqual(@as(usize, 5), state.last_launch_scalar_count);
3562     try std.testing.expectEqual(@as(u32, 1), state.last_launch_scalar_u32_values[0]);
3563     try std.testing.expectEqual(@as(u32, 16), state.last_launch_scalar_u32_values[1]);
3564     try std.testing.expectEqual(@as(u32, 64), state.last_launch_scalar_u32_values[2]);
3565     try std.testing.expectEqual(@as(u32, 1), state.last_launch_scalar_u32_values[3]);
3566     try std.testing.expectEqual(@as(u32, 64), state.last_launch_scalar_u32_values[4]);
3567     try std.testing.expect(recordingDestroyedId(&state, state.last_loaded_id.?));
3568     for (state.allocated_buffer_ids[0..state.buffer_allocate_count]) |buffer_id| {
3569         try std.testing.expect(recordingDestroyedId(&state, buffer_id));
3570     }
3571 }
3572 
3573 test "coo spmv family measurements emit structure candidates" {
3574     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3575     defer arena_state.deinit();
3576     const options = config.Options{
3577         .workload_kind = .reduction_row_norm,
3578         .elements = 8,
3579         .chain = 8,
3580         .warmup = 1,
3581         .samples = 2,
3582     };
3583     const workload = workload_mod.Workload.init(options);
3584     var state = gpu.recording.BackendState{
3585         .allocator = std.testing.allocator,
3586         .kind = .cuda,
3587         .format = .cuda_ptx,
3588     };
3589     const handle = state.handle();
3590     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3591     defer writer.deinit();
3592     var phase_emitter = BackendPhaseEmitter{ .out = &writer.writer, .options = options, .workload = workload };
3593     var sink = FamilyTuningSink{
3594         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3595         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3596     };
3597     defer sink.deinit();
3598 
3599     try runCudaSpmvCooFamilyCandidateMeasurements(
3600         arena_state.allocator(),
3601         std.testing.allocator,
3602         &writer.writer,
3603         options,
3604         workload,
3605         handle,
3606         null,
3607         &phase_emitter,
3608         &sink,
3609     );
3610 
3611     const output = writer.written();
3612     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmv_coo_family_candidate_measurement\"") != null);
3613     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmv_coo_family_candidate_best\"") != null);
3614     try std.testing.expect(std.mem.indexOf(u8, output, "\"phase\":\"measure_spmv_coo_family_candidates\"") != null);
3615     try std.testing.expect(sink.accumulator.count() >= 1);
3616     try std.testing.expect(state.launch_count > 0);
3617     try std.testing.expectEqual(@as(usize, 5), state.last_launch_buffer_count);
3618     try std.testing.expectEqual(gpu.BufferAccess.read_write, state.last_buffer_access[0]);
3619     try std.testing.expectEqual(gpu.BufferAccess.read_only, state.last_buffer_access[1]);
3620     try std.testing.expectEqual(gpu.BufferAccess.read_only, state.last_buffer_access[2]);
3621     try std.testing.expectEqual(gpu.BufferAccess.read_only, state.last_buffer_access[3]);
3622     try std.testing.expectEqual(gpu.BufferAccess.read_only, state.last_buffer_access[4]);
3623     try std.testing.expectEqual(@as(usize, 3), state.last_launch_scalar_count);
3624     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[0]);
3625     try std.testing.expectEqual(@as(u32, 64), state.last_launch_scalar_u32_values[1]);
3626     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[2]);
3627     for (state.allocated_buffer_ids[0..state.buffer_allocate_count]) |buffer_id| {
3628         try std.testing.expect(recordingDestroyedId(&state, buffer_id));
3629     }
3630 }
3631 
3632 test "csr spmm family measurements emit thread candidates" {
3633     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3634     defer arena_state.deinit();
3635     const options = config.Options{
3636         .workload_kind = .reduction_row_norm,
3637         .elements = 70,
3638         .chain = 8,
3639         .warmup = 1,
3640         .samples = 2,
3641     };
3642     const workload = workload_mod.Workload.init(options);
3643     var state = gpu.recording.BackendState{
3644         .allocator = std.testing.allocator,
3645         .kind = .cuda,
3646         .format = .cuda_ptx,
3647     };
3648     const handle = state.handle();
3649     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3650     defer writer.deinit();
3651     var phase_emitter = BackendPhaseEmitter{ .out = &writer.writer, .options = options, .workload = workload };
3652     var sink = FamilyTuningSink{
3653         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3654         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3655     };
3656     defer sink.deinit();
3657 
3658     try runCudaSpmmCsrFamilyCandidateMeasurements(
3659         arena_state.allocator(),
3660         std.testing.allocator,
3661         &writer.writer,
3662         options,
3663         workload,
3664         handle,
3665         null,
3666         &phase_emitter,
3667         &sink,
3668     );
3669 
3670     const output = writer.written();
3671     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmm_csr_family_candidate_measurement\"") != null);
3672     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmm_csr_family_candidate_best\"") != null);
3673     try std.testing.expect(std.mem.indexOf(u8, output, "\"phase\":\"measure_spmm_csr_family_candidates\"") != null);
3674     try std.testing.expect(sink.accumulator.count() > 1);
3675     try std.testing.expect(state.launch_count > 0);
3676     try std.testing.expectEqual(@as(usize, 4), state.last_launch_scalar_count);
3677     try std.testing.expectEqual(@as(u32, 70), state.last_launch_scalar_u32_values[0]);
3678     try std.testing.expectEqual(@as(u32, 560), state.last_launch_scalar_u32_values[1]);
3679     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[2]);
3680     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[3]);
3681     for (state.allocated_buffer_ids[0..state.buffer_allocate_count]) |buffer_id| {
3682         try std.testing.expect(recordingDestroyedId(&state, buffer_id));
3683     }
3684 }
3685 
3686 test "ell spmv family measurements emit thread candidates" {
3687     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3688     defer arena_state.deinit();
3689     const options = config.Options{
3690         .workload_kind = .reduction_row_norm,
3691         .elements = 70,
3692         .chain = 8,
3693         .warmup = 1,
3694         .samples = 2,
3695     };
3696     const workload = workload_mod.Workload.init(options);
3697     var state = gpu.recording.BackendState{
3698         .allocator = std.testing.allocator,
3699         .kind = .cuda,
3700         .format = .cuda_ptx,
3701     };
3702     const handle = state.handle();
3703     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3704     defer writer.deinit();
3705     var phase_emitter = BackendPhaseEmitter{ .out = &writer.writer, .options = options, .workload = workload };
3706     var sink = FamilyTuningSink{
3707         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3708         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3709     };
3710     defer sink.deinit();
3711 
3712     try runCudaSpmvEllFamilyCandidateMeasurements(
3713         arena_state.allocator(),
3714         std.testing.allocator,
3715         &writer.writer,
3716         options,
3717         workload,
3718         handle,
3719         null,
3720         &phase_emitter,
3721         &sink,
3722     );
3723 
3724     const output = writer.written();
3725     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmv_ell_family_candidate_measurement\"") != null);
3726     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmv_ell_family_candidate_best\"") != null);
3727     try std.testing.expect(std.mem.indexOf(u8, output, "\"phase\":\"measure_spmv_ell_family_candidates\"") != null);
3728     try std.testing.expect(sink.accumulator.count() > 1);
3729     try std.testing.expect(state.launch_count > 0);
3730     try std.testing.expectEqual(@as(usize, 3), state.last_launch_scalar_count);
3731     try std.testing.expectEqual(@as(u32, 70), state.last_launch_scalar_u32_values[0]);
3732     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[1]);
3733     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[2]);
3734     for (state.allocated_buffer_ids[0..state.buffer_allocate_count]) |buffer_id| {
3735         try std.testing.expect(recordingDestroyedId(&state, buffer_id));
3736     }
3737 }
3738 
3739 test "sell spmv family measurements emit thread candidates" {
3740     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3741     defer arena_state.deinit();
3742     const options = config.Options{
3743         .workload_kind = .reduction_row_norm,
3744         .elements = 70,
3745         .chain = 8,
3746         .warmup = 1,
3747         .samples = 2,
3748     };
3749     const workload = workload_mod.Workload.init(options);
3750     var state = gpu.recording.BackendState{
3751         .allocator = std.testing.allocator,
3752         .kind = .cuda,
3753         .format = .cuda_ptx,
3754     };
3755     const handle = state.handle();
3756     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3757     defer writer.deinit();
3758     var phase_emitter = BackendPhaseEmitter{ .out = &writer.writer, .options = options, .workload = workload };
3759     var sink = FamilyTuningSink{
3760         .device_fingerprint = family_tuning.deviceFingerprint(try handle.queryCapabilities()),
3761         .accumulator = family_tuning.FamilyMeasurementAccumulator.init(std.testing.allocator),
3762     };
3763     defer sink.deinit();
3764 
3765     try runCudaSpmvSellFamilyCandidateMeasurements(
3766         arena_state.allocator(),
3767         std.testing.allocator,
3768         &writer.writer,
3769         options,
3770         workload,
3771         handle,
3772         null,
3773         &phase_emitter,
3774         &sink,
3775     );
3776 
3777     const output = writer.written();
3778     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmv_sell_family_candidate_measurement\"") != null);
3779     try std.testing.expect(std.mem.indexOf(u8, output, "\"kind\":\"spmv_sell_family_candidate_best\"") != null);
3780     try std.testing.expect(std.mem.indexOf(u8, output, "\"phase\":\"measure_spmv_sell_family_candidates\"") != null);
3781     try std.testing.expect(sink.accumulator.count() > 1);
3782     try std.testing.expect(state.launch_count > 0);
3783     try std.testing.expectEqual(@as(usize, 3), state.last_launch_scalar_count);
3784     try std.testing.expectEqual(@as(u32, 70), state.last_launch_scalar_u32_values[0]);
3785     try std.testing.expectEqual(@as(u32, 576), state.last_launch_scalar_u32_values[1]);
3786     try std.testing.expectEqual(@as(u32, 8), state.last_launch_scalar_u32_values[2]);
3787     for (state.allocated_buffer_ids[0..state.buffer_allocate_count]) |buffer_id| {
3788         try std.testing.expect(recordingDestroyedId(&state, buffer_id));
3789     }
3790 }
3791 
3792 test "matrix product schedule tuning sink folds candidates into executable artifact" {
3793     const allocator = std.testing.allocator;
3794     var state = gpu.recording.BackendState{
3795         .allocator = allocator,
3796         .kind = .cuda,
3797         .format = .cuda_ptx,
3798     };
3799     const caps = try state.handle().queryCapabilities();
3800     var sink = MatrixProductFamilyScheduleTuningSink.init(allocator, caps);
3801     defer sink.deinit();
3802 
3803     const candidates = [_]family_tuning.MatrixProductFamilyScheduleThreads{
3804         .{ .x = 17, .y = 9 },
3805         .{ .x = 16, .y = 16 },
3806         .{ .x = 8, .y = 8 },
3807     };
3808     const problem = family_tuning.MatrixProductFamilyScheduleTuningProblem{
3809         .format = .cuda_ptx,
3810         .m = 17,
3811         .n = 17,
3812         .k = 13,
3813         .dtype = .f32,
3814         .accumulation_dtype = .f32,
3815         .family_version = accy.kernel.library.linalg.matrix_product_family_version,
3816         .candidates = candidates[0..],
3817     };
3818 
3819     try sink.recordCandidateSet(problem, &.{
3820         .{ .threads = candidates[0], .median_ns = 1000, .sample_count = 30 },
3821         .{ .threads = candidates[1], .median_ns = 1010, .sample_count = 30 },
3822         .{ .threads = candidates[2], .median_ns = 1200, .sample_count = 30 },
3823     });
3824     try std.testing.expectEqual(@as(usize, 0), sink.cache.count());
3825 
3826     try sink.recordCandidateSet(problem, &.{
3827         .{ .threads = candidates[0], .median_ns = 1100, .sample_count = 30 },
3828         .{ .threads = candidates[1], .median_ns = 900, .sample_count = 30 },
3829         .{ .threads = candidates[2], .median_ns = 1300, .sample_count = 30 },
3830     });
3831     try std.testing.expectEqual(@as(usize, 1), sink.cache.count());
3832     try std.testing.expectEqual(@as(usize, 6), sink.measurement_count);
3833 
3834     const records = try sink.cache.exportRecords(allocator);
3835     defer allocator.free(records);
3836     try std.testing.expectEqual(@as(usize, 1), records.len);
3837     try std.testing.expectEqual(candidates[1], records[0].selection.threads);
3838     try std.testing.expectEqual(@as(u64, 900), records[0].selection.winner_median_ns);
3839     try std.testing.expectEqual(@as(u64, 1100), records[0].selection.runner_up_median_ns);
3840 
3841     const artifact = try schedule_tuning.encodeMatrixProductFamilyScheduleTuningArtifact(allocator, records);
3842     defer allocator.free(artifact);
3843     const decoded = try schedule_tuning.decodeMatrixProductFamilyScheduleTuningArtifact(allocator, artifact);
3844     defer allocator.free(decoded);
3845     try std.testing.expectEqual(records.len, decoded.len);
3846     try std.testing.expect(records[0].key.eql(decoded[0].key));
3847     try std.testing.expectEqual(records[0].selection.threads, decoded[0].selection.threads);
3848 }
3849 
3850 test "runSuite writes quick Accy Choir profiling suite records" {
3851     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3852     defer arena_state.deinit();
3853     const arena = arena_state.allocator();
3854 
3855     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3856     defer writer.deinit();
3857     try runSuite(arena, std.testing.allocator, &writer.writer, .{
3858         .suite_kind = .quick,
3859         .warmup = 0,
3860         .samples = 1,
3861     });
3862     const record = writer.written();
3863 
3864     try std.testing.expect(std.mem.indexOf(u8, record, "\"protocol\":\"accy.choir_bench/v36\"") != null);
3865     try std.testing.expect(std.mem.indexOf(u8, record, "\"metadata\":{") != null);
3866     try std.testing.expect(std.mem.indexOf(u8, record, "\"git_sha\":") != null);
3867     try std.testing.expect(std.mem.indexOf(u8, record, "\"host_os\":") != null);
3868     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"elementwise-chain\"") != null);
3869     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"indexing-mix\"") != null);
3870     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"reduction-row-norm\"") != null);
3871     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"coordinate-mesh\"") != null);
3872     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"nbody-all-pairs\"") != null);
3873     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"mlp-two-layer\"") != null);
3874     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"residual-block\"") != null);
3875     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"halo-stencil\"") != null);
3876     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"attention-softmax\"") != null);
3877     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"transformer-block\"") != null);
3878     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"decoder-block\"") != null);
3879     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_shape_ops\":10") != null);
3880     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_shape_ops\":4") != null);
3881     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_shape_ops\":6") != null);
3882     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_shape_ops\":17") != null);
3883     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_shape_ops\":14") != null);
3884     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_shape_ops\":12") != null);
3885     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_reduction_ops\":3") != null);
3886     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_reduction_ops\":1") != null);
3887     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_reduction_ops\":2") != null);
3888     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_reduction_ops\":6") != null);
3889     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_reduction_ops\":4") != null);
3890     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_dot_general_ops\":2") != null);
3891     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_dot_general_ops\":8") != null);
3892     try std.testing.expect(std.mem.indexOf(u8, record, "\"source_dot_general_ops\":9") != null);
3893     try std.testing.expect(std.mem.indexOf(u8, record, "\"summary\":\"pass\"") != null);
3894     try std.testing.expect(std.mem.indexOf(u8, record, "\"pass_ir_sizes\":false") != null);
3895     try std.testing.expect(std.mem.indexOf(u8, record, "\"summary\":\"memory\"") != null);
3896     try std.testing.expect(std.mem.indexOf(u8, record, "\"event\":\"end\"") != null);
3897 }
3898 
3899 test "runSuite writes scaling Accy Choir profiling suite records" {
3900     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3901     defer arena_state.deinit();
3902     const arena = arena_state.allocator();
3903 
3904     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3905     defer writer.deinit();
3906     try runSuite(arena, std.testing.allocator, &writer.writer, .{
3907         .suite_kind = .scaling,
3908         .warmup = 0,
3909         .samples = 1,
3910     });
3911     const record = writer.written();
3912 
3913     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"elementwise-chain\"") != null);
3914     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"indexing-mix\"") != null);
3915     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"reduction-row-norm\"") != null);
3916     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"nbody-all-pairs\"") != null);
3917     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"transformer-block\"") != null);
3918     try std.testing.expect(std.mem.indexOf(u8, record, "\"workload\":\"decoder-block\"") != null);
3919     try std.testing.expect(std.mem.indexOf(u8, record, "\"elements\":32") != null);
3920     try std.testing.expect(std.mem.indexOf(u8, record, "\"elements\":4096") != null);
3921     try std.testing.expect(std.mem.indexOf(u8, record, "\"chain\":128") != null);
3922     try std.testing.expect(std.mem.indexOf(u8, record, "\"event\":\"end\"") != null);
3923 }
3924 
3925 test "runWorkload records pass IR sizes when requested" {
3926     var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
3927     defer arena_state.deinit();
3928     const arena = arena_state.allocator();
3929 
3930     var writer = std.Io.Writer.Allocating.init(std.testing.allocator);
3931     defer writer.deinit();
3932     try runWorkload(arena, std.testing.allocator, &writer.writer, .{
3933         .workload_kind = .transformer_block,
3934         .elements = 16,
3935         .chain = 32,
3936         .warmup = 0,
3937         .samples = 1,
3938         .pass_ir_sizes = true,
3939     });
3940     const record = writer.written();
3941 
3942     try std.testing.expect(std.mem.indexOf(u8, record, "\"pass_ir_sizes\":true") != null);
3943     try std.testing.expect(std.mem.indexOf(u8, record, "\"pass\":\"accy-choir-plan-kernelization\"") != null);
3944     try std.testing.expect(std.mem.indexOf(u8, record, "\"pass_choir_ops_before\":0") == null);
3945     try std.testing.expect(std.mem.indexOf(u8, record, "\"mean_pass_choir_ops_before\":0") == null);
3946 }