lib/accy/src/preparation/test.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const gpu = @import("gpu");
3 const choir_abi = @import("choir_abi");
4 const preparation = @import("root.zig");
5
6 test {
7 _ = @import("coverage.zig");
8 _ = @import("kernelization/test.zig");
9 _ = @import("numerics.zig");
10 @import("test_discovery").discover(preparation);
11 }
12
13 test "accy preparation declaration coverage" {
14 std.testing.refAllDecls(preparation);
15 }
16
17 const choir = @import("choir");
18 const records = @import("../choir/record/root.zig");
19 const semantic = @import("../choir/root.zig").semantic;
20 const kernel_model = @import("../kernel/model/root.zig");
21
22 const StageRequest = struct {
23 pipeline: ?[]const revision.record.Version = null,
24 recipe_version: u32 = preparation.recipe.identity.version,
25 allowance: revision.WorkVector = revision.WorkVector.uniform(1024 * 1024 * 1024),
26 workspace: u64 = 128 * 1024 * 1024,
27 gate_scratch_bytes: u32 = configuration.gate_scratch,
28 };
29
30 fn stageBuilder(
31 chain: *Chain,
32 root: *choir.ir.Operation,
33 comptime stage: publication.Stage,
34 options: preparation.BackendPreparationRunOptions,
35 request: StageRequest,
36 ) !*revision.Builder {
37 const index = @backingInt(stage);
38 const source: StageSource = .{
39 .store = chain.store,
40 .kind = chain.kinds[index],
41 .parent = chain.products[index - 1],
42 };
43 return source.begin(root, stage, options, request);
44 }
45
46 const StageSource = struct {
47 store: *revision.Store,
48 kind: *const revision.Kind,
49 parent: operation.Product,
50
51 fn create(
52 parent: operation.Product,
53 comptime stage: publication.Stage,
54 scratch_bytes: u32,
55 ) !StageSource {
56 const allocator = std.testing.allocator;
57 const store = try revision.Store.create(allocator, .{
58 .revisions = 1,
59 .kinds = 1,
60 .builders = 1,
61 .compiler_manifests = 2,
62 .record_bytes = 32 * 1024 * 1024,
63 .gate_scratch_bytes = scratch_bytes,
64 .candidate_count = 8,
65 .screening_bytes = 32 * 1024 * 1024,
66 });
67 errdefer store.release();
68 const kind = try preparation.publication.registerKind(
69 allocator,
70 store,
71 stage,
72 configuration,
73 );
74 store.freeze();
75 return .{ .store = store, .kind = kind, .parent = parent };
76 }
77
78 fn begin(
79 self: StageSource,
80 root: *choir.ir.Operation,
81 comptime stage: publication.Stage,
82 options: preparation.BackendPreparationRunOptions,
83 request: StageRequest,
84 ) !*revision.Builder {
85 const allocator = std.testing.allocator;
86 const bytes = try preparation.recipe.encode(allocator, stage, root, options);
87 defer allocator.free(bytes);
88 const policy = try choir.product.recipe.encode(allocator, .{
89 .arithmetic = root.context.arithmetic_policy,
90 .policy = "accounted-stage-test",
91 });
92 defer allocator.free(policy);
93 return self.store.begin(.{
94 .kind = self.kind,
95 .address = .{
96 .producer = "accy",
97 .source = "accounted-stage-test",
98 .stage = stage.name(),
99 .variant = "",
100 },
101 .inputs = .{
102 .compiler_manifest = try choir.product.compiler.manifest(),
103 .versions = &.{ stage.schema(), operation.schema_identity, .{
104 .name = preparation.recipe.identity.name,
105 .version = request.recipe_version,
106 } },
107 .pipeline = request.pipeline orelse preparation.recipe.pipeline(stage),
108 .options = bytes,
109 .policy = policy,
110 },
111 .dependencies = &.{.{ .role = "source", .revision = self.parent.revision }},
112 .parent = self.parent.revision,
113 }, .{
114 .allowance = request.allowance,
115 .workspace = request.workspace,
116 .events = 256,
117 });
118 }
119 };
120
121 fn abortStage(builder: *revision.Builder) void {
122 if (builder.abort(.rejected)) |failure| {
123 var owned = failure;
124 owned.deinit();
125 }
126 }
127
128 fn expectStageWork(
129 receipt: revision.WorkReceiptV1,
130 comptime stage: publication.Stage,
131 workspace: u64,
132 ) !void {
133 try std.testing.expectEqual(.success, receipt.outcome);
134 try std.testing.expect(!receipt.missing_work_contract);
135 try std.testing.expectEqual(
136 preparation.recipe.pipeline(stage).len,
137 receipt.executed.counters.pass_runs,
138 );
139 try std.testing.expect(receipt.executed.counters.analysis_misses > 0);
140 var found = false;
141 for (receipt.events) |*event| {
142 if (!std.mem.eql(u8, event.identity().name, "accy-stage-setup")) continue;
143 found = true;
144 const capacity = workspace - configuration.gate_scratch;
145 try std.testing.expectEqual(capacity, event.workspace);
146 try std.testing.expectEqual(capacity, event.charged.allocation_capacity);
147 try std.testing.expect(event.executed.work.exceeded(event.charged) == null);
148 }
149 try std.testing.expect(found);
150 }
151
152 test "Accy publication runs recorded Dispatch and Memory producers before sealing" {
153 const allocator = std.testing.allocator;
154 var fixture = try Fixture.init();
155 defer fixture.deinit();
156 var captured = try captureFixture(&fixture);
157 defer captured.deinit();
158 var chain = try Chain.init(fixture.source.choir_module, captured);
159 defer chain.deinit();
160 inline for (.{ publication.Stage.dispatch, publication.Stage.memory }) |stage| {
161 const index = @backingInt(stage);
162 const builder = try stageBuilder(&chain, fixture.source.choir_module, stage, .{}, .{});
163 var open = true;
164 defer if (open) abortStage(builder);
165 try preparation.publication.capture(
166 allocator,
167 builder,
168 chain.products[index - 1],
169 stage,
170 .{},
171 configuration,
172 );
173 const product = try operation.Product.seal(builder, chain.kinds[index]);
174 open = false;
175 defer product.release();
176 chain.products[index].release();
177 chain.products[index] = try product.retain();
178 try expectStageWork(product.revision.view().work, stage, (StageRequest{}).workspace);
179 const module = try publication.Module(stage).fromRevision(allocator, product.revision);
180 defer module.deinit();
181 const image = try module.open(allocator, configuration.image);
182 defer image.destroy();
183 const bytes = image.stage();
184 var plan = try records.codec.decode(allocator, publication.Plan(stage), stage, bytes);
185 defer plan.deinit();
186 if (stage == .dispatch) {
187 try std.testing.expectEqual(@as(usize, 1), plan.value.schedule.work_items.len);
188 } else {
189 try std.testing.expectEqual(@as(usize, 3), plan.value.buffers.slots.len);
190 }
191 }
192 }
193
194 fn markSavedProgram(fixture: *Fixture) !void {
195 var context = fixture.context();
196 defer context.deinit();
197 const generated = try preparation.kernelization.getKernelizationAnalysis(
198 &context,
199 fixture.source.choir_module,
200 );
201 const source = &generated.kernels.items[0];
202 const program_context = source.program.kernelModule().context;
203 try source.program.storage.kernel.func().op.setAttr(
204 "restoration_marker",
205 try program_context.getStringAttr("retain this exact program"),
206 );
207 source.body_fingerprint = try source.program.bodyFingerprint(std.testing.allocator);
208 }
209
210 test "Accy publication restores exact Kernel programs into Target after source destruction" {
211 const allocator = std.testing.allocator;
212 var fixture = try Fixture.init();
213 var live = true;
214 defer if (live) fixture.deinit();
215 try markSavedProgram(&fixture);
216 var captured = try captureFixture(&fixture);
217 defer captured.deinit();
218 var chain = try Chain.init(fixture.source.choir_module, captured);
219 defer chain.deinit();
220 var expected = try records.codec.decode(
221 allocator,
222 records.kernel.Record,
223 .kernel,
224 captured.kernel,
225 );
226 defer expected.deinit();
227 const options = preparation.BackendPreparationRunOptions{ .target_profile = .{
228 .backend_kind = .cpu,
229 .artifact_format = .cpu_object,
230 } };
231 try preparation.recipe.applyTargetOptions(allocator, fixture.source.choir_module, options);
232 const request = StageRequest{
233 .allowance = revision.WorkVector.uniform(64 * 1024 * 1024 * 1024),
234 .workspace = 256 * 1024 * 1024,
235 };
236 const builder = try stageBuilder(
237 &chain,
238 fixture.source.choir_module,
239 .target,
240 options,
241 request,
242 );
243 var open = true;
244 defer if (open) abortStage(builder);
245 fixture.deinit();
246 live = false;
247 @memset(captured.kernel, 0xa5);
248 try preparation.publication.capture(
249 allocator,
250 builder,
251 chain.products[5],
252 .target,
253 options,
254 configuration,
255 );
256 const product = try operation.Product.seal(builder, chain.kinds[6]);
257 open = false;
258 defer product.release();
259 try expectStageWork(product.revision.view().work, .target, request.workspace);
260 const target = try publication.Module(.target).fromRevision(allocator, product.revision);
261 defer target.deinit();
262 var plan = try target.plan(allocator, configuration.image);
263 defer plan.deinit();
264 try std.testing.expectEqual(expected.value.generated.kernels.len, plan.value.kernels.len);
265 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
266 defer references.deinit();
267 for (plan.value.kernels, expected.value.generated.kernels) |actual, wanted| {
268 try records.codec.compare(actual.lowered, wanted, &references);
269 }
270 }
271
272 const variant_options = [_]preparation.BackendPreparationRunOptions{
273 .{ .target_profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object } },
274 .{ .target_profile = .{ .backend_kind = .cuda, .artifact_format = .cuda_ptx } },
275 };
276
277 fn variantBuilders(fixture: *Fixture, chain: *Chain) ![variant_options.len]*revision.Builder {
278 var builders: [variant_options.len]*revision.Builder = undefined;
279 var initialized: usize = 0;
280 errdefer for (builders[0..initialized]) |builder| abortStage(builder);
281 for (variant_options, &builders) |options, *builder| {
282 const root = fixture.source.choir_module;
283 try preparation.recipe.applyTargetOptions(std.testing.allocator, root, options);
284 builder.* = try stageBuilder(chain, root, .target, options, .{
285 .allowance = revision.WorkVector.uniform(64 * 1024 * 1024 * 1024),
286 .workspace = 256 * 1024 * 1024,
287 });
288 initialized += 1;
289 }
290 return builders;
291 }
292
293 fn expectTargetVariant(
294 product: operation.Product,
295 profile: preparation.BackendTargetProfile,
296 parent: records.kernel.Record,
297 ) !void {
298 const allocator = std.testing.allocator;
299 const module = try publication.Module(.target).fromRevision(allocator, product.revision);
300 defer module.deinit();
301 var plan = try module.plan(allocator, configuration.image);
302 defer plan.deinit();
303 try std.testing.expectEqualDeep(profile, plan.value.profile.?);
304 try std.testing.expectEqual(parent.generated.kernels.len, plan.value.kernels.len);
305 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
306 defer references.deinit();
307 const extra: u32 = if (profile.backend_kind == .cpu) 7 else 0;
308 for (plan.value.kernels, parent.generated.kernels) |actual, expected| {
309 try records.codec.compare(actual.lowered, expected, &references);
310 try std.testing.expectEqual(expected.argument_count + extra, actual.abi.?.argument_count);
311 try std.testing.expectEqual(extra, actual.abi.?.static_arguments.len);
312 }
313 }
314
315 test "Accy publication keeps CPU and CUDA successors independent on one sealed Kernel" {
316 const allocator = std.testing.allocator;
317 var fixture = try Fixture.init();
318 var live = true;
319 defer if (live) fixture.deinit();
320 try markSavedProgram(&fixture);
321 var captured = try captureFixture(&fixture);
322 defer captured.deinit();
323 var chain = try Chain.init(fixture.source.choir_module, captured);
324 defer chain.deinit();
325 var expected = try records.codec.decode(
326 allocator,
327 records.kernel.Record,
328 .kernel,
329 captured.kernel,
330 );
331 defer expected.deinit();
332 const parent = chain.products[5].revision;
333 const parent_image = try allocator.dupe(u8, parent.view().exact.image);
334 defer allocator.free(parent_image);
335 const builders = try variantBuilders(&fixture, &chain);
336 var sealed: usize = 0;
337 defer for (builders[sealed..]) |builder| abortStage(builder);
338 var first_image: ?[]u8 = null;
339 defer if (first_image) |bytes| allocator.free(bytes);
340 var products: [variant_options.len]operation.Product = undefined;
341 defer for (products[0..sealed]) |product| product.release();
342 fixture.deinit();
343 live = false;
344 @memset(captured.kernel, 0xa5);
345 for (variant_options, builders, &products) |options, builder, *product| {
346 try preparation.publication.capture(
347 allocator,
348 builder,
349 chain.products[5],
350 .target,
351 options,
352 configuration,
353 );
354 product.* = try operation.Product.seal(builder, chain.kinds[6]);
355 sealed += 1;
356 if (sealed == 1) {
357 first_image = try allocator.dupe(u8, product.revision.view().exact.image);
358 }
359 try expectStageWork(product.revision.view().work, .target, 256 * 1024 * 1024);
360 }
361 try std.testing.expect(!products[0].revision.eql(products[1].revision));
362 try std.testing.expectEqualSlices(u8, first_image.?, products[0].revision.view().exact.image);
363 try std.testing.expectEqualSlices(u8, parent_image, parent.view().exact.image);
364 for (products, variant_options) |product, options| {
365 try expectTargetVariant(product, options.target_profile.?, expected.value);
366 }
367 }
368
369 test "Accy publication reports exhausted Target restoration workspace after completing its passes" {
370 var fixture = try Fixture.init();
371 defer fixture.deinit();
372 var captured = try captureFixture(&fixture);
373 defer captured.deinit();
374 var chain = try Chain.init(fixture.source.choir_module, captured);
375 defer chain.deinit();
376 const options = preparation.BackendPreparationRunOptions{ .target_profile = .{
377 .backend_kind = .cpu,
378 .artifact_format = .cpu_object,
379 } };
380 try preparation.recipe.applyTargetOptions(
381 std.testing.allocator,
382 fixture.source.choir_module,
383 options,
384 );
385 const builder = try stageBuilder(&chain, fixture.source.choir_module, .target, options, .{
386 .allowance = revision.WorkVector.uniform(64 * 1024 * 1024 * 1024),
387 });
388 defer abortStage(builder);
389 try std.testing.expectError(error.WorkExhausted, preparation.publication.capture(
390 std.testing.allocator,
391 builder,
392 chain.products[5],
393 .target,
394 options,
395 configuration,
396 ));
397 const receipt = builder.accounting().view();
398 try std.testing.expectEqual(.exhausted, receipt.outcome);
399 try std.testing.expectEqual(
400 preparation.recipe.pipeline(.target).len,
401 receipt.executed.counters.pass_runs,
402 );
403 const last = receipt.events[receipt.events.len - 1];
404 try std.testing.expectEqualStrings("accy-stage-record", last.identity().name);
405 }
406
407 test "Accy publication checks ordered pipeline identities and recipe versions before execution" {
408 var fixture = try Fixture.init();
409 defer fixture.deinit();
410 var captured = try captureFixture(&fixture);
411 defer captured.deinit();
412 var chain = try Chain.init(fixture.source.choir_module, captured);
413 defer chain.deinit();
414 const pipeline = preparation.recipe.pipeline(.dispatch);
415 const requests = [_]StageRequest{
416 .{ .pipeline = &.{ pipeline[1], pipeline[0] } },
417 .{ .recipe_version = preparation.recipe.identity.version + 1 },
418 };
419 for (requests) |request| {
420 const builder = try stageBuilder(
421 &chain,
422 fixture.source.choir_module,
423 .dispatch,
424 .{},
425 request,
426 );
427 defer abortStage(builder);
428 try std.testing.expectError(error.RecipeMismatch, preparation.publication.capture(
429 std.testing.allocator,
430 builder,
431 chain.products[2],
432 .dispatch,
433 .{},
434 configuration,
435 ));
436 const receipt = builder.accounting().view();
437 try std.testing.expectEqual(.rejected, receipt.outcome);
438 try std.testing.expectEqual(@as(u64, 0), receipt.executed.counters.pass_runs);
439 }
440 }
441
442 test "Accy publication exhausts admission before executing a successor" {
443 var fixture = try Fixture.init();
444 defer fixture.deinit();
445 var captured = try captureFixture(&fixture);
446 defer captured.deinit();
447 var chain = try Chain.init(fixture.source.choir_module, captured);
448 defer chain.deinit();
449 var accounting_limit = StageRequest{};
450 accounting_limit.allowance.structural_visits = 0;
451 const requests = [_]StageRequest{
452 accounting_limit,
453 .{ .workspace = configuration.gate_scratch + 1024 },
454 };
455 for (requests) |request| {
456 const builder = try stageBuilder(
457 &chain,
458 fixture.source.choir_module,
459 .dispatch,
460 .{},
461 request,
462 );
463 defer abortStage(builder);
464 try std.testing.expectError(error.WorkExhausted, preparation.publication.capture(
465 std.testing.allocator,
466 builder,
467 chain.products[2],
468 .dispatch,
469 .{},
470 configuration,
471 ));
472 const receipt = builder.accounting().view();
473 try std.testing.expectEqual(.exhausted, receipt.outcome);
474 try std.testing.expectEqual(@as(u64, 0), receipt.executed.counters.pass_runs);
475 try std.testing.expectEqual(@as(u64, 0), receipt.executed.counters.analysis_misses);
476 }
477 }
478
479 test "Accy publication rejects unrecorded options before running a producer" {
480 var fixture = try Fixture.init();
481 defer fixture.deinit();
482 var captured = try captureFixture(&fixture);
483 defer captured.deinit();
484 var chain = try Chain.init(fixture.source.choir_module, captured);
485 defer chain.deinit();
486 const builder = try stageBuilder(&chain, fixture.source.choir_module, .tensor, .{}, .{});
487 defer abortStage(builder);
488 var options = preparation.BackendPreparationRunOptions{};
489 options.tensor.einsum.beam_width += 1;
490 try std.testing.expectError(error.RecipeMismatch, preparation.publication.capture(
491 std.testing.allocator,
492 builder,
493 chain.products[1],
494 .tensor,
495 options,
496 configuration,
497 ));
498 const receipt = builder.accounting().view();
499 try std.testing.expectEqual(.rejected, receipt.outcome);
500 try std.testing.expectEqual(@as(u64, 0), receipt.executed.counters.pass_runs);
501 }
502
503 test "Accy publication runs recorded Kernel producers before sealing" {
504 var fixture = try Fixture.init();
505 defer fixture.deinit();
506 var captured = try captureFixture(&fixture);
507 defer captured.deinit();
508 var chain = try Chain.init(fixture.source.choir_module, captured);
509 defer chain.deinit();
510 const request: StageRequest = .{
511 .allowance = revision.WorkVector.uniform(std.math.maxInt(u64)),
512 .workspace = 512 * 1024 * 1024,
513 };
514 const builder = try stageBuilder(&chain, fixture.source.choir_module, .kernel, .{}, request);
515 var open = true;
516 defer if (open) abortStage(builder);
517 try preparation.publication.capture(
518 std.testing.allocator,
519 builder,
520 chain.products[@backingInt(publication.Stage.memory)],
521 .kernel,
522 .{},
523 configuration,
524 );
525 const product = try operation.Product.seal(builder, chain.kinds[@backingInt(publication.Stage.kernel)]);
526 open = false;
527 defer product.release();
528 try expectStageWork(product.revision.view().work, .kernel, request.workspace);
529 const module = try publication.Module(.kernel).fromRevision(std.testing.allocator, product.revision);
530 defer module.deinit();
531 var actual = try module.plan(std.testing.allocator, configuration.image);
532 defer actual.deinit();
533 var expected = try records.codec.decode(
534 std.testing.allocator,
535 records.kernel.Record,
536 .kernel,
537 captured.kernel,
538 );
539 defer expected.deinit();
540 try std.testing.expectEqual(@as(usize, 1), actual.value.generated.kernels.len);
541 var references = records.reference.Index{ .allocator = std.testing.allocator, .limit = 0 };
542 defer references.deinit();
543 try records.codec.compare(actual.value, expected.value, &references);
544 }
545
546 test "Accy publication Contract producers seal only after accounting" {
547 var fixture = try Fixture.init();
548 defer fixture.deinit();
549 var captured = try captureFixture(&fixture);
550 defer captured.deinit();
551 var chain = try Chain.init(fixture.source.choir_module, captured);
552 defer chain.deinit();
553 const product = try publishStage(&chain, fixture.source.choir_module, .contract);
554 defer product.release();
555 const receipt = product.revision.view().work;
556 try std.testing.expectEqual(.success, receipt.outcome);
557 try std.testing.expect(!receipt.missing_work_contract);
558 try std.testing.expectEqual(preparation.recipe.pipeline(.contract).len, receipt.executed.counters.pass_runs);
559 const module = try publication.Module(.contract).fromRevision(std.testing.allocator, product.revision);
560 defer module.deinit();
561 const image = try module.open(std.testing.allocator, configuration.image);
562 defer image.destroy();
563 try std.testing.expectEqualSlices(u8, &publication.irRecord(.contract), image.stage());
564 }
565
566 const Fixture = struct {
567 source: *semantic.SemanticModule,
568 cache: choir.passes.AnalysisCache,
569
570 fn init() !Fixture {
571 return initNames(&.{"plan_image"});
572 }
573
574 fn initNames(names: []const []const u8) !Fixture {
575 const allocator = std.testing.allocator;
576 var builder = try semantic.Builder.init(allocator, .standard);
577 defer builder.deinit();
578 const typ = try builder.tensor(.f32, &.{4});
579 for (names) |name| {
580 var function = try builder.beginFunction(name, &.{ typ, typ }, &.{typ});
581 const first = try function.add(function.parameter(0), function.parameter(1));
582 const second = try function.add(first, function.parameter(1));
583 try function.return_(&.{second});
584 try function.finish();
585 }
586 return .{
587 .source = try builder.finish(),
588 .cache = choir.passes.AnalysisCache.init(allocator, null),
589 };
590 }
591
592 fn deinit(self: *Fixture) void {
593 self.cache.deinit();
594 self.source.deinit();
595 }
596
597 fn context(self: *Fixture) choir.passes.PassContext {
598 return choir.passes.PassContext.init(
599 self.source.choir_module,
600 self.source.context(),
601 std.testing.allocator,
602 &self.cache,
603 );
604 }
605 };
606
607 const Snapshot = struct {
608 dispatch: []u8,
609 memory: []u8,
610 kernel: []u8,
611 target: []u8,
612 image: []u8,
613
614 fn deinit(self: *Snapshot) void {
615 const allocator = std.testing.allocator;
616 allocator.free(self.dispatch);
617 allocator.free(self.memory);
618 allocator.free(self.kernel);
619 allocator.free(self.target);
620 allocator.free(self.image);
621 }
622 };
623
624 fn snapshot() !Snapshot {
625 var fixture = try Fixture.init();
626 defer fixture.deinit();
627 return captureFixture(&fixture);
628 }
629
630 fn captureFixture(fixture: *Fixture) !Snapshot {
631 const allocator = std.testing.allocator;
632 var context = fixture.context();
633 defer context.deinit();
634 const root = fixture.source.choir_module;
635 const fusion = try preparation.fusion.getFusionPlanAnalysis(&context, root);
636 const schedule = try preparation.schedule.getSchedulePlanAnalysis(&context, root);
637 const buffers = try preparation.bufferization.getBufferPlanAnalysis(&context, root);
638 const spaces = try preparation.memory.getMemorySpacePlanAnalysis(&context, root);
639 const layouts = try preparation.layout.getLayoutPlanAnalysis(&context, root);
640 const dispatch = try preparation.capture.dispatch(allocator, root, fusion, schedule, 1024);
641 errdefer allocator.free(dispatch);
642 const memory = try preparation.capture.memory(allocator, root, buffers, spaces, layouts, 1024);
643 errdefer allocator.free(memory);
644 const outlines = try preparation.outlining.getKernelOutlinePlanAnalysis(&context, root);
645 const generated = try preparation.kernelization.getKernelizationAnalysis(&context, root);
646 const kernel = try preparation.capture.kernel(
647 allocator,
648 root,
649 outlines,
650 generated,
651 configuration,
652 );
653 errdefer allocator.free(kernel);
654 const target = try preparation.capture.target(
655 allocator,
656 root,
657 schedule,
658 generated.kernels.items,
659 configuration,
660 );
661 errdefer allocator.free(target);
662 return .{
663 .dispatch = dispatch,
664 .memory = memory,
665 .kernel = kernel,
666 .target = target,
667 .image = try choir.bytecode.encodeModule(allocator, root),
668 };
669 }
670
671 test "Accy stage records own dispatch and memory plans after job destruction" {
672 const allocator = std.testing.allocator;
673 var captured = try snapshot();
674 defer captured.deinit();
675 const image = try choir.bytecode.image.Index.create(allocator, captured.image, .{
676 .bytes = 65536,
677 .entities = 1024,
678 .depth = 32,
679 });
680 defer image.destroy();
681 var dispatch = try records.codec.decode(
682 allocator,
683 records.dispatch.Record,
684 .dispatch,
685 captured.dispatch,
686 );
687 defer dispatch.deinit();
688 var memory = try records.codec.decode(
689 allocator,
690 records.memory.Record,
691 .memory,
692 captured.memory,
693 );
694 defer memory.deinit();
695 try records.codec.validateReferences(dispatch.value, image.view());
696 try records.codec.validateReferences(memory.value, image.view());
697 try std.testing.expect(dispatch.value.schedule.work_items.len > 0);
698 try std.testing.expect(dispatch.value.fusion.clusters.len > 0);
699 try std.testing.expect(memory.value.buffers.slots.len > 0);
700 var functions: usize = 0;
701 var outputs: usize = 0;
702 for (memory.value.buffers.slots) |slot| {
703 if (slot.function != null) functions += 1;
704 }
705 for (memory.value.spaces.assignments) |assignment| {
706 if (assignment.output_source != null) outputs += 1;
707 }
708 try std.testing.expect(functions > 0);
709 try std.testing.expect(outputs > 0);
710 @memset(captured.dispatch, 0xa5);
711 @memset(captured.memory, 0xa5);
712 try records.codec.validateReferences(dispatch.value, image.view());
713 try records.codec.validateReferences(memory.value, image.view());
714 }
715
716 test "Accy stage records include fusion kinds and memory output sources" {
717 const allocator = std.testing.allocator;
718 var fixture = try Fixture.init();
719 defer fixture.deinit();
720 var context = fixture.context();
721 defer context.deinit();
722 const root = fixture.source.choir_module;
723 const fusion = try preparation.fusion.getFusionPlanAnalysis(&context, root);
724 const schedule = try preparation.schedule.getSchedulePlanAnalysis(&context, root);
725 const before = try preparation.capture.dispatch(allocator, root, fusion, schedule, 1024);
726 defer allocator.free(before);
727 try std.testing.expect(fusion.clusters.items.len > 0);
728 const clusters = @constCast(fusion).clusters.items;
729 clusters[0].kind = .row_pipeline;
730 var decoded_before = try records.codec.decode(
731 allocator,
732 records.dispatch.Record,
733 .dispatch,
734 before,
735 );
736 defer decoded_before.deinit();
737 var references = try records.reference.Index.init(allocator, root, 1024);
738 defer references.deinit();
739 try std.testing.expectError(error.UnencodableProduct, records.codec.compare(
740 decoded_before.value,
741 .{ .fusion = fusion, .schedule = schedule },
742 &references,
743 ));
744 const after = try preparation.capture.dispatch(allocator, root, fusion, schedule, 1024);
745 defer allocator.free(after);
746 try std.testing.expect(!std.mem.eql(u8, before, after));
747 clusters[0].kind = .elementwise;
748 const buffers = try preparation.bufferization.getBufferPlanAnalysis(&context, root);
749 const spaces = try preparation.memory.getMemorySpacePlanAnalysis(&context, root);
750 const layouts = try preparation.layout.getLayoutPlanAnalysis(&context, root);
751 const initial = try preparation.capture.memory(allocator, root, buffers, spaces, layouts, 1024);
752 defer allocator.free(initial);
753 var changed = false;
754 for (@constCast(spaces).assignments.items) |*assignment| {
755 if (assignment.output_source == null) continue;
756 assignment.output_source = .{ .aliased = 0 };
757 changed = true;
758 break;
759 }
760 try std.testing.expect(changed);
761 const updated = try preparation.capture.memory(allocator, root, buffers, spaces, layouts, 1024);
762 defer allocator.free(updated);
763 try std.testing.expect(!std.mem.eql(u8, initial, updated));
764 }
765
766 fn decodeFailure(allocator: std.mem.Allocator, bytes: []const u8) !void {
767 var decoded = try records.codec.decode(allocator, records.memory.Record, .memory, bytes);
768 defer decoded.deinit();
769 try std.testing.expect(decoded.value.buffers.slots.len > 0);
770 }
771
772 test "Accy stage records clean partial decoding and reject framing corruption" {
773 const allocator = std.testing.allocator;
774 var captured = try snapshot();
775 defer captured.deinit();
776 var moving = std.testing.FailingAllocator.init(allocator, .{ .resize_fail_index = 0 });
777 try std.testing.checkAllAllocationFailures(
778 moving.allocator(),
779 decodeFailure,
780 .{captured.memory},
781 );
782 try std.testing.expectError(error.WrongStage, records.codec.decode(
783 allocator,
784 records.dispatch.Record,
785 .dispatch,
786 captured.memory,
787 ));
788 try std.testing.expectError(error.Truncated, records.codec.decode(
789 allocator,
790 records.memory.Record,
791 .memory,
792 captured.memory[0 .. captured.memory.len - 1],
793 ));
794 }
795
796 const publication = @import("../choir/root.zig").publication;
797 const revision = choir.product.revision;
798 const operation = choir.product.operation;
799 const configuration = operation.Configuration{
800 .context = choir.ir.Context.Limits.testing,
801 .register = registerContext,
802 .registration = .{ .name = "accy-plan-test-context", .version = 1 },
803 .codec = .{ .operations = 100, .entities = 1000, .fields = 1000, .depth = 32 },
804 .image = .{ .bytes = 65536, .entities = 1024, .depth = 32 },
805 .roots = 1,
806 .gate_scratch = 64 * 1024 * 1024,
807 .verify = choir.ir.verify.default_options,
808 };
809
810 fn registerContext(context: *choir.ir.Context) !void {
811 try choir.dialects.registerChoirDialect(context);
812 try @import("../choir/root.zig").registerAccyDialect(context);
813 try choir.backends.gpu.registerTargetDialects(context);
814 }
815
816 fn makeProgram(allocator: std.mem.Allocator) !kernel_model.Program {
817 var builder = try kernel_model.Builder.init(allocator, .testing, "retained_program", &.{
818 .{ .buffer = .{ .dtype = .f32, .size = 32 } }, .{ .scalar = .u32 },
819 });
820 errdefer builder.deinit();
821 const axis = try builder.axis("items", 32);
822 const split = try builder.split(axis, 8);
823 try builder.bind(split.outer, .block_x);
824 try builder.bind(split.inner, .thread_x);
825 try builder.vectorize(split.inner, 4);
826 try builder.unroll(split.outer, 2);
827 try builder.return_();
828 return builder.finish();
829 }
830
831 fn captureProgram(allocator: std.mem.Allocator) !records.codec.Decoded(records.program.Record) {
832 var program = try makeProgram(allocator);
833 defer program.deinit();
834 return records.program.capture(allocator, &program, configuration);
835 }
836
837 test "Accy stage records retain complete generated programs after source destruction" {
838 const allocator = std.testing.allocator;
839 var captured = try captureProgram(allocator);
840 defer captured.deinit();
841 const record = captured.value;
842 try std.testing.expectEqual(@as(usize, 2), record.params.len);
843 try std.testing.expectEqual(@as(?u64, 32), record.params[0].buffer.size);
844 try std.testing.expectEqual(.u32, record.params[1].scalar);
845 var context = try choir.ir.Context.init(allocator, configuration.context);
846 defer context.deinit(allocator);
847 try registerContext(&context);
848 var decoded = try choir.bytecode.decodeModule(allocator, &context, record.image);
849 defer decoded.deinit();
850 try choir.ir.verifyOperation(decoded.module, configuration.verify);
851 var references = try records.reference.Index.init(allocator, decoded.module, 1024);
852 defer references.deinit();
853 var operations = decoded.module.getRegion(0).?.getEntryBlock().?.getOperations();
854 const function = operations.next().?;
855 try std.testing.expectEqual(record.function, (try references.operation(function)).ordinal);
856 try std.testing.expectEqualStrings("retained_program", (choir.dialects.FuncDialect.FuncOp{
857 .op = function,
858 }).getName().?);
859 var replayed = try kernel_model.core.schedule.Schedule.init(allocator, record.schedule.replay_limits);
860 defer replayed.deinit(allocator);
861 try replayed.replay(record.schedule);
862 try std.testing.expectEqualDeep(record.launch, try replayed.launch());
863 try std.testing.expectEqualStrings("items", record.schedule.steps[0].axis.name);
864 try std.testing.expectEqual(@as(u32, 2), record.launch.block[0]);
865 try std.testing.expectEqual(@as(u32, 4), record.launch.grid[0]);
866 }
867
868 fn restoreProgram(allocator: std.mem.Allocator, record: records.program.Record) !void {
869 var restored = try records.program.restore(
870 allocator,
871 record,
872 "retained_program",
873 configuration,
874 );
875 defer restored.deinit();
876 }
877
878 test "Accy stage records restore independent arithmetic and complete schedules" {
879 const allocator = std.testing.allocator;
880 var captured = try captureProgram(allocator);
881 var live = true;
882 defer if (live) captured.deinit();
883 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
884 defer references.deinit();
885 const expected = try records.codec.encode(
886 allocator,
887 records.program.Record,
888 .kernel,
889 captured.value,
890 &references,
891 );
892 defer allocator.free(expected);
893 var first = try records.program.restore(
894 allocator,
895 captured.value,
896 "retained_program",
897 configuration,
898 );
899 defer first.deinit();
900 var changed = captured.value;
901 changed.arithmetic.environment_observable = !changed.arithmetic.environment_observable;
902 var second = try records.program.restore(allocator, changed, "retained_program", configuration);
903 defer second.deinit();
904 const second_policy = changed.arithmetic;
905 captured.deinit();
906 live = false;
907 try std.testing.expect(first.kernelModule().context != second.kernelModule().context);
908 try std.testing.expectEqualDeep(second_policy, second.kernelModule().context.arithmetic_policy);
909 var actual = try records.program.capture(allocator, &first, configuration);
910 defer actual.deinit();
911 var wanted = try records.codec.decode(allocator, records.program.Record, .kernel, expected);
912 defer wanted.deinit();
913 try records.codec.compare(actual.value, wanted.value, &references);
914 var alternate = try records.program.capture(allocator, &second, configuration);
915 defer alternate.deinit();
916 wanted.value.arithmetic = second_policy;
917 try records.codec.compare(alternate.value, wanted.value, &references);
918 }
919
920 test "Accy stage records reject invalid restoration and release failed allocations before retry" {
921 const allocator = std.testing.allocator;
922 var captured = try captureProgram(allocator);
923 defer captured.deinit();
924 var changed = captured.value;
925 changed.function = std.math.maxInt(u32);
926 try std.testing.expectError(error.UnboundProductInput, restoreProgram(allocator, changed));
927 changed = captured.value;
928 changed.launch.grid[0] += 1;
929 try std.testing.expectError(error.InvalidStageRecord, restoreProgram(allocator, changed));
930 const constrained = comptime limited: {
931 var result = configuration;
932 result.context.operations.nested_bytes = 1;
933 break :limited result;
934 };
935 try std.testing.expectError(error.WorkExhausted, records.program.restore(
936 allocator,
937 captured.value,
938 "retained_program",
939 constrained,
940 ));
941 try std.testing.checkAllAllocationFailures(allocator, restoreProgram, .{captured.value});
942 try restoreProgram(allocator, captured.value);
943 }
944
945 test "Accy stage records preserve full width schedule axis identities and reject changed replay" {
946 const allocator = std.testing.allocator;
947 const schedule = kernel_model.core.schedule;
948 const limits = schedule.Schedule.Limits{
949 .axes = 300,
950 .steps = 300,
951 .name_bytes = 1024,
952 .axis_ids = 300,
953 };
954 var source = try schedule.Schedule.init(allocator, limits);
955 defer source.deinit(allocator);
956 for (0..300) |_| _ = try source.addAxis("x", 1);
957 var snapshot_value = try schedule.createSnapshot(allocator, .{
958 .axes = 300,
959 .steps = 300,
960 .name_bytes = 1024,
961 }, &source);
962 defer snapshot_value.deinit(allocator);
963 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
964 defer references.deinit();
965 const bytes = try records.codec.encode(
966 allocator,
967 schedule.Record,
968 .kernel,
969 try snapshot_value.record(),
970 &references,
971 );
972 defer allocator.free(bytes);
973 var decoded = try records.codec.decode(allocator, schedule.Record, .kernel, bytes);
974 defer decoded.deinit();
975 @memset(bytes, 0xa5);
976 try std.testing.expectEqual(@as(u32, 299), decoded.value.axes[299].id.index());
977 var replayed = try schedule.Schedule.init(allocator, limits);
978 defer replayed.deinit(allocator);
979 try replayed.replay(decoded.value);
980 try records.codec.compare(decoded.value, try snapshot_value.record(), &references);
981 var changed_axes = try allocator.dupe(schedule.Axis, decoded.value.axes);
982 defer allocator.free(changed_axes);
983 changed_axes[299].extent = 2;
984 var changed = decoded.value;
985 changed.axes = changed_axes;
986 var rejected = try schedule.Schedule.init(allocator, limits);
987 defer rejected.deinit(allocator);
988 try std.testing.expectError(error.InvalidReplay, rejected.replay(changed));
989 try std.testing.expectEqual(@as(usize, 0), rejected.allAxes().len);
990 changed = decoded.value;
991 changed.replay_limits.axes -= 1;
992 try std.testing.expectError(error.InvalidReplay, rejected.replay(changed));
993 try std.testing.expectEqual(@as(usize, 0), rejected.allAxes().len);
994 try rejected.replay(decoded.value);
995 }
996
997 fn captureKernels() ![]u8 {
998 var fixture = try Fixture.init();
999 defer fixture.deinit();
1000 var context = fixture.context();
1001 defer context.deinit();
1002 const root = fixture.source.choir_module;
1003 const outlines = try preparation.outlining.getKernelOutlinePlanAnalysis(&context, root);
1004 const generated = try preparation.kernelization.getKernelizationAnalysis(&context, root);
1005 return preparation.capture.kernel(std.testing.allocator, root, outlines, generated, configuration);
1006 }
1007
1008 test "Accy stage records capture actual kernel outlines and generated programs" {
1009 const allocator = std.testing.allocator;
1010 const bytes = try captureKernels();
1011 defer allocator.free(bytes);
1012 var captured = try records.codec.decode(allocator, records.kernel.Record, .kernel, bytes);
1013 defer captured.deinit();
1014 @memset(bytes, 0xa5);
1015 try std.testing.expect(captured.value.outlines.kernels.len > 0);
1016 try std.testing.expect(captured.value.generated.kernels.len > 0);
1017 for (captured.value.generated.kernels) |item| {
1018 try std.testing.expect(item.program.image.len > 0);
1019 try std.testing.expectEqual(item.argument_count, item.program.params.len);
1020 try std.testing.expectEqual(.generic, std.meta.activeTag(item.body));
1021 try std.testing.expect(item.entry_name.len > 0);
1022 }
1023 }
1024
1025 fn failingProgramCapture(allocator: std.mem.Allocator, program: *const kernel_model.Program) !void {
1026 var captured = try records.program.capture(allocator, program, configuration);
1027 defer captured.deinit();
1028 }
1029
1030 test "Accy stage records qualify program capture and release failed allocations" {
1031 const allocator = std.testing.allocator;
1032 var program = try makeProgram(allocator);
1033 defer program.deinit();
1034 try std.testing.checkAllAllocationFailures(allocator, failingProgramCapture, .{&program});
1035 const metadata: u32 = 7;
1036 program.kernelModule().location = .{ .fused = .{ .locations = &.{.unknown}, .metadata = &metadata } };
1037 try std.testing.expectError(error.UnencodableProduct, failingProgramCapture(allocator, &program));
1038 }
1039
1040 fn seal(
1041 store: *revision.Store,
1042 kind: *const revision.Kind,
1043 root: *choir.ir.Operation,
1044 comptime stage: publication.Stage,
1045 bytes: []const u8,
1046 parent: ?operation.Product,
1047 ) !operation.Product {
1048 const allocator = std.testing.allocator;
1049 const policy = try choir.product.recipe.encode(allocator, .{
1050 .arithmetic = root.getContext().arithmetic_policy,
1051 .policy = "plan-capture-test",
1052 });
1053 defer allocator.free(policy);
1054 const builder = try store.begin(.{
1055 .kind = kind,
1056 .address = .{
1057 .producer = "accy",
1058 .source = "plan-test",
1059 .stage = stage.name(),
1060 .variant = "",
1061 },
1062 .inputs = .{
1063 .compiler_manifest = try choir.product.compiler.manifest(),
1064 .versions = &.{ stage.schema(), operation.schema_identity },
1065 .pipeline = &.{},
1066 .options = &.{},
1067 .policy = policy,
1068 },
1069 .dependencies = if (parent) |source|
1070 &.{.{ .role = "source", .revision = source.revision }}
1071 else
1072 &.{},
1073 .parent = if (parent) |source| source.revision else null,
1074 }, .{
1075 .allowance = revision.WorkVector.uniform(1024 * 1024 * 1024),
1076 .workspace = 128 * 1024 * 1024,
1077 .events = 64,
1078 });
1079 var open = true;
1080 errdefer if (open) {
1081 if (builder.abort(.rejected)) |failure| {
1082 var owned = failure;
1083 owned.deinit();
1084 }
1085 };
1086 try operation.capture(builder, &.{.{ .operation = root }}, bytes, &.{}, configuration);
1087 const product = try operation.Product.seal(builder, kind);
1088 open = false;
1089 return product;
1090 }
1091
1092 const Chain = struct {
1093 store: *revision.Store,
1094 kinds: [7]*const revision.Kind,
1095 products: [7]operation.Product,
1096
1097 fn init(root: *choir.ir.Operation, captured: Snapshot) !Chain {
1098 const allocator = std.testing.allocator;
1099 const store = try revision.Store.create(allocator, .{
1100 .revisions = 10,
1101 .kinds = 7,
1102 .builders = 4,
1103 .compiler_manifests = 2,
1104 .record_bytes = 32 * 1024 * 1024,
1105 .gate_scratch_bytes = configuration.gate_scratch,
1106 .candidate_count = 8,
1107 .screening_bytes = 32 * 1024 * 1024,
1108 });
1109 errdefer store.release();
1110 const stages = [_]publication.Stage{
1111 .semantic, .contract, .tensor, .dispatch, .memory, .kernel, .target,
1112 };
1113 var kinds: [7]*const revision.Kind = undefined;
1114 inline for (stages, 0..) |stage, index| {
1115 kinds[index] = try preparation.publication.registerKind(
1116 allocator,
1117 store,
1118 stage,
1119 configuration,
1120 );
1121 }
1122 store.freeze();
1123 const bytes = [_][]const u8{
1124 &publication.irRecord(.semantic), &publication.irRecord(.contract),
1125 &publication.irRecord(.tensor), captured.dispatch,
1126 captured.memory, captured.kernel,
1127 captured.target,
1128 };
1129 var products: [7]operation.Product = undefined;
1130 var count: usize = 0;
1131 errdefer for (products[0..count]) |product| product.release();
1132 inline for (stages, 0..) |stage, index| {
1133 products[index] = try seal(
1134 store,
1135 kinds[index],
1136 root,
1137 stage,
1138 bytes[index],
1139 if (index == 0) null else products[index - 1],
1140 );
1141 count += 1;
1142 }
1143 return .{ .store = store, .kinds = kinds, .products = products };
1144 }
1145
1146 fn deinit(self: *Chain) void {
1147 for (self.products) |product| product.release();
1148 self.store.release();
1149 }
1150 };
1151
1152 test "sealed Accy stages own typed plans and reject invalid references" {
1153 const allocator = std.testing.allocator;
1154 var fixture = try Fixture.init();
1155 var live = true;
1156 defer if (live) fixture.deinit();
1157 var captured = try captureFixture(&fixture);
1158 defer captured.deinit();
1159 var chain = try Chain.init(fixture.source.choir_module, captured);
1160 defer chain.deinit();
1161 try rejectCorruption(&chain, fixture.source.choir_module, captured);
1162 try rejectKernelCorruption(&chain, fixture.source.choir_module, captured);
1163 try rejectTargetCorruption(&chain, fixture.source.choir_module, captured);
1164 const dispatch = try publication.Module(.dispatch).fromRevision(
1165 allocator,
1166 chain.products[3].revision,
1167 );
1168 defer dispatch.deinit();
1169 const memory = try publication.Module(.memory).fromRevision(
1170 allocator,
1171 chain.products[4].revision,
1172 );
1173 defer memory.deinit();
1174 const kernel = try publication.Module(.kernel).fromRevision(
1175 allocator,
1176 chain.products[5].revision,
1177 );
1178 defer kernel.deinit();
1179 const target = try publication.Module(.target).fromRevision(
1180 allocator,
1181 chain.products[6].revision,
1182 );
1183 defer target.deinit();
1184 fixture.deinit();
1185 live = false;
1186 @memset(captured.dispatch, 0xa5);
1187 @memset(captured.memory, 0xa5);
1188 @memset(captured.kernel, 0xa5);
1189 @memset(captured.target, 0xa5);
1190 var dispatch_plan = try dispatch.plan(allocator, configuration.image);
1191 defer dispatch_plan.deinit();
1192 var memory_plan = try memory.plan(allocator, configuration.image);
1193 defer memory_plan.deinit();
1194 var kernel_plan = try kernel.plan(allocator, configuration.image);
1195 defer kernel_plan.deinit();
1196 var target_plan = try target.plan(allocator, configuration.image);
1197 defer target_plan.deinit();
1198 try std.testing.expect(dispatch_plan.value.schedule.work_items.len > 0);
1199 try std.testing.expect(memory_plan.value.buffers.slots.len > 0);
1200 try std.testing.expect(kernel_plan.value.generated.kernels.len > 0);
1201 try records.kernel.validate(allocator, kernel_plan.value, memory_plan.value);
1202 try std.testing.expectEqual(null, target_plan.value.profile);
1203 try std.testing.expect(target_plan.value.kernels.len > 0);
1204 try std.testing.expectEqual(null, target_plan.value.kernels[0].abi);
1205 }
1206
1207 const TargetCorruption = enum { work, count, profile, function, math, arguments, scalars };
1208
1209 fn corruptTarget(value: *records.target.Record, corruption: TargetCorruption) anyerror {
1210 const kernel = &@constCast(value.kernels)[0];
1211 switch (corruption) {
1212 .work => kernel.lowered.work_item_id = 99999,
1213 .count => value.kernels = value.kernels[0..0],
1214 .profile => value.profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object },
1215 .function => {
1216 kernel.lowered.program.function = 99999;
1217 return error.UnboundProductInput;
1218 },
1219 .math => value.math_tier = .tf32_tensor,
1220 .arguments => kernel.lowered.argument_count += 1,
1221 .scalars => kernel.runtime_scalar_argument_count = kernel.lowered.argument_count + 1,
1222 }
1223 return error.InvalidStageRecord;
1224 }
1225
1226 fn rejectTargetCorruption(chain: *Chain, root: *choir.ir.Operation, captured: Snapshot) !void {
1227 const allocator = std.testing.allocator;
1228 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
1229 defer references.deinit();
1230 for (std.enums.values(TargetCorruption)) |corruption| {
1231 var decoded = try records.codec.decode(
1232 allocator,
1233 records.target.Record,
1234 .target,
1235 captured.target,
1236 );
1237 defer decoded.deinit();
1238 const expected = corruptTarget(&decoded.value, corruption);
1239 const bytes = try records.codec.encode(
1240 allocator,
1241 records.target.Record,
1242 .target,
1243 decoded.value,
1244 &references,
1245 );
1246 defer allocator.free(bytes);
1247 try expectRejected(expected, seal(
1248 chain.store,
1249 chain.kinds[6],
1250 root,
1251 .target,
1252 bytes,
1253 chain.products[5],
1254 ));
1255 }
1256 try expectRejected(error.InvalidStageDependency, seal(
1257 chain.store,
1258 chain.kinds[6],
1259 root,
1260 .target,
1261 captured.target,
1262 chain.products[4],
1263 ));
1264 }
1265
1266 const KernelCorruption = enum {
1267 outline_slot,
1268 outline_root,
1269 program_function,
1270 program_nonfunction,
1271 program_parameter,
1272 program_entry,
1273 program_launch,
1274 schedule_axis,
1275 counters,
1276 argument_count,
1277 work,
1278 };
1279
1280 fn corruptKernel(value: *records.kernel.Record, corruption: KernelCorruption) anyerror {
1281 const outline = &@constCast(value.outlines.kernels)[0];
1282 const kernel = &@constCast(value.generated.kernels)[0];
1283 switch (corruption) {
1284 .outline_slot => outline.output_slot_id = 99999,
1285 .outline_root => {
1286 outline.root.ordinal = 99999;
1287 return error.UnboundProductInput;
1288 },
1289 .program_function => {
1290 kernel.program.function = 99999;
1291 return error.UnboundProductInput;
1292 },
1293 .program_nonfunction => kernel.program.function = 0,
1294 .program_parameter => @constCast(kernel.program.params)[0] = .{ .scalar = .u32 },
1295 .program_entry => kernel.entry_name = "wrong_entry",
1296 .program_launch => kernel.program.launch.block[0] += 1,
1297 .schedule_axis => {
1298 @constCast(kernel.program.schedule.axes)[0].extent = 999;
1299 return error.InvalidReplay;
1300 },
1301 .counters => value.outlines.total_input_slots += 1,
1302 .argument_count => kernel.argument_count += 1,
1303 .work => kernel.work_item_id = 99999,
1304 }
1305 return error.InvalidStageRecord;
1306 }
1307
1308 fn rejectKernelCorruption(chain: *Chain, root: *choir.ir.Operation, captured: Snapshot) !void {
1309 const allocator = std.testing.allocator;
1310 var references = try records.reference.Index.init(allocator, root, 1024);
1311 defer references.deinit();
1312 for (std.enums.values(KernelCorruption)) |corruption| {
1313 var decoded = try records.codec.decode(
1314 allocator,
1315 records.kernel.Record,
1316 .kernel,
1317 captured.kernel,
1318 );
1319 defer decoded.deinit();
1320 const expected = corruptKernel(&decoded.value, corruption);
1321 const bytes = try records.codec.encode(
1322 allocator,
1323 records.kernel.Record,
1324 .kernel,
1325 decoded.value,
1326 &references,
1327 );
1328 defer allocator.free(bytes);
1329 try expectRejected(expected, seal(
1330 chain.store,
1331 chain.kinds[5],
1332 root,
1333 .kernel,
1334 bytes,
1335 chain.products[4],
1336 ));
1337 }
1338 try expectRejected(error.InvalidStageDependency, seal(
1339 chain.store,
1340 chain.kinds[5],
1341 root,
1342 .kernel,
1343 captured.kernel,
1344 chain.products[3],
1345 ));
1346 }
1347
1348 test "sealed Accy Kernel validates multiple program images within the same gate scratch" {
1349 const allocator = std.testing.allocator;
1350 var fixture = try Fixture.initNames(&.{ "first", "second" });
1351 var live = true;
1352 defer if (live) fixture.deinit();
1353 var captured = try captureFixture(&fixture);
1354 defer captured.deinit();
1355 var chain = try Chain.init(fixture.source.choir_module, captured);
1356 defer chain.deinit();
1357 fixture.deinit();
1358 live = false;
1359 @memset(captured.kernel, 0xa5);
1360 const kernel = try publication.Module(.kernel).fromRevision(
1361 allocator,
1362 chain.products[5].revision,
1363 );
1364 defer kernel.deinit();
1365 var plan = try kernel.plan(allocator, configuration.image);
1366 defer plan.deinit();
1367 try std.testing.expectEqual(@as(usize, 2), plan.value.generated.kernels.len);
1368 try std.testing.expect(!std.mem.eql(
1369 u8,
1370 plan.value.generated.kernels[0].entry_name,
1371 plan.value.generated.kernels[1].entry_name,
1372 ));
1373 }
1374
1375 fn targetVariant(
1376 chain: *Chain,
1377 fixture: *Fixture,
1378 profile: preparation.BackendTargetProfile,
1379 ) !operation.Product {
1380 const allocator = std.testing.allocator;
1381 const root = try fixture.source.choir_module.clone();
1382 defer root.erase();
1383 try preparation.setBackendTargetProfile(root.context, root, profile);
1384 try preparation.target.setGeneratedScanSchedules(allocator, root.context, root, &.{.{
1385 .total = 4,
1386 .schedule = .{ .threads = 32, .items = 1 },
1387 }});
1388 try preparation.target.setGeneratedRowPipelineSchedules(allocator, root.context, root, &.{.{
1389 .shape = .{ .rows = 1, .cols = 4 },
1390 .schedule = .{ .threads = 32 },
1391 }});
1392 var context = fixture.context();
1393 defer context.deinit();
1394 const source = fixture.source.choir_module;
1395 const schedule = try preparation.schedule.getSchedulePlanAnalysis(&context, source);
1396 const generated = try preparation.kernelization.getKernelizationAnalysis(&context, source);
1397 const bytes = try preparation.capture.target(
1398 allocator,
1399 root,
1400 schedule,
1401 generated.kernels.items,
1402 configuration,
1403 );
1404 defer allocator.free(bytes);
1405 return seal(chain.store, chain.kinds[6], root, .target, bytes, chain.products[5]);
1406 }
1407
1408 test "sealed Accy Target owns sibling target contracts after source destruction" {
1409 const allocator = std.testing.allocator;
1410 var fixture = try Fixture.init();
1411 var live = true;
1412 defer if (live) fixture.deinit();
1413 var captured = try captureFixture(&fixture);
1414 defer captured.deinit();
1415 var chain = try Chain.init(fixture.source.choir_module, captured);
1416 defer chain.deinit();
1417 const cpu_product = try targetVariant(&chain, &fixture, .{
1418 .backend_kind = .cpu,
1419 .artifact_format = .cpu_object,
1420 });
1421 defer cpu_product.release();
1422 const gpu_product = try targetVariant(&chain, &fixture, .{
1423 .backend_kind = .cuda,
1424 .artifact_format = .cuda_ptx,
1425 });
1426 defer gpu_product.release();
1427 fixture.deinit();
1428 live = false;
1429 @memset(captured.target, 0xa5);
1430 const cpu = try publication.Module(.target).fromRevision(allocator, cpu_product.revision);
1431 defer cpu.deinit();
1432 const accelerated = try publication.Module(.target).fromRevision(allocator, gpu_product.revision);
1433 defer accelerated.deinit();
1434 try std.testing.expect(!cpu.eql(accelerated));
1435 var cpu_plan = try cpu.plan(allocator, configuration.image);
1436 defer cpu_plan.deinit();
1437 var gpu_plan = try accelerated.plan(allocator, configuration.image);
1438 defer gpu_plan.deinit();
1439 const host = cpu_plan.value.kernels[0];
1440 const device = gpu_plan.value.kernels[0];
1441 try std.testing.expectEqual(host.lowered.argument_count + 7, host.abi.?.argument_count);
1442 try std.testing.expectEqual(@as(usize, 7), host.abi.?.static_arguments.len);
1443 try std.testing.expectEqual(@as(u32, 4), host.abi.?.static_arguments[0].u32);
1444 try std.testing.expectEqual(@as(?u32, 4), host.abi.?.compile_options.cpu_vector_width);
1445 try std.testing.expectEqual(device.lowered.argument_count, device.abi.?.argument_count);
1446 try std.testing.expectEqual(@as(usize, 0), device.abi.?.static_arguments.len);
1447 try std.testing.expectEqual(@as(?u32, null), device.abi.?.compile_options.cpu_vector_width);
1448 try std.testing.expectEqualStrings("4=32x1", cpu_plan.value.generated_scan_schedules.?);
1449 try std.testing.expectEqualStrings(
1450 "1x4=32",
1451 cpu_plan.value.generated_row_pipeline_schedules.?,
1452 );
1453 const expected_dtypes = gpu.DTypeSet.init(&.{ .f32, .i32 });
1454 try std.testing.expectEqual(expected_dtypes.bits, host.required_dtype_bits);
1455 try std.testing.expectEqual(.exact, cpu_plan.value.math_tier);
1456 try std.testing.expectEqual(@as(u32, 0), host.runtime_scalar_argument_count);
1457 }
1458
1459 test "Accy stage records capture nonempty target requirements from program operations" {
1460 const allocator = std.testing.allocator;
1461 var fixture = try Fixture.init();
1462 defer fixture.deinit();
1463 var context = fixture.context();
1464 defer context.deinit();
1465 const root = fixture.source.choir_module;
1466 const schedule = try preparation.schedule.getSchedulePlanAnalysis(&context, root);
1467 const generated = try preparation.kernelization.getKernelizationAnalysis(&context, root);
1468 const kernel = &generated.kernels.items[0];
1469 const program_context = kernel.program.kernelModule().context;
1470 const block = kernel.program.storage.kernel.func().getEntryBlock();
1471 const terminator: *choir.ir.Operation = @ptrCast(@alignCast(block.operations.tail.?));
1472 const Gpu = choir.dialects.gpu.GpuDialect;
1473 const active = try Gpu.ActiveMaskOp.create(program_context, .unknown);
1474 try block.insertBefore(active.op, terminator);
1475 const commit = try Gpu.CpAsyncCommitOp.create(program_context, .unknown);
1476 try block.insertBefore(commit.op, terminator);
1477 kernel.body_fingerprint = try kernel.program.bodyFingerprint(allocator);
1478 const bytes = try preparation.capture.target(
1479 allocator,
1480 root,
1481 schedule,
1482 generated.kernels.items,
1483 configuration,
1484 );
1485 defer allocator.free(bytes);
1486 var decoded = try records.codec.decode(allocator, records.target.Record, .target, bytes);
1487 defer decoded.deinit();
1488 try std.testing.expect(decoded.value.kernels[0].required_features.tensor_cores);
1489 try std.testing.expect(decoded.value.kernels[0].required_subgroup.supported);
1490 try std.testing.expect(decoded.value.kernels[0].required_subgroup.ballot);
1491 }
1492
1493 test "Accy stage records preserve floating ABI argument bits" {
1494 const allocator = std.testing.allocator;
1495 const T = struct { arguments: []const choir_abi.ScalarArgument };
1496 const source = T{ .arguments = &.{
1497 .{ .f32 = @bitCast(@as(u32, 0x7fc00043)) },
1498 .{ .f64 = @bitCast(@as(u64, 0x7ff8000000000012)) },
1499 .{ .f64 = @bitCast(@as(u64, 0x8000000000000000)) },
1500 } };
1501 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
1502 defer references.deinit();
1503 const bytes = try records.codec.encode(allocator, T, .target, source, &references);
1504 defer allocator.free(bytes);
1505 var decoded = try records.codec.decode(allocator, T, .target, bytes);
1506 defer decoded.deinit();
1507 try records.codec.compare(decoded.value, source, &references);
1508 const first_bits: u32 = @bitCast(decoded.value.arguments[0].f32);
1509 try std.testing.expectEqual(@as(u32, 0x7fc00043), first_bits);
1510 try std.testing.expectEqual(
1511 @as(u64, 0x8000000000000000),
1512 @as(u64, @bitCast(decoded.value.arguments[2].f64)),
1513 );
1514 @constCast(decoded.value.arguments)[1].f64 = @bitCast(@as(u64, 0x7ff8000000000013));
1515 try std.testing.expectError(error.UnencodableProduct, records.codec.compare(
1516 decoded.value,
1517 source,
1518 &references,
1519 ));
1520 }
1521
1522 fn rejectCorruption(chain: *Chain, root: *choir.ir.Operation, captured: Snapshot) !void {
1523 const allocator = std.testing.allocator;
1524 var dispatch = try records.codec.decode(
1525 allocator,
1526 records.dispatch.Record,
1527 .dispatch,
1528 captured.dispatch,
1529 );
1530 defer dispatch.deinit();
1531 @constCast(dispatch.value.schedule.work_items)[0].root.ordinal = 99999;
1532 var references = try records.reference.Index.init(allocator, root, 1024);
1533 defer references.deinit();
1534 const bad_dispatch = try records.codec.encode(
1535 allocator,
1536 records.dispatch.Record,
1537 .dispatch,
1538 dispatch.value,
1539 &references,
1540 );
1541 defer allocator.free(bad_dispatch);
1542 try expectRejected(error.UnboundProductInput, seal(
1543 chain.store,
1544 chain.kinds[3],
1545 root,
1546 .dispatch,
1547 bad_dispatch,
1548 chain.products[2],
1549 ));
1550 var memory = try records.codec.decode(
1551 allocator,
1552 records.memory.Record,
1553 .memory,
1554 captured.memory,
1555 );
1556 defer memory.deinit();
1557 @constCast(memory.value.spaces.assignments)[0].output_source = .{ .aliased = 99999 };
1558 const bad_memory = try records.codec.encode(
1559 allocator,
1560 records.memory.Record,
1561 .memory,
1562 memory.value,
1563 &references,
1564 );
1565 defer allocator.free(bad_memory);
1566 try expectRejected(error.InvalidStageRecord, seal(
1567 chain.store,
1568 chain.kinds[4],
1569 root,
1570 .memory,
1571 bad_memory,
1572 chain.products[3],
1573 ));
1574 }
1575
1576 test "Accy stage records bound reference indexing and reject stale analysis maps" {
1577 const allocator = std.testing.allocator;
1578 var fixture = try Fixture.init();
1579 defer fixture.deinit();
1580 var foreign = try Fixture.init();
1581 defer foreign.deinit();
1582 var index = try records.reference.Index.init(allocator, fixture.source.choir_module, 1024);
1583 defer index.deinit();
1584 try std.testing.expectError(
1585 error.UnboundProductInput,
1586 index.operation(foreign.source.choir_module),
1587 );
1588 const bound: u32 = @intCast(@max(index.operations.count(), index.values.count()));
1589 var exact = try records.reference.Index.init(allocator, fixture.source.choir_module, bound);
1590 defer exact.deinit();
1591 try std.testing.expectError(
1592 error.RecordLimit,
1593 records.reference.Index.init(allocator, fixture.source.choir_module, bound - 1),
1594 );
1595 var context = fixture.context();
1596 defer context.deinit();
1597 const root = fixture.source.choir_module;
1598 const fusion = try preparation.fusion.getFusionPlanAnalysis(&context, root);
1599 const schedule = try preparation.schedule.getSchedulePlanAnalysis(&context, root);
1600 @constCast(schedule).root_to_item.clearRetainingCapacity();
1601 try std.testing.expectError(error.UnencodableProduct, preparation.capture.dispatch(
1602 allocator,
1603 root,
1604 fusion,
1605 schedule,
1606 1024,
1607 ));
1608 }
1609
1610 fn expectRejected(expected: anyerror, result: anyerror!operation.Product) !void {
1611 const product = result catch |err| {
1612 try std.testing.expectEqual(expected, err);
1613 return;
1614 };
1615 product.release();
1616 return error.TestExpectedError;
1617 }
1618
1619 fn captureFailure(
1620 allocator: std.mem.Allocator,
1621 root: *choir.ir.Operation,
1622 buffers: *const preparation.bufferization.BufferPlanAnalysis,
1623 spaces: *const preparation.memory.MemorySpacePlanAnalysis,
1624 layouts: *const preparation.layout.LayoutPlanAnalysis,
1625 ) !void {
1626 const bytes = try preparation.capture.memory(allocator, root, buffers, spaces, layouts, 1024);
1627 defer allocator.free(bytes);
1628 try std.testing.expect(bytes.len > 5);
1629 }
1630
1631 test "Accy stage records clean partial capture allocations" {
1632 const allocator = std.testing.allocator;
1633 var fixture = try Fixture.init();
1634 defer fixture.deinit();
1635 var context = fixture.context();
1636 defer context.deinit();
1637 const root = fixture.source.choir_module;
1638 const buffers = try preparation.bufferization.getBufferPlanAnalysis(&context, root);
1639 const spaces = try preparation.memory.getMemorySpacePlanAnalysis(&context, root);
1640 const layouts = try preparation.layout.getLayoutPlanAnalysis(&context, root);
1641 try std.testing.checkAllAllocationFailures(
1642 allocator,
1643 captureFailure,
1644 .{ root, buffers, spaces, layouts },
1645 );
1646 }
1647
1648 test "Accy publication recipes match the real ordered stage pipelines" {
1649 const recipe = preparation.recipe;
1650 const options = preparation.BackendPreparationRunOptions{};
1651 inline for (comptime std.meta.tags(publication.Stage)) |stage| {
1652 var manager = choir.passes.PassManager.init(std.testing.allocator);
1653 defer manager.deinit();
1654 try recipe.configure(&manager, stage, &options);
1655 try std.testing.expect(manager.verifier_enabled);
1656 try std.testing.expectEqualDeep(recipe.policy.verification, manager.verifier_options);
1657 try std.testing.expectEqual(@as(usize, 1), recipe.runOptions().max_threads);
1658 try std.testing.expectEqual(null, recipe.runOptions().worker_allocator);
1659 const identities = recipe.pipeline(stage);
1660 try std.testing.expectEqual(identities.len, manager.root.pipeline.items.len);
1661 for (identities, manager.root.pipeline.items) |identity, entry| {
1662 try std.testing.expectEqualStrings(identity.name, entry.pass.name);
1663 try std.testing.expectEqual(@as(u32, 1), identity.version);
1664 }
1665 }
1666 }
1667
1668 fn recipeBytes(
1669 fixture: *Fixture,
1670 comptime stage: publication.Stage,
1671 options: preparation.BackendPreparationRunOptions,
1672 ) ![]u8 {
1673 return preparation.recipe.encode(
1674 std.testing.allocator,
1675 stage,
1676 fixture.source.choir_module,
1677 options,
1678 );
1679 }
1680
1681 test "Accy publication recipes record every tensor choice and default" {
1682 const allocator = std.testing.allocator;
1683 var fixture = try Fixture.init();
1684 defer fixture.deinit();
1685 const original = try recipeBytes(&fixture, .tensor, .{});
1686 defer allocator.free(original);
1687 var decoded = try records.codec.decode(
1688 allocator,
1689 preparation.recipe.Record(.tensor),
1690 .tensor,
1691 original,
1692 );
1693 defer decoded.deinit();
1694 var references = records.reference.Index{ .allocator = allocator, .limit = 0 };
1695 defer references.deinit();
1696 const defaults = preparation.TensorLoweringOptions{};
1697 try records.codec.compare(decoded.value.options, defaults, &references);
1698 try std.testing.expectEqualDeep(preparation.recipe.policy, decoded.value.execution);
1699 for (0..13) |choice| {
1700 var options = preparation.BackendPreparationRunOptions{};
1701 switch (choice) {
1702 0 => options.tensor.activation.kernel_library = .enabled,
1703 1 => options.tensor.einsum.strategy = .greedy,
1704 2 => options.tensor.einsum.exact_state_limit += 1,
1705 3 => options.tensor.einsum.beam_width += 1,
1706 4 => options.tensor.einsum.auto_beam_width += 1,
1707 5 => options.tensor.einsum.kernel_library = .enabled,
1708 6 => options.tensor.einsum.matrix_product_schedule = .{
1709 .thread_blocks = .{ .x = 8, .y = 4 },
1710 },
1711 7 => options.tensor.indexing.kernel_library = .enabled,
1712 8 => options.tensor.indexing.gather_schedule = .{ .thread_blocks = 32 },
1713 9 => options.tensor.indexing.scatter_schedule = .{ .thread_blocks = 64 },
1714 10 => options.tensor.indexing.scatter_add_schedule = .{ .shared_bins = 128 },
1715 11 => options.tensor.loss.kernel_library = .enabled,
1716 12 => options.tensor.loss.row_sparse_cross_entropy_schedule = .{
1717 .thread_blocks = 256,
1718 },
1719 else => unreachable,
1720 }
1721 const changed = try recipeBytes(&fixture, .tensor, options);
1722 defer allocator.free(changed);
1723 try std.testing.expect(!std.mem.eql(u8, original, changed));
1724 }
1725 }
1726
1727 test "Accy publication recipes preserve inherited targets and explicit overrides" {
1728 const allocator = std.testing.allocator;
1729 var fixture = try Fixture.init();
1730 defer fixture.deinit();
1731 const absent = try recipeBytes(&fixture, .kernel, .{});
1732 defer allocator.free(absent);
1733 const options = preparation.BackendPreparationRunOptions{
1734 .target_profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object },
1735 .generated_scan_schedules = &.{.{ .schedule = .{ .threads = 32, .items = 2 } }},
1736 .generated_row_pipeline_schedules = &.{.{ .schedule = .{ .threads = 64 } }},
1737 };
1738 const root = fixture.source.choir_module;
1739 const explicit = try recipeBytes(&fixture, .kernel, options);
1740 defer allocator.free(explicit);
1741 const unchanged = try recipeBytes(&fixture, .kernel, .{});
1742 defer allocator.free(unchanged);
1743 try std.testing.expectEqualStrings(absent, unchanged);
1744 try preparation.recipe.applyTargetOptions(allocator, root, options);
1745 const applied = try recipeBytes(&fixture, .kernel, options);
1746 defer allocator.free(applied);
1747 try std.testing.expectEqualStrings(explicit, applied);
1748 try preparation.recipe.applyTargetOptions(allocator, root, .{});
1749 const inherited = try recipeBytes(&fixture, .kernel, .{});
1750 defer allocator.free(inherited);
1751 try std.testing.expectEqualStrings(explicit, inherited);
1752 try std.testing.expect(!std.mem.eql(u8, absent, inherited));
1753 var decoded = try records.codec.decode(
1754 allocator,
1755 preparation.recipe.Record(.kernel),
1756 .kernel,
1757 inherited,
1758 );
1759 defer decoded.deinit();
1760 try std.testing.expectEqual(.cpu_object, decoded.value.options.profile.?.artifact_format);
1761 try std.testing.expectEqualStrings("32x2", decoded.value.options.generated_scan_schedules.?);
1762 try std.testing.expectEqualStrings(
1763 "64",
1764 decoded.value.options.generated_row_pipeline_schedules.?,
1765 );
1766 try preparation.recipe.applyTargetOptions(allocator, root, .{
1767 .target_profile = .{ .backend_kind = .cuda, .artifact_format = .cuda_ptx },
1768 });
1769 const overridden = try recipeBytes(&fixture, .kernel, .{});
1770 defer allocator.free(overridden);
1771 try std.testing.expect(!std.mem.eql(u8, inherited, overridden));
1772 }
1773
1774 test "Accy publication recipes exclude diagnostics" {
1775 const allocator = std.testing.allocator;
1776 var fixture = try Fixture.init();
1777 defer fixture.deinit();
1778 const original = try recipeBytes(&fixture, .tensor, .{});
1779 defer allocator.free(original);
1780 var failure = preparation.BackendPreparationFailure{};
1781 defer failure.deinit(allocator);
1782 var timing = preparation.BackendPreparationTiming.init(allocator);
1783 defer timing.deinit();
1784 const diagnostic = try recipeBytes(&fixture, .tensor, .{
1785 .failure = &failure,
1786 .timing = &timing,
1787 .now = struct {
1788 fn now() i128 {
1789 return 42;
1790 }
1791 }.now,
1792 });
1793 defer allocator.free(diagnostic);
1794 try std.testing.expectEqualStrings(original, diagnostic);
1795 }
1796
1797 const TuningRecipeCase = struct {
1798 const library = @import("../kernel/library/root.zig");
1799 const tuning = library.tuning;
1800 const device: u64 = 912;
1801
1802 arena: std.heap.ArenaAllocator,
1803 records: []tuning.FamilyTuningRecord,
1804
1805 fn matrix() library.linalg.MatrixProduct {
1806 var instance: library.linalg.MatrixProduct = .{ .m = 64, .n = 32, .k = 16 };
1807 const candidates = library.linalg.matrixProductThreadCandidatesForExtents(64, 32);
1808 instance.threads = candidates.slice()[candidates.slice().len - 1];
1809 return instance;
1810 }
1811
1812 fn gather() library.indexing.Gather {
1813 var instance: library.indexing.Gather = .{ .axis_size = 64, .gathered = 32 };
1814 const candidates = library.indexing.gatherThreadCandidatesForTotal(instance.total());
1815 instance.threads = candidates.slice()[candidates.slice().len - 1];
1816 return instance;
1817 }
1818
1819 fn init() !TuningRecipeCase {
1820 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
1821 errdefer arena.deinit();
1822 const allocator = arena.allocator();
1823 const items = try allocator.alloc(tuning.FamilyTuningRecord, 2);
1824 items[0] = .{
1825 .key = try library.linalg.matrixProductFamilyTuningKey(allocator, device, matrix()),
1826 .target = try library.linalg.matrixProductFamilyTarget(allocator, matrix()),
1827 .winner_median_ns = 7,
1828 .runner_up_median_ns = 11,
1829 .sample_count = 3,
1830 };
1831 items[1] = .{
1832 .key = try library.indexing.gatherFamilyTuningKey(allocator, device, gather()),
1833 .target = try library.indexing.gatherFamilyTarget(allocator, gather()),
1834 .winner_median_ns = 13,
1835 .runner_up_median_ns = 17,
1836 .sample_count = 5,
1837 };
1838 return .{ .arena = arena, .records = items };
1839 }
1840
1841 fn reader(self: TuningRecipeCase) tuning.FamilyTuningReader {
1842 return .{ .device_fingerprint = device, .table = .{ .records = self.records } };
1843 }
1844
1845 fn options(reader_: *const tuning.FamilyTuningReader) preparation.BackendPreparationRunOptions {
1846 return .{ .tensor = .{
1847 .einsum = .{ .kernel_library = .enabled, .family_tuning = reader_ },
1848 .indexing = .{ .kernel_library = .enabled, .family_tuning = reader_ },
1849 } };
1850 }
1851
1852 fn deinit(self: *TuningRecipeCase) void {
1853 self.arena.deinit();
1854 self.* = undefined;
1855 }
1856 };
1857
1858 fn independentTuningRecipe() ![]u8 {
1859 var fixture = try Fixture.init();
1860 defer fixture.deinit();
1861 var source = try TuningRecipeCase.init();
1862 defer source.deinit();
1863 var reader = source.reader();
1864 const bytes = try recipeBytes(&fixture, .tensor, TuningRecipeCase.options(&reader));
1865 reader.device_fingerprint = 0;
1866 for (source.records) |*record| {
1867 @memset(@constCast(record.target), 'x');
1868 record.key.extents = @splat(0);
1869 }
1870 return bytes;
1871 }
1872
1873 test "Accy publication recipes own tuning hits and misses after source destruction" {
1874 const allocator = std.testing.allocator;
1875 const library = TuningRecipeCase.library;
1876 const bytes = try independentTuningRecipe();
1877 defer allocator.free(bytes);
1878 var decoded = try records.codec.decode(
1879 allocator,
1880 preparation.recipe.Record(.tensor),
1881 .tensor,
1882 bytes,
1883 );
1884 defer decoded.deinit();
1885 try std.testing.expect(decoded.value.options.einsum.family_tuning != null);
1886 try std.testing.expect(decoded.value.options.indexing.family_tuning != null);
1887 const matrix_reader = decoded.value.options.einsum.family_tuning.?;
1888 const gather_reader = decoded.value.options.indexing.family_tuning.?;
1889 try std.testing.expectEqual(TuningRecipeCase.device, matrix_reader.device_fingerprint);
1890 try std.testing.expectEqual(@as(usize, 2), matrix_reader.table.records.len);
1891 try std.testing.expectEqual(@as(u64, 7), matrix_reader.table.records[0].winner_median_ns);
1892 try std.testing.expectEqualDeep(matrix_reader, gather_reader);
1893 const matrix = TuningRecipeCase.matrix();
1894 const gather = TuningRecipeCase.gather();
1895 try std.testing.expectEqual(matrix.threads, (try library.linalg.resolveMatrixProductSchedule(
1896 allocator,
1897 matrix_reader,
1898 matrix,
1899 )).?);
1900 try std.testing.expectEqual(gather.threads, (try library.indexing.resolveGatherSchedule(
1901 allocator,
1902 gather_reader,
1903 gather,
1904 )).?);
1905 var absent_matrix = matrix;
1906 absent_matrix.k += 1;
1907 var absent_gather = gather;
1908 absent_gather.axis_size += 1;
1909 try std.testing.expectEqual(null, try library.linalg.resolveMatrixProductSchedule(
1910 allocator,
1911 matrix_reader,
1912 absent_matrix,
1913 ));
1914 try std.testing.expectEqual(null, try library.indexing.resolveGatherSchedule(
1915 allocator,
1916 gather_reader,
1917 absent_gather,
1918 ));
1919 }
1920
1921 fn tuningRecipeDiffers(fixture: *Fixture, baseline: []const u8, reader: anytype) !void {
1922 const changed = try recipeBytes(fixture, .tensor, TuningRecipeCase.options(reader));
1923 defer std.testing.allocator.free(changed);
1924 try std.testing.expect(!std.mem.eql(u8, baseline, changed));
1925 }
1926
1927 test "Accy publication recipes compare tuning values independent of pointer identity" {
1928 const allocator = std.testing.allocator;
1929 var fixture = try Fixture.init();
1930 defer fixture.deinit();
1931 var source = try TuningRecipeCase.init();
1932 defer source.deinit();
1933 var reader = source.reader();
1934 const original = try recipeBytes(&fixture, .tensor, TuningRecipeCase.options(&reader));
1935 defer allocator.free(original);
1936 var relocated = try TuningRecipeCase.init();
1937 defer relocated.deinit();
1938 const other = relocated.reader();
1939 const same = try recipeBytes(&fixture, .tensor, TuningRecipeCase.options(&other));
1940 defer allocator.free(same);
1941 try std.testing.expectEqualSlices(u8, original, same);
1942 reader.device_fingerprint += 1;
1943 try tuningRecipeDiffers(&fixture, original, &reader);
1944 reader = source.reader();
1945 for (0..5) |field| {
1946 const saved = source.records[0];
1947 switch (field) {
1948 0 => source.records[0].key.extents[0] += 1,
1949 1 => source.records[0].key.dtype = .f16,
1950 2 => source.records[0].target = "absent-target",
1951 3 => source.records[0].sample_count += 1,
1952 4 => source.records[0].key.family_version += 1,
1953 else => unreachable,
1954 }
1955 try tuningRecipeDiffers(&fixture, original, &reader);
1956 source.records[0] = saved;
1957 }
1958 std.mem.swap(
1959 TuningRecipeCase.tuning.FamilyTuningRecord,
1960 &source.records[0],
1961 &source.records[1],
1962 );
1963 try tuningRecipeDiffers(&fixture, original, &reader);
1964 }
1965
1966 test "Accy publication recipes distinguish absent and empty tuning providers" {
1967 const linalg = TuningRecipeCase.library.linalg;
1968 const allocator = std.testing.allocator;
1969 var fixture = try Fixture.init();
1970 defer fixture.deinit();
1971 const absent = try recipeBytes(&fixture, .tensor, .{});
1972 defer allocator.free(absent);
1973 var reader: TuningRecipeCase.tuning.FamilyTuningReader = .{
1974 .device_fingerprint = TuningRecipeCase.device,
1975 .table = .{},
1976 };
1977 var options = preparation.BackendPreparationRunOptions{};
1978 for (0..2) |kind| {
1979 switch (kind) {
1980 0 => options.tensor.einsum.family_tuning = &reader,
1981 else => options.tensor.indexing.family_tuning = &reader,
1982 }
1983 const bytes = try recipeBytes(&fixture, .tensor, options);
1984 defer allocator.free(bytes);
1985 try std.testing.expect(!std.mem.eql(u8, absent, bytes));
1986 var decoded = try records.codec.decode(
1987 allocator,
1988 preparation.recipe.Record(.tensor),
1989 .tensor,
1990 bytes,
1991 );
1992 defer decoded.deinit();
1993 const present = switch (kind) {
1994 0 => decoded.value.options.einsum.family_tuning,
1995 else => decoded.value.options.indexing.family_tuning,
1996 };
1997 try std.testing.expect(present != null);
1998 const captured = present.?;
1999 try std.testing.expectEqual(@as(usize, 0), captured.table.records.len);
2000 try std.testing.expectEqual(null, try linalg.resolveMatrixProductSchedule(
2001 allocator,
2002 captured,
2003 TuningRecipeCase.matrix(),
2004 ));
2005 options = .{};
2006 }
2007 }
2008
2009 fn tuningRecipeFailure(
2010 allocator: std.mem.Allocator,
2011 root: *choir.ir.Operation,
2012 reader: *const TuningRecipeCase.tuning.FamilyTuningReader,
2013 ) !void {
2014 const bytes = try preparation.recipe.encode(
2015 allocator,
2016 .tensor,
2017 root,
2018 TuningRecipeCase.options(reader),
2019 );
2020 defer allocator.free(bytes);
2021 }
2022
2023 test "Accy publication recipes clean partial tuning capture allocations" {
2024 var fixture = try Fixture.init();
2025 defer fixture.deinit();
2026 var source = try TuningRecipeCase.init();
2027 defer source.deinit();
2028 const reader = source.reader();
2029 try std.testing.checkAllAllocationFailures(std.testing.allocator, tuningRecipeFailure, .{
2030 fixture.source.choir_module, &reader,
2031 });
2032 }
2033
2034 test "Accy publication recipes preserve first-match tuning record order" {
2035 const allocator = std.testing.allocator;
2036 var fixture = try Fixture.init();
2037 defer fixture.deinit();
2038 var source = try TuningRecipeCase.init();
2039 defer source.deinit();
2040 source.records[1] = source.records[0];
2041 source.records[0].target = "stale-target";
2042 const reader = source.reader();
2043 var first: ?[]u8 = null;
2044 defer if (first) |bytes| allocator.free(bytes);
2045 for (0..2) |order| {
2046 const bytes = try recipeBytes(&fixture, .tensor, TuningRecipeCase.options(&reader));
2047 defer if (order != 0) allocator.free(bytes);
2048 if (order == 0) first = bytes else try std.testing.expect(!std.mem.eql(u8, first.?, bytes));
2049 var decoded = try records.codec.decode(
2050 allocator,
2051 preparation.recipe.Record(.tensor),
2052 .tensor,
2053 bytes,
2054 );
2055 defer decoded.deinit();
2056 try std.testing.expect(decoded.value.options.einsum.family_tuning != null);
2057 const found = try TuningRecipeCase.library.linalg.resolveMatrixProductSchedule(
2058 allocator,
2059 decoded.value.options.einsum.family_tuning.?,
2060 TuningRecipeCase.matrix(),
2061 );
2062 if (order == 0) {
2063 try std.testing.expectEqual(null, found);
2064 } else {
2065 try std.testing.expectEqual(TuningRecipeCase.matrix().threads, found.?);
2066 }
2067 std.mem.swap(
2068 TuningRecipeCase.tuning.FamilyTuningRecord,
2069 &source.records[0],
2070 &source.records[1],
2071 );
2072 }
2073 }
2074
2075 const MatrixRecipeCase = struct {
2076 const library = TuningRecipeCase.library;
2077 const tuning = library.tuning;
2078 const Reader = library.linalg.MatrixProductScheduleReader;
2079 const Record = tuning.MatrixProductFamilyScheduleTuningRecord;
2080 const device: gpu.DeviceIdentity = .{
2081 .backend = .cuda,
2082 .family = .nvidia_cuda,
2083 .name = "matrix recipe device",
2084 .vendor_id = 0x10de,
2085 .device_id = 42,
2086 .driver_version = "matrix recipe driver",
2087 };
2088
2089 fn record() !Record {
2090 const instance = TuningRecipeCase.matrix();
2091 const threads = library.linalg.matrixProductThreadCandidatesForExtents(
2092 instance.m,
2093 instance.n,
2094 );
2095 const capacity = tuning.matrix_product_family_schedule_tuning_max_candidates;
2096 var candidates: [capacity]tuning.MatrixProductFamilyScheduleThreads = undefined;
2097 for (threads.slice(), 0..) |value, index| {
2098 candidates[index] = .{ .x = value.x, .y = value.y };
2099 }
2100 return .{
2101 .key = try tuning.MatrixProductFamilyScheduleTuningKey.init(device, .{
2102 .format = .cuda_ptx,
2103 .m = instance.m,
2104 .n = instance.n,
2105 .k = instance.k,
2106 .dtype = instance.dtype,
2107 .accumulation_dtype = instance.accumulation_dtype,
2108 .family_version = library.linalg.matrix_product_family_version,
2109 .candidates = candidates[0..threads.slice().len],
2110 }),
2111 .selection = .{
2112 .threads = .{ .x = instance.threads.x, .y = instance.threads.y },
2113 .winner_median_ns = 3,
2114 .runner_up_median_ns = 5,
2115 .sample_count = 7,
2116 },
2117 };
2118 }
2119
2120 fn reader(items: []const Record) Reader {
2121 return .{ .device = device, .format = .cuda_ptx, .records = items };
2122 }
2123
2124 fn options(value: ?Reader) preparation.BackendPreparationRunOptions {
2125 return .{ .tensor = .{ .einsum = .{
2126 .kernel_library = .enabled,
2127 .matrix_product_tuning = value,
2128 } } };
2129 }
2130 };
2131
2132 fn independentMatrixRecipe() ![]u8 {
2133 var fixture = try Fixture.init();
2134 defer fixture.deinit();
2135 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
2136 defer arena.deinit();
2137 const source = arena.allocator();
2138 const items = try source.alloc(MatrixRecipeCase.Record, 1);
2139 items[0] = try MatrixRecipeCase.record();
2140 var reader = MatrixRecipeCase.reader(items);
2141 reader.device.name = try source.dupe(u8, reader.device.name);
2142 reader.device.driver_version = try source.dupe(u8, reader.device.driver_version.?);
2143 const bytes = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(reader));
2144 @memset(@constCast(reader.device.name), 'x');
2145 @memset(@constCast(reader.device.driver_version.?), 'y');
2146 items[0].key.k = 0;
2147 items[0].selection.threads = .{ .x = 99, .y = 99 };
2148 return bytes;
2149 }
2150
2151 test "Accy publication matrix product schedule recipes own device facts hits and misses" {
2152 const allocator = std.testing.allocator;
2153 const bytes = try independentMatrixRecipe();
2154 defer allocator.free(bytes);
2155 var decoded = try records.codec.decode(
2156 allocator,
2157 preparation.recipe.Record(.tensor),
2158 .tensor,
2159 bytes,
2160 );
2161 defer decoded.deinit();
2162 const reader = decoded.value.options.einsum.matrix_product_tuning.?;
2163 try std.testing.expectEqualStrings(MatrixRecipeCase.device.name, reader.device.name);
2164 try std.testing.expectEqualStrings(
2165 MatrixRecipeCase.device.driver_version.?,
2166 reader.device.driver_version.?,
2167 );
2168 const instance = TuningRecipeCase.matrix();
2169 try std.testing.expectEqual(instance.threads, (try reader.resolve(instance)).?);
2170 for (0..5) |index| {
2171 var absent = instance;
2172 switch (index) {
2173 0 => absent.m += 1,
2174 1 => absent.n += 1,
2175 2 => absent.k += 1,
2176 3 => absent.dtype = .f16,
2177 else => absent.accumulation_dtype = .f16,
2178 }
2179 try std.testing.expectEqual(null, try reader.resolve(absent));
2180 }
2181 for (0..8) |index| {
2182 var absent = reader;
2183 switch (index) {
2184 0 => absent.device.backend = .vulkan,
2185 1 => absent.device.family = .external,
2186 2 => absent.device.name = "another device",
2187 3 => absent.device.vendor_id = null,
2188 4 => absent.device.device_id = null,
2189 5 => absent.device.driver_version = null,
2190 6 => absent.device.driver_version = "another driver",
2191 else => absent.format = .vulkan_spirv,
2192 }
2193 try std.testing.expectEqual(null, try absent.resolve(instance));
2194 }
2195 }
2196
2197 test "Accy publication matrix product schedule recipes compare relocated reader values" {
2198 const allocator = std.testing.allocator;
2199 var fixture = try Fixture.init();
2200 defer fixture.deinit();
2201 const items = [_]MatrixRecipeCase.Record{try MatrixRecipeCase.record()};
2202 const reader = MatrixRecipeCase.reader(&items);
2203 const original = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(reader));
2204 defer allocator.free(original);
2205 const relocated = try allocator.dupe(MatrixRecipeCase.Record, &items);
2206 defer allocator.free(relocated);
2207 var other = reader;
2208 other.records = relocated;
2209 other.device.name = try allocator.dupe(u8, reader.device.name);
2210 defer allocator.free(other.device.name);
2211 other.device.driver_version = try allocator.dupe(u8, reader.device.driver_version.?);
2212 defer allocator.free(other.device.driver_version.?);
2213 const copied = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(other));
2214 defer allocator.free(copied);
2215 try std.testing.expectEqualStrings(original, copied);
2216 try std.testing.expect(reader.eql(other));
2217 for (0..6) |index| {
2218 var changed = reader;
2219 var entry_ = items;
2220 changed.records = &entry_;
2221 switch (index) {
2222 0 => changed.device.driver_version = "another driver",
2223 1 => changed.device.name = "another device",
2224 2 => changed.format = .vulkan_spirv,
2225 3 => entry_[0].key.k += 1,
2226 4 => entry_[0].selection.threads.x += 1,
2227 else => entry_[0].selection.sample_count += 1,
2228 }
2229 const bytes = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(changed));
2230 defer allocator.free(bytes);
2231 try std.testing.expect(!std.mem.eql(u8, original, bytes));
2232 try std.testing.expect(!reader.eql(changed));
2233 }
2234 }
2235
2236 test "Accy publication matrix product schedule recipes preserve presence and record order" {
2237 const allocator = std.testing.allocator;
2238 var fixture = try Fixture.init();
2239 defer fixture.deinit();
2240 const empty = MatrixRecipeCase.reader(&.{});
2241 const absent = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(null));
2242 defer allocator.free(absent);
2243 const present = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(empty));
2244 defer allocator.free(present);
2245 try std.testing.expect(!std.mem.eql(u8, absent, present));
2246 const item = try MatrixRecipeCase.record();
2247 var items = [_]MatrixRecipeCase.Record{ item, item };
2248 items[1].selection.threads = .{ .x = 99, .y = 99 };
2249 const reader = MatrixRecipeCase.reader(&items);
2250 const first = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(reader));
2251 defer allocator.free(first);
2252 std.mem.swap(MatrixRecipeCase.Record, &items[0], &items[1]);
2253 const second = try recipeBytes(&fixture, .tensor, MatrixRecipeCase.options(reader));
2254 defer allocator.free(second);
2255 try std.testing.expect(!std.mem.eql(u8, first, second));
2256 for ([_][]const u8{ first, second }, 0..) |bytes, index| {
2257 var decoded = try records.codec.decode(
2258 allocator,
2259 preparation.recipe.Record(.tensor),
2260 .tensor,
2261 bytes,
2262 );
2263 defer decoded.deinit();
2264 const restored = decoded.value.options.einsum.matrix_product_tuning.?;
2265 if (index == 0) {
2266 try std.testing.expectEqual(
2267 TuningRecipeCase.matrix().threads,
2268 (try restored.resolve(TuningRecipeCase.matrix())).?,
2269 );
2270 } else {
2271 try std.testing.expectError(
2272 error.InvalidArtifact,
2273 restored.resolve(TuningRecipeCase.matrix()),
2274 );
2275 }
2276 }
2277 }
2278
2279 fn matrixRecipeFailure(
2280 allocator: std.mem.Allocator,
2281 fixture: *Fixture,
2282 reader: MatrixRecipeCase.Reader,
2283 ) !void {
2284 const bytes = try preparation.recipe.encode(
2285 allocator,
2286 .tensor,
2287 fixture.source.choir_module,
2288 MatrixRecipeCase.options(reader),
2289 );
2290 defer allocator.free(bytes);
2291 var decoded = try records.codec.decode(
2292 allocator,
2293 preparation.recipe.Record(.tensor),
2294 .tensor,
2295 bytes,
2296 );
2297 defer decoded.deinit();
2298 const reader_ = decoded.value.options.einsum.matrix_product_tuning.?;
2299 try std.testing.expectEqual(
2300 TuningRecipeCase.matrix().threads,
2301 (try reader_.resolve(TuningRecipeCase.matrix())).?,
2302 );
2303 }
2304
2305 test "Accy publication matrix product schedule recipes clean partial capture allocations" {
2306 var fixture = try Fixture.init();
2307 defer fixture.deinit();
2308 const items = [_]MatrixRecipeCase.Record{try MatrixRecipeCase.record()};
2309 try std.testing.checkAllAllocationFailures(std.testing.allocator, matrixRecipeFailure, .{
2310 &fixture, MatrixRecipeCase.reader(&items),
2311 });
2312 }
2313
2314 const StageObservation = struct {
2315 outcome: revision.receipt.Outcome,
2316 charged: revision.WorkVector,
2317 counters: revision.receipt.Counters,
2318 maximum_live_storage: u64,
2319 exceeded: ?revision.WorkVector.Component,
2320
2321 fn from(receipt: revision.WorkReceiptV1) StageObservation {
2322 return .{
2323 .outcome = receipt.outcome,
2324 .charged = receipt.charged,
2325 .counters = receipt.executed.counters,
2326 .maximum_live_storage = receipt.maximum_live_storage,
2327 .exceeded = receipt.exceeded,
2328 };
2329 }
2330 };
2331
2332 fn stageMatrixRequest(comptime stage: publication.Stage) StageRequest {
2333 if (stage == .kernel or stage == .contract) return .{
2334 .allowance = revision.WorkVector.uniform(std.math.maxInt(u64)),
2335 .workspace = 512 * 1024 * 1024,
2336 };
2337 return if (stage == .target) .{
2338 .allowance = revision.WorkVector.uniform(64 * 1024 * 1024 * 1024),
2339 .workspace = 256 * 1024 * 1024,
2340 } else .{};
2341 }
2342
2343 fn publishStage(
2344 chain: *Chain,
2345 root: *choir.ir.Operation,
2346 comptime stage: publication.Stage,
2347 ) !operation.Product {
2348 const builder = try stageBuilder(chain, root, stage, .{}, stageMatrixRequest(stage));
2349 errdefer abortStage(builder);
2350 try preparation.publication.capture(
2351 std.testing.allocator,
2352 builder,
2353 chain.products[@backingInt(stage) - 1],
2354 stage,
2355 .{},
2356 configuration,
2357 );
2358 return operation.Product.seal(builder, chain.kinds[@backingInt(stage)]);
2359 }
2360
2361 fn failedStage(
2362 store: *revision.Store,
2363 builder: *revision.Builder,
2364 err: anyerror,
2365 ) !StageObservation {
2366 var failure = builder.abort(.rejected) orelse return error.MissingFailureReceipt;
2367 defer failure.deinit();
2368 if (err != error.WorkExhausted and err != error.UnexpectedImage) return err;
2369 try std.testing.expectEqual(@as(u32, 0), store.publicationCount());
2370 const expected: revision.receipt.Outcome = if (err == error.WorkExhausted)
2371 .exhausted
2372 else
2373 .rejected;
2374 try std.testing.expectEqual(expected, failure.work.outcome);
2375 return StageObservation.from(failure.work);
2376 }
2377
2378 const StageRoute = enum { cold, warm, restored, restored_changed };
2379
2380 fn screenStage(
2381 builder: *revision.Builder,
2382 candidate: operation.Product,
2383 changed: bool,
2384 ) !void {
2385 const record = candidate.revision.view().semantic_record;
2386 if (!changed) return std.testing.expect(try builder.screenRestored(record, try choir.product.compiler.manifest()));
2387 const allocator = std.testing.allocator;
2388 const bytes = try allocator.dupe(u8, record);
2389 defer allocator.free(bytes);
2390 const exact = try revision.record.decodeExact(bytes);
2391 try std.testing.expect(exact.image.len > 0);
2392 @constCast(exact.image)[exact.image.len - 1] ^= 1;
2393 try std.testing.expect(try builder.screenRestored(bytes, try choir.product.compiler.manifest()));
2394 @memset(bytes, 0xa5);
2395 }
2396
2397 fn stageAttempt(
2398 root: *choir.ir.Operation,
2399 parent: operation.Product,
2400 comptime stage: publication.Stage,
2401 request: StageRequest,
2402 route: StageRoute,
2403 candidate: operation.Product,
2404 ) !StageObservation {
2405 const source = try StageSource.create(parent, stage, request.gate_scratch_bytes);
2406 defer source.store.release();
2407 const builder = try source.begin(root, stage, .{}, request);
2408 var open = true;
2409 defer if (open) abortStage(builder);
2410 if (route == .warm) {
2411 const admitted = builder.admitReuse(candidate.revision) catch |err| {
2412 open = false;
2413 return failedStage(source.store, builder, err);
2414 };
2415 if (admitted) |value| {
2416 var reused = value;
2417 open = false;
2418 defer reused.deinit();
2419 try std.testing.expect(candidate.revision.eql(reused.revision));
2420 try std.testing.expectEqual(@as(u32, 0), source.store.publicationCount());
2421 return StageObservation.from(reused.work);
2422 }
2423 }
2424 if (route == .restored or route == .restored_changed) {
2425 try screenStage(builder, candidate, route == .restored_changed);
2426 }
2427 preparation.publication.capture(
2428 std.testing.allocator,
2429 builder,
2430 parent,
2431 stage,
2432 .{},
2433 configuration,
2434 ) catch |err| {
2435 open = false;
2436 return failedStage(source.store, builder, err);
2437 };
2438 const product = operation.Product.seal(builder, source.kind) catch |err| {
2439 open = false;
2440 return failedStage(source.store, builder, err);
2441 };
2442 open = false;
2443 defer product.release();
2444 try std.testing.expectEqualSlices(
2445 u8,
2446 candidate.revision.view().semantic_record,
2447 product.revision.view().semantic_record,
2448 );
2449 return StageObservation.from(product.revision.view().work);
2450 }
2451
2452 fn checkChangedStage(
2453 root: *choir.ir.Operation,
2454 parent: operation.Product,
2455 comptime stage: publication.Stage,
2456 candidate: operation.Product,
2457 ) !void {
2458 const changed = try stageAttempt(
2459 root,
2460 parent,
2461 stage,
2462 stageMatrixRequest(stage),
2463 .restored_changed,
2464 candidate,
2465 );
2466 try std.testing.expectEqual(.rejected, changed.outcome);
2467 try std.testing.expectEqual(
2468 preparation.recipe.pipeline(stage).len,
2469 changed.counters.pass_runs,
2470 );
2471 }
2472
2473 fn checkStageMatrix(comptime stage: publication.Stage) !void {
2474 var fixture = try Fixture.init();
2475 defer fixture.deinit();
2476 var captured = try captureFixture(&fixture);
2477 defer captured.deinit();
2478 const root = fixture.source.choir_module;
2479 var chain = try Chain.init(root, captured);
2480 defer chain.deinit();
2481 const baseline = try publishStage(&chain, root, stage);
2482 defer baseline.release();
2483 const receipt = baseline.revision.view().work;
2484 const parent = chain.products[@backingInt(stage) - 1];
2485 try std.testing.expectEqual(.success, receipt.outcome);
2486 try std.testing.expect(!receipt.missing_work_contract);
2487 try std.testing.expectEqual(
2488 preparation.recipe.pipeline(stage).len,
2489 receipt.executed.counters.pass_runs,
2490 );
2491 try checkChangedStage(root, parent, stage, baseline);
2492 inline for (@typeInfo(revision.WorkVector).@"struct".field_names) |field| {
2493 const value = @field(receipt.charged, field);
2494 for ([_]i64{ -1, 0, 1 }) |offset| {
2495 if (value == 0 and offset < 0) continue;
2496 var request = stageMatrixRequest(stage);
2497 request.allowance = receipt.charged;
2498 @field(request.allowance, field) = @intCast(@as(i64, @intCast(value)) + offset);
2499 const cold = try stageAttempt(root, parent, stage, request, .cold, baseline);
2500 const warm = try stageAttempt(root, parent, stage, request, .warm, baseline);
2501 const restored = try stageAttempt(root, parent, stage, request, .restored, baseline);
2502 const expected: revision.receipt.Outcome = if (offset < 0) .exhausted else .success;
2503 try std.testing.expectEqual(expected, cold.outcome);
2504 try std.testing.expectEqual(cold.outcome, warm.outcome);
2505 try std.testing.expectEqualDeep(cold, restored);
2506 try std.testing.expectEqualDeep(cold.charged, warm.charged);
2507 try std.testing.expectEqual(cold.exceeded, warm.exceeded);
2508 try std.testing.expectEqual(cold.maximum_live_storage, warm.maximum_live_storage);
2509 try std.testing.expectEqual(@as(u64, 0), warm.counters.pass_runs);
2510 }
2511 }
2512 }
2513
2514 test "Accy publication Tensor cold warm and restored share every work boundary" {
2515 try checkStageMatrix(.tensor);
2516 }
2517
2518 test "Accy publication Dispatch cold warm and restored share every work boundary" {
2519 try checkStageMatrix(.dispatch);
2520 }
2521
2522 test "Accy publication Memory cold warm and restored share every work boundary" {
2523 try checkStageMatrix(.memory);
2524 }
2525
2526 test "Accy publication Target cold warm and restored share every work boundary" {
2527 try checkStageMatrix(.target);
2528 }
2529
2530 test "Accy publication Kernel cold warm and restored share every work boundary" {
2531 try checkStageMatrix(.kernel);
2532 }
2533
2534 fn checkWorkspaceRequest(
2535 root: *choir.ir.Operation,
2536 parent: operation.Product,
2537 comptime stage: publication.Stage,
2538 request: StageRequest,
2539 candidate: operation.Product,
2540 ) !void {
2541 const measured = try stageAttempt(root, parent, stage, request, .cold, candidate);
2542 try std.testing.expectEqual(.success, measured.outcome);
2543 for ([_]i64{ -1, 0, 1 }) |offset| {
2544 var exact = request;
2545 exact.allowance = measured.charged;
2546 const capacity: i64 = @intCast(measured.charged.allocation_capacity);
2547 exact.allowance.allocation_capacity = @intCast(capacity + offset);
2548 const cold = try stageAttempt(root, parent, stage, exact, .cold, candidate);
2549 const warm = try stageAttempt(root, parent, stage, exact, .warm, candidate);
2550 const restored = try stageAttempt(root, parent, stage, exact, .restored, candidate);
2551 const expected: revision.receipt.Outcome = if (offset < 0) .exhausted else .success;
2552 try std.testing.expectEqual(expected, cold.outcome);
2553 try std.testing.expectEqualDeep(cold, restored);
2554 try std.testing.expectEqual(cold.outcome, warm.outcome);
2555 try std.testing.expectEqualDeep(cold.charged, warm.charged);
2556 try std.testing.expectEqual(cold.exceeded, warm.exceeded);
2557 try std.testing.expectEqual(cold.maximum_live_storage, warm.maximum_live_storage);
2558 }
2559 }
2560
2561 fn checkStageWorkspace(comptime stage: publication.Stage) !void {
2562 var fixture = try Fixture.init();
2563 defer fixture.deinit();
2564 var captured = try captureFixture(&fixture);
2565 defer captured.deinit();
2566 const root = fixture.source.choir_module;
2567 var chain = try Chain.init(root, captured);
2568 defer chain.deinit();
2569 const baseline = try publishStage(&chain, root, stage);
2570 defer baseline.release();
2571 const parent = chain.products[@backingInt(stage) - 1];
2572 for ([_]i64{ -1, 1 }) |offset| {
2573 var request = stageMatrixRequest(stage);
2574 request.workspace = @intCast(@as(i64, @intCast(request.workspace)) + offset);
2575 try checkWorkspaceRequest(root, parent, stage, request, baseline);
2576 }
2577 var partition = stageMatrixRequest(stage);
2578 partition.gate_scratch_bytes += 1;
2579 try checkWorkspaceRequest(root, parent, stage, partition, baseline);
2580 }
2581
2582 test "Accy publication workspace changes preserve Tensor budget outcomes" {
2583 try checkStageWorkspace(.tensor);
2584 }
2585
2586 test "Accy publication workspace changes preserve Dispatch budget outcomes" {
2587 try checkStageWorkspace(.dispatch);
2588 }
2589
2590 test "Accy publication workspace changes preserve Memory budget outcomes" {
2591 try checkStageWorkspace(.memory);
2592 }
2593
2594 test "Accy publication workspace changes preserve Target budget outcomes" {
2595 try checkStageWorkspace(.target);
2596 }
2597
2598 test "Accy publication workspace changes preserve Kernel budget outcomes" {
2599 try checkStageWorkspace(.kernel);
2600 }
2601
2602 test "Accy publication Contract cold warm and restored share every work boundary" {
2603 try checkStageMatrix(.contract);
2604 }
2605
2606 test "Accy publication Contract workspace changes preserve budget outcomes" {
2607 try checkStageWorkspace(.contract);
2608 }
2609
2610 test "Accy publication Contract through Target consumes each newly sealed predecessor" {
2611 const allocator = std.testing.allocator;
2612 var fixture = try Fixture.init();
2613 defer fixture.deinit();
2614 const root = fixture.source.choir_module;
2615 const store = try revision.Store.create(allocator, .{
2616 .revisions = 7,
2617 .kinds = 7,
2618 .builders = 1,
2619 .compiler_manifests = 2,
2620 .record_bytes = 32 * 1024 * 1024,
2621 .gate_scratch_bytes = configuration.gate_scratch,
2622 .candidate_count = 8,
2623 .screening_bytes = 32 * 1024 * 1024,
2624 });
2625 defer store.release();
2626 const stages = [_]publication.Stage{
2627 .semantic, .contract, .tensor, .dispatch, .memory, .kernel, .target,
2628 };
2629 var kinds: [stages.len]*const revision.Kind = undefined;
2630 inline for (stages, 0..) |stage, index| {
2631 kinds[index] = try preparation.publication.registerKind(allocator, store, stage, configuration);
2632 }
2633 store.freeze();
2634 var product = try seal(store, kinds[0], root, .semantic, &publication.irRecord(.semantic), null);
2635 defer product.release();
2636 var revisions: [7]*const revision.Revision = undefined;
2637 revisions[0] = product.revision;
2638 inline for (stages[1..], 1..) |stage, index| {
2639 const source = StageSource{ .store = store, .kind = kinds[index], .parent = product };
2640 const builder = try source.begin(root, stage, .{}, stageMatrixRequest(stage));
2641 errdefer abortStage(builder);
2642 try preparation.publication.capture(allocator, builder, product, stage, .{}, configuration);
2643 const next = try operation.Product.seal(builder, kinds[index]);
2644 product.release();
2645 product = next;
2646 revisions[index] = next.revision;
2647 const receipt = product.revision.view().work;
2648 try std.testing.expectEqual(.success, receipt.outcome);
2649 try std.testing.expect(!receipt.missing_work_contract);
2650 try std.testing.expectEqual(preparation.recipe.pipeline(stage).len, receipt.executed.counters.pass_runs);
2651 }
2652 try std.testing.expectEqual(7, store.publicationCount());
2653 const prepared = try preparation.pipeline.BackendPreparedModule.create(allocator, revisions);
2654 defer prepared.deinit();
2655 try std.testing.expect(prepared.stage(.target).eql(product.revision));
2656 var graph = try prepared.productGraph(allocator);
2657 defer graph.deinit(allocator);
2658 try std.testing.expectEqual(7, graph.products.len);
2659 const module = try publication.Module(.target).fromRevision(allocator, product.revision);
2660 defer module.deinit();
2661 var plan = try module.plan(allocator, configuration.image);
2662 defer plan.deinit();
2663 try std.testing.expectEqual(1, plan.value.kernels.len);
2664 }
2665
2666 test "Accy retained preparation owns exact metadata after its source chain is released" {
2667 const allocator = std.testing.allocator;
2668 var fixture = try Fixture.init();
2669 defer fixture.deinit();
2670 var captured = try captureFixture(&fixture);
2671 defer captured.deinit();
2672 var chain = try Chain.init(fixture.source.choir_module, captured);
2673 var live = true;
2674 defer if (live) chain.deinit();
2675 var graph = try retainedPreparationGraph(&chain);
2676 defer graph.deinit(allocator);
2677 chain.deinit();
2678 live = false;
2679 try std.testing.expectEqual(7, graph.products.len);
2680 for (graph.products) |key| {
2681 const exact = try revision.record.decodeExact(key.record.bytes());
2682 try std.testing.expect(exact.image.len > 0);
2683 try std.testing.expect(key.ref.eql(try revision.record.decodeAddress(exact.address)));
2684 }
2685 }
2686
2687 fn preparationRevisions(chain: *const Chain) [7]*const revision.Revision {
2688 var result: [7]*const revision.Revision = undefined;
2689 for (chain.products, &result) |product, *item| item.* = product.revision;
2690 return result;
2691 }
2692
2693 fn retainedPreparationGraph(chain: *const Chain) !choir.product.incremental.ProductGraph {
2694 const allocator = std.testing.allocator;
2695 const Module = preparation.pipeline.BackendPreparedModule;
2696 var failing = std.testing.FailingAllocator.init(allocator, .{ .fail_index = 0 });
2697 try std.testing.expectError(error.OutOfMemory, Module.create(failing.allocator(), preparationRevisions(chain)));
2698 const prepared = try Module.create(allocator, preparationRevisions(chain));
2699 defer prepared.deinit();
2700 const retained = try prepared.retain(allocator);
2701 defer retained.deinit();
2702 return retained.productGraph(allocator);
2703 }
2704
2705 test "Accy retained preparation rejects a mixed exact predecessor and wrong stage" {
2706 const allocator = std.testing.allocator;
2707 const Module = preparation.pipeline.BackendPreparedModule;
2708 var first = try Fixture.init();
2709 defer first.deinit();
2710 var second = try Fixture.initNames(&.{"different_source"});
2711 defer second.deinit();
2712 var first_capture = try captureFixture(&first);
2713 defer first_capture.deinit();
2714 var second_capture = try captureFixture(&second);
2715 defer second_capture.deinit();
2716 var first_chain = try Chain.init(first.source.choir_module, first_capture);
2717 defer first_chain.deinit();
2718 var second_chain = try Chain.init(second.source.choir_module, second_capture);
2719 defer second_chain.deinit();
2720 var items = preparationRevisions(&first_chain);
2721 items[6] = second_chain.products[6].revision;
2722 try std.testing.expect(items[6].address().eql(first_chain.products[6].revision.address()));
2723 if (Module.create(allocator, items)) |unexpected| {
2724 unexpected.deinit();
2725 return error.TestExpectedError;
2726 } else |err| try std.testing.expect(err == error.UnboundProductInput);
2727 items[6] = first_chain.products[5].revision;
2728 try std.testing.expectError(error.WrongStage, Module.create(allocator, items));
2729 }
2730
2731 fn preparationRequest() preparation.publication.PreparationRequest {
2732 return .{
2733 .source = "preparation-publication",
2734 .work = .{
2735 .allowance = revision.WorkVector.uniform(std.math.maxInt(u64)),
2736 .workspace = 512 * 1024 * 1024,
2737 .events = 256,
2738 },
2739 .record_bytes = 32 * 1024 * 1024,
2740 };
2741 }
2742
2743 fn prepareAndRelease(
2744 source: *semantic.SemanticModule,
2745 request: preparation.publication.PreparationRequest,
2746 report: *preparation.publication.PreparationReport,
2747 ) !void {
2748 const prepared = try preparation.publication.prepare(
2749 std.testing.allocator,
2750 .{ .draft = source },
2751 request,
2752 report,
2753 configuration,
2754 );
2755 prepared.deinit();
2756 }
2757
2758 test "Accy preparation publication replays exact stages with current request receipts" {
2759 const allocator = std.testing.allocator;
2760 var fixture = try Fixture.init();
2761 defer fixture.deinit();
2762 var cold = preparation.publication.PreparationReport{};
2763 defer cold.deinit();
2764 var request = preparationRequest();
2765 request.options = .{
2766 .target_profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object },
2767 .generated_scan_schedules = &.{.{ .schedule = .{ .threads = 32, .items = 2 } }},
2768 .generated_row_pipeline_schedules = &.{.{ .schedule = .{ .threads = 64 } }},
2769 };
2770 const first = try preparation.publication.prepare(
2771 allocator,
2772 .{ .draft = fixture.source },
2773 request,
2774 &cold,
2775 configuration,
2776 );
2777 defer first.deinit();
2778 try std.testing.expectEqual(7, cold.completed);
2779 request.candidate = first;
2780 request.work.allowance = try cold.charged();
2781 var warm = preparation.publication.PreparationReport{};
2782 defer warm.deinit();
2783 const second = try preparation.publication.prepare(
2784 allocator,
2785 .{ .draft = fixture.source },
2786 request,
2787 &warm,
2788 configuration,
2789 );
2790 defer second.deinit();
2791 try std.testing.expectEqualDeep(try cold.charged(), try warm.charged());
2792 inline for (std.enums.values(publication.Stage), 0..) |stage, index| {
2793 try std.testing.expect(first.stage(stage).eql(second.stage(stage)));
2794 const work = warm.stages[index].work();
2795 try std.testing.expectEqual(.success, work.outcome);
2796 try std.testing.expectEqualDeep(cold.stages[index].work().charged, work.charged);
2797 if (stage == .semantic) {
2798 try std.testing.expect(warm.stages[index] == .cold);
2799 } else {
2800 try std.testing.expect(warm.stages[index] == .warm);
2801 try std.testing.expectEqual(0, work.executed.counters.pass_runs);
2802 }
2803 }
2804 try std.testing.expectEqual(null, preparation.readBackendTargetProfile(
2805 fixture.source.choir_module,
2806 ));
2807 }
2808
2809 test "Accy preparation publication carries one allowance through cold and warm stages" {
2810 const allocator = std.testing.allocator;
2811 var fixture = try Fixture.init();
2812 defer fixture.deinit();
2813 var cold = preparation.publication.PreparationReport{};
2814 defer cold.deinit();
2815 var request = preparationRequest();
2816 const prepared = try preparation.publication.prepare(
2817 allocator,
2818 .{ .draft = fixture.source },
2819 request,
2820 &cold,
2821 configuration,
2822 );
2823 defer prepared.deinit();
2824 const charged = try cold.charged();
2825 inline for (@typeInfo(revision.WorkVector).@"struct".field_names) |field| {
2826 try std.testing.expect(@field(charged, field) > 0);
2827 for ([_]bool{ false, true }) |warm| {
2828 request.candidate = if (warm) prepared else null;
2829 request.work.allowance = charged;
2830 @field(request.work.allowance, field) -= 1;
2831 var failed = preparation.publication.PreparationReport{};
2832 defer failed.deinit();
2833 try std.testing.expectError(error.WorkExhausted, prepareAndRelease(
2834 fixture.source,
2835 request,
2836 &failed,
2837 ));
2838 try std.testing.expect(failed.completed < 7);
2839 if (failed.failure) |failure| {
2840 try std.testing.expectEqual(.exhausted, failure.work.outcome);
2841 }
2842 try std.testing.expectEqual(.success, prepared.stage(.target).view().work.outcome);
2843 }
2844 }
2845 inline for (@typeInfo(revision.WorkVector).@"struct".field_names) |field| {
2846 @field(request.work.allowance, field) = @field(charged, field) + 1;
2847 }
2848 for ([_]bool{ false, true }) |warm| {
2849 request.candidate = if (warm) prepared else null;
2850 var accepted = preparation.publication.PreparationReport{};
2851 defer accepted.deinit();
2852 try prepareAndRelease(fixture.source, request, &accepted);
2853 try std.testing.expectEqualDeep(charged, try accepted.charged());
2854 }
2855 }
2856
2857 test "Accy preparation publication replaces a changed exact source without altering its candidate" {
2858 const allocator = std.testing.allocator;
2859 var fixture = try Fixture.init();
2860 defer fixture.deinit();
2861 var cold = preparation.publication.PreparationReport{};
2862 defer cold.deinit();
2863 var request = preparationRequest();
2864 const first = try preparation.publication.prepare(
2865 allocator,
2866 .{ .draft = fixture.source },
2867 request,
2868 &cold,
2869 configuration,
2870 );
2871 defer first.deinit();
2872 const old_image = try allocator.dupe(u8, first.stage(.semantic).view().exact.image);
2873 defer allocator.free(old_image);
2874 try fixture.source.choir_module.setAttr(
2875 "source_revision",
2876 try fixture.source.ctx.getStringAttr("next"),
2877 );
2878 var next = preparation.publication.PreparationReport{};
2879 defer next.deinit();
2880 request.candidate = first;
2881 const second = try preparation.publication.prepare(
2882 allocator,
2883 .{ .draft = fixture.source },
2884 request,
2885 &next,
2886 configuration,
2887 );
2888 defer second.deinit();
2889 try std.testing.expect(!first.stage(.semantic).eql(second.stage(.semantic)));
2890 for (next.stages[0..next.completed]) |item| try std.testing.expect(item == .cold);
2891 try std.testing.expectEqualSlices(u8, old_image, first.stage(.semantic).view().exact.image);
2892 }
2893
2894 test "Accy preparation publication cache owns updates across replacement and failure" {
2895 const allocator = std.testing.allocator;
2896 var cache = preparation.BackendPreparationCache.init(allocator);
2897 defer cache.deinit();
2898 var first_report = preparation.publication.PreparationReport{};
2899 defer first_report.deinit();
2900 var first = try cachePreparationFixture(&cache, &first_report, preparationRequest());
2901 defer first.deinit(allocator);
2902 try std.testing.expect(!first.reused);
2903 try std.testing.expect(first.request_changed);
2904 var warm_report = preparation.publication.PreparationReport{};
2905 defer warm_report.deinit();
2906 var warm = try cachePreparationFixture(&cache, &warm_report, preparationRequest());
2907 defer warm.deinit(allocator);
2908 try std.testing.expect(warm.reused);
2909 try std.testing.expect(!warm.request_changed);
2910 try std.testing.expectEqualDeep(try first_report.charged(), try warm_report.charged());
2911 var failed_report = preparation.publication.PreparationReport{};
2912 defer failed_report.deinit();
2913 var limited = preparationRequest();
2914 limited.work.allowance.rewrite_attempts = 0;
2915 try std.testing.expectError(error.WorkExhausted, cachePreparationFixture(
2916 &cache,
2917 &failed_report,
2918 limited,
2919 ));
2920 const current = cache.currentPrepared().?;
2921 try std.testing.expect(current.stage(.target).eql(warm.prepared.stage(.target)));
2922 var replacement_report = preparation.publication.PreparationReport{};
2923 defer replacement_report.deinit();
2924 var changed = preparationRequest();
2925 changed.variant = "another-target";
2926 var replacement = try cachePreparationFixture(&cache, &replacement_report, changed);
2927 defer replacement.deinit(allocator);
2928 try std.testing.expect(replacement.request_changed);
2929 try std.testing.expect(!replacement.reused);
2930 try std.testing.expect(first.prepared.stage(.target).eql(warm.prepared.stage(.target)));
2931 try std.testing.expect(!first.prepared.stage(.target).eql(replacement.prepared.stage(.target)));
2932 try std.testing.expectEqualStrings("", first.prepared.stage(.target).address().variant);
2933 }
2934
2935 fn cachePreparationFixture(
2936 cache: *preparation.BackendPreparationCache,
2937 report: *preparation.publication.PreparationReport,
2938 request: preparation.publication.PreparationRequest,
2939 ) !preparation.BackendPreparationCacheUpdate {
2940 var fixture = try Fixture.init();
2941 fixture.cache.deinit();
2942 return cache.refreshFromSemanticModule(fixture.source, request, report, configuration);
2943 }
2944
2945 fn captureTargetRecipe(allocator: std.mem.Allocator, fixture: *Fixture) !void {
2946 const bytes = try preparation.recipe.encode(allocator, .target, fixture.source.choir_module, .{
2947 .target_profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object },
2948 .generated_scan_schedules = &.{.{ .schedule = .{ .threads = 32, .items = 2 } }},
2949 .generated_row_pipeline_schedules = &.{.{ .schedule = .{ .threads = 64 } }},
2950 });
2951 allocator.free(bytes);
2952 }
2953
2954 test "Accy publication recipes release partial target override encoding allocations" {
2955 var fixture = try Fixture.init();
2956 defer fixture.deinit();
2957 try std.testing.checkAllAllocationFailures(
2958 std.testing.allocator,
2959 captureTargetRecipe,
2960 .{&fixture},
2961 );
2962 try std.testing.expectEqual(null, preparation.readBackendTargetProfile(
2963 fixture.source.choir_module,
2964 ));
2965 }
2966
2967 test "Accy preparation publication cache rolls back its last replacement allocation and retries" {
2968 const allocator = std.testing.allocator;
2969 var failing = std.testing.FailingAllocator.init(allocator, .{ .resize_fail_index = 0 });
2970 var cache = preparation.BackendPreparationCache.init(failing.allocator());
2971 defer cache.deinit();
2972 var initial_report = preparation.publication.PreparationReport{};
2973 defer initial_report.deinit();
2974 var initial = try cachePreparationFixture(&cache, &initial_report, preparationRequest());
2975 defer initial.deinit(failing.allocator());
2976 var warm_report = preparation.publication.PreparationReport{};
2977 defer warm_report.deinit();
2978 const start = failing.alloc_index;
2979 var warm = try cachePreparationFixture(&cache, &warm_report, preparationRequest());
2980 defer warm.deinit(failing.allocator());
2981 const allocations = failing.alloc_index - start;
2982 try std.testing.expect(allocations > 0);
2983 const previous = cache.currentPrepared().?;
2984 var failed_report = preparation.publication.PreparationReport{};
2985 defer failed_report.deinit();
2986 failing.fail_index = failing.alloc_index + allocations - 1;
2987 if (cachePreparationFixture(
2988 &cache,
2989 &failed_report,
2990 preparationRequest(),
2991 )) |value| {
2992 var unexpected = value;
2993 unexpected.deinit(failing.allocator());
2994 return error.ExpectedAllocationFailure;
2995 } else |err| try std.testing.expectEqual(error.OutOfMemory, err);
2996 try std.testing.expect(failing.has_induced_failure);
2997 try std.testing.expectEqual(7, failed_report.completed);
2998 try std.testing.expectEqual(previous, cache.currentPrepared().?);
2999 failing.fail_index = std.math.maxInt(usize);
3000 var retry_report = preparation.publication.PreparationReport{};
3001 defer retry_report.deinit();
3002 var retry = try cachePreparationFixture(&cache, &retry_report, preparationRequest());
3003 defer retry.deinit(failing.allocator());
3004 try std.testing.expect(retry.reused);
3005 try std.testing.expect(retry.prepared.stage(.target).eql(warm.prepared.stage(.target)));
3006 }
3007
3008 test "Accy preparation retained Semantic source replays every stage under the same allowance" {
3009 const allocator = std.testing.allocator;
3010 var cache = preparation.BackendPreparationCache.init(allocator);
3011 defer cache.deinit();
3012 var cold = preparation.publication.PreparationReport{};
3013 defer cold.deinit();
3014 var first = try cachePreparationFixture(&cache, &cold, preparationRequest());
3015 defer first.deinit(allocator);
3016 const source = first.prepared.stage(.semantic);
3017 var request = preparationRequest();
3018 request.work.allowance = try cold.charged();
3019 var warm = preparation.publication.PreparationReport{};
3020 defer warm.deinit();
3021 var second = try cache.refreshFromSemanticRevision(source, request, &warm, configuration);
3022 defer second.deinit(allocator);
3023 try std.testing.expect(second.reused);
3024 try std.testing.expect(!second.request_changed);
3025 try std.testing.expectEqualDeep(try cold.charged(), try warm.charged());
3026 for (warm.stages[0..warm.completed]) |stage| {
3027 try std.testing.expect(stage == .warm);
3028 try std.testing.expectEqual(0, stage.work().executed.counters.pass_runs);
3029 }
3030 var failed = preparation.publication.PreparationReport{};
3031 defer failed.deinit();
3032 request.work.allowance.rewrite_attempts -= 1;
3033 try std.testing.expectError(error.WorkExhausted, cache.refreshFromSemanticRevision(
3034 source,
3035 request,
3036 &failed,
3037 configuration,
3038 ));
3039 const current = cache.currentPrepared().?;
3040 try std.testing.expect(current.stage(.target).eql(first.prepared.stage(.target)));
3041 }
3042
3043 test "Accy preparation retained source restores original target inheritance after an override" {
3044 const allocator = std.testing.allocator;
3045 var fixture = try Fixture.init();
3046 fixture.cache.deinit();
3047 var source_owned = true;
3048 defer if (source_owned) fixture.source.deinit();
3049 try preparation.recipe.applyTargetOptions(allocator, fixture.source.choir_module, .{
3050 .target_profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object },
3051 .generated_scan_schedules = &.{.{ .schedule = .{ .threads = 32, .items = 2 } }},
3052 });
3053 var cache = preparation.BackendPreparationCache.init(allocator);
3054 defer cache.deinit();
3055 var first_report = preparation.publication.PreparationReport{};
3056 defer first_report.deinit();
3057 var request = preparationRequest();
3058 request.options = .{
3059 .target_profile = .{ .backend_kind = .cuda, .artifact_format = .cuda_ptx },
3060 .generated_scan_schedules = &.{.{ .schedule = .{ .threads = 64, .items = 4 } }},
3061 };
3062 source_owned = false;
3063 var first = try cache.refreshFromSemanticModule(
3064 fixture.source,
3065 request,
3066 &first_report,
3067 configuration,
3068 );
3069 defer first.deinit(allocator);
3070 try expectPreparationTarget(first.prepared, .cuda, "64x4");
3071 var next_report = preparation.publication.PreparationReport{};
3072 defer next_report.deinit();
3073 request.options = .{};
3074 var next = try cache.refreshFromSemanticRevision(
3075 first.prepared.stage(.semantic),
3076 request,
3077 &next_report,
3078 configuration,
3079 );
3080 defer next.deinit(allocator);
3081 try std.testing.expect(!next.reused);
3082 try std.testing.expect(next.request_changed);
3083 for (next_report.stages[0..5]) |stage| try std.testing.expect(stage == .warm);
3084 for (next_report.stages[5..7]) |stage| try std.testing.expect(stage == .cold);
3085 try expectPreparationTarget(next.prepared, .cpu, "32x2");
3086 try expectPreparationTarget(first.prepared, .cuda, "64x4");
3087 }
3088
3089 fn expectPreparationTarget(
3090 prepared: *const preparation.pipeline.BackendPreparedModule,
3091 kind: gpu.BackendKind,
3092 scans: []const u8,
3093 ) !void {
3094 const product = operation.Product{ .revision = prepared.stage(.target) };
3095 const image = try product.open(std.testing.allocator, configuration.image);
3096 defer image.destroy();
3097 var decoded = try records.codec.decode(
3098 std.testing.allocator,
3099 records.target.Record,
3100 .target,
3101 image.stage(),
3102 );
3103 defer decoded.deinit();
3104 try std.testing.expectEqual(kind, decoded.value.profile.?.backend_kind);
3105 try std.testing.expectEqualStrings(scans, decoded.value.generated_scan_schedules.?);
3106 }
3107
3108 test "Accy preparation retained input outlives its cache and recomputes uncached stages" {
3109 const allocator = std.testing.allocator;
3110 var cache = preparation.BackendPreparationCache.init(allocator);
3111 var cache_live = true;
3112 defer if (cache_live) cache.deinit();
3113 var first_report = preparation.publication.PreparationReport{};
3114 defer first_report.deinit();
3115 var request = preparationRequest();
3116 request.options = .{
3117 .target_profile = .{ .backend_kind = .cpu, .artifact_format = .cpu_object },
3118 .generated_scan_schedules = &.{.{ .schedule = .{ .threads = 32, .items = 2 } }},
3119 };
3120 var first = try cachePreparationFixture(&cache, &first_report, request);
3121 var first_live = true;
3122 defer if (first_live) first.deinit(allocator);
3123 const source = try first_report.source.?.retain(allocator);
3124 defer source.deinit();
3125 request.work.allowance = try first_report.charged();
3126 first.deinit(allocator);
3127 first_live = false;
3128 first_report.deinit();
3129 cache.deinit();
3130 cache_live = false;
3131 var next_report = preparation.publication.PreparationReport{};
3132 defer next_report.deinit();
3133 const next = try preparation.publication.prepare(
3134 allocator,
3135 .{ .retained = source },
3136 request,
3137 &next_report,
3138 configuration,
3139 );
3140 defer next.deinit();
3141 try std.testing.expect(next_report.stages[0] == .warm);
3142 for (next_report.stages[1..7]) |stage| {
3143 try std.testing.expect(stage == .cold);
3144 try std.testing.expect(stage.work().executed.counters.pass_runs > 0);
3145 }
3146 try std.testing.expectEqualDeep(request.work.allowance, try next_report.charged());
3147 try expectPreparationTarget(next, .cpu, "32x2");
3148 }
3149
3150 test "Accy preparation retained cache rejects a different exact Semantic at the same address" {
3151 const allocator = std.testing.allocator;
3152 var cache = preparation.BackendPreparationCache.init(allocator);
3153 defer cache.deinit();
3154 var first_report = preparation.publication.PreparationReport{};
3155 defer first_report.deinit();
3156 var first = try cachePreparationFixture(&cache, &first_report, preparationRequest());
3157 defer first.deinit(allocator);
3158 var fixture = try Fixture.initNames(&.{"another_semantic"});
3159 defer fixture.deinit();
3160 var other_report = preparation.publication.PreparationReport{};
3161 defer other_report.deinit();
3162 const other = try preparation.publication.prepare(
3163 allocator,
3164 .{ .draft = fixture.source },
3165 preparationRequest(),
3166 &other_report,
3167 configuration,
3168 );
3169 defer other.deinit();
3170 const source = other.stage(.semantic);
3171 try std.testing.expect(source.address().eql(first.prepared.stage(.semantic).address()));
3172 try std.testing.expect(!source.eql(first.prepared.stage(.semantic)));
3173 var rejected = preparation.publication.PreparationReport{};
3174 defer rejected.deinit();
3175 try std.testing.expectError(error.SemanticModuleRequired, retainedRefreshAndRelease(
3176 &cache,
3177 source,
3178 preparationRequest(),
3179 &rejected,
3180 ));
3181 try std.testing.expectEqual(0, rejected.completed);
3182 const current = cache.currentPrepared().?;
3183 try std.testing.expect(current.stage(.semantic).eql(first.prepared.stage(.semantic)));
3184 }
3185
3186 fn retainedRefreshAndRelease(
3187 cache: *preparation.BackendPreparationCache,
3188 source: *const revision.Revision,
3189 request: preparation.publication.PreparationRequest,
3190 report: *preparation.publication.PreparationReport,
3191 ) !void {
3192 var update = try cache.refreshFromSemanticRevision(source, request, report, configuration);
3193 update.deinit(std.testing.allocator);
3194 }
3195
3196 fn savedArtifactPreparation(profile: preparation.BackendTargetProfile) !*preparation.pipeline.BackendPreparedModule {
3197 const allocator = std.testing.allocator;
3198 var fixture = try Fixture.init();
3199 defer fixture.deinit();
3200 try preparation.setBackendTargetProfile(fixture.source.context(), fixture.source.choir_module, profile);
3201 try markSavedProgram(&fixture);
3202 var captured = try captureFixture(&fixture);
3203 defer captured.deinit();
3204 var chain = try Chain.init(fixture.source.choir_module, captured);
3205 defer chain.deinit();
3206 return preparation.pipeline.BackendPreparedModule.create(allocator, preparationRevisions(&chain));
3207 }
3208
3209 test "Accy retained artifact job restores captured plans and marked programs without sources" {
3210 const allocator = std.testing.allocator;
3211 const artifact = @import("../artifact/root.zig");
3212 const prepared = try savedArtifactPreparation(.{
3213 .backend_kind = .cuda,
3214 .artifact_format = .cuda_ptx,
3215 .dtype_bits = gpu.DTypeSet.init(&.{.f32}).bits,
3216 });
3217 var retained = true;
3218 defer if (retained) prepared.deinit();
3219 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3220 defer allocator.free(workspace);
3221 try std.testing.expectError(error.WorkExhausted, artifact.InputJob.create(&.{}, prepared, configuration));
3222 const job = try artifact.InputJob.create(workspace, prepared, configuration);
3223 defer job.destroy();
3224 prepared.deinit();
3225 retained = false;
3226 const plans = job.plans();
3227 try std.testing.expectEqual(1, plans.outlines.kernelCount());
3228 try std.testing.expectEqual(1, plans.generated.kernels.items.len);
3229 const work = plans.schedule.work_items.items[0];
3230 try std.testing.expect(plans.schedule.getWorkForRoot(work.root).? == &plans.schedule.work_items.items[0]);
3231 try std.testing.expect(plans.buffers.getSlot(work.output_value) != null);
3232 for (plans.buffers.slots.items) |slot| {
3233 try std.testing.expect(plans.spaces.getAssignmentForSlot(slot.id) != null);
3234 try std.testing.expect(plans.layouts.getAssignmentForSlot(slot.id) != null);
3235 }
3236 const generated = &plans.generated.kernels.items[0];
3237 const marker = generated.program.storage.kernel.func().op.getAttrAs(choir.ir.Attribute.StringAttr, "restoration_marker") orelse return error.TestExpectedResult;
3238 try std.testing.expectEqualStrings("retain this exact program", marker.getValue());
3239 var actual = try records.program.capture(allocator, &generated.program, configuration);
3240 defer actual.deinit();
3241 var refs = records.reference.Index{ .allocator = allocator, .limit = 0 };
3242 defer refs.deinit();
3243 try records.codec.compare(actual.value, plans.target.?.kernels[0].lowered.program, &refs);
3244 try std.testing.expect(job.fixed.status().peak_bytes <= workspace.len);
3245 const context = generated.program.kernelModule().context;
3246 const handler = try context.registerDiagnosticHandler(.{ .handle = reportRetainedArtifactDiagnostic });
3247 defer context.eraseDiagnosticHandler(handler);
3248 const payload = (try @import("../target/root.zig").compileKernelForArtifactFormat(
3249 allocator,
3250 .cuda_ptx,
3251 generated.entry_name,
3252 generated.program.kernelModule(),
3253 .{},
3254 )).payload;
3255 defer allocator.free(payload.text);
3256 try std.testing.expect(payload.text.len != 0);
3257 }
3258
3259 test "Accy retained artifact compilation owns its plan after workspace and preparation end" {
3260 const allocator = std.testing.allocator;
3261 const artifact = @import("../artifact/root.zig");
3262 const prepared = try savedArtifactPreparation(.{
3263 .backend_kind = .cuda,
3264 .artifact_format = .cuda_ptx,
3265 .dtype_bits = gpu.DTypeSet.init(&.{.f32}).bits,
3266 });
3267 var retained = true;
3268 defer if (retained) prepared.deinit();
3269 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3270 var workspace_live = true;
3271 defer if (workspace_live) allocator.free(workspace);
3272 var state = gpu.recording.BackendState{ .allocator = allocator };
3273 var plan = try artifact.createBackendArtifactPlanFromPreparedModule(allocator, state.handle(), prepared, .{}, workspace, configuration);
3274 defer plan.deinit();
3275 prepared.deinit();
3276 retained = false;
3277 @memset(workspace, 0xa5);
3278 allocator.free(workspace);
3279 workspace_live = false;
3280 try std.testing.expectEqual(1, state.create_count);
3281 try std.testing.expect(state.last_create_had_payload);
3282 try std.testing.expectEqual(1, plan.kernelCount());
3283 try std.testing.expectEqual(2, plan.input_slot_ids.len);
3284 try std.testing.expectEqual(1, plan.output_slot_ids.len);
3285 const kernel = plan.kernels.items[0];
3286 try std.testing.expectEqual(kernel.output_slot_id, plan.output_slot_ids[0]);
3287 try std.testing.expectEqualStrings(kernel.compile.entry_name, kernel.artifact.entry_name);
3288 try std.testing.expectEqual(kernel.compile.argument_count, state.last_create_argument_count);
3289 }
3290
3291 test "Accy retained artifact reads the actual published chain and rejects before native effects" {
3292 const allocator = std.testing.allocator;
3293 const artifact = @import("../artifact/root.zig");
3294 var fixture = try Fixture.init();
3295 defer fixture.deinit();
3296 var report = preparation.publication.PreparationReport{};
3297 defer report.deinit();
3298 var request = preparationRequest();
3299 request.options.target_profile = .{
3300 .backend_kind = .cuda,
3301 .artifact_format = .cuda_ptx,
3302 .dtype_bits = gpu.DTypeSet.init(&.{.f32}).bits,
3303 };
3304 const prepared = try preparation.publication.prepare(allocator, .{ .draft = fixture.source }, request, &report, configuration);
3305 defer prepared.deinit();
3306 var state = gpu.recording.BackendState{ .allocator = allocator };
3307 try std.testing.expectError(error.WorkExhausted, artifact.createBackendArtifactPlanFromPreparedModule(allocator, state.handle(), prepared, .{}, &.{}, configuration));
3308 try std.testing.expectEqual(0, state.create_count);
3309 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3310 defer allocator.free(workspace);
3311 var plan = try artifact.createBackendArtifactPlanFromPreparedModule(allocator, state.handle(), prepared, .{}, workspace, configuration);
3312 defer plan.deinit();
3313 try std.testing.expectEqual(7, report.completed);
3314 try std.testing.expectEqual(1, plan.kernelCount());
3315 try std.testing.expectEqual(1, state.create_count);
3316 }
3317
3318 fn reportRetainedArtifactDiagnostic(
3319 _: ?*anyopaque,
3320 diagnostic: *const choir.diagnostics.Diagnostic,
3321 ) !choir.diagnostics.HandlerResult {
3322 const pretty = @import("pretty");
3323 var arena = @import("alloc_arena").Arena.init(std.testing.allocator);
3324 defer arena.deinit();
3325 var report = try pretty.diagnostic.Report.init(arena.allocator(), diagnostic.message);
3326 defer report.deinit();
3327 if (diagnostic.error_name) |name| try report.field("error", "{s}", .{name});
3328 for (diagnostic.metadata) |entry| try report.field(entry.name, "{s}", .{entry.value});
3329 pretty.diagnostic.writeStderr(&report, .{ .width = 100 });
3330 return .consumed;
3331 }
3332
3333 test "Accy retained artifact compilation preserves captured CPU ABI arguments" {
3334 if (comptime !@import("../target/root.zig").native_cpu_artifacts_supported) return error.SkipZigTest;
3335 const allocator = std.testing.allocator;
3336 const artifact = @import("../artifact/root.zig");
3337 const prepared = try savedArtifactPreparation(.{
3338 .backend_kind = .cpu,
3339 .artifact_format = .cpu_object,
3340 .dtype_bits = gpu.DTypeSet.init(&.{.f32}).bits,
3341 });
3342 defer prepared.deinit();
3343 const target = try publication.Module(.target).fromRevision(allocator, prepared.stage(.target));
3344 defer target.deinit();
3345 var captured = try target.plan(allocator, configuration.image);
3346 defer captured.deinit();
3347 const expected = captured.value.kernels[0].abi.?;
3348 try std.testing.expect(expected.static_arguments.len != 0);
3349 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3350 defer allocator.free(workspace);
3351 var state = gpu.cpu.State.init(allocator);
3352 defer state.deinit();
3353 var plan = try artifact.createBackendArtifactPlanFromPreparedModule(allocator, state.handle(), prepared, .{}, workspace, configuration);
3354 var plan_live = true;
3355 defer if (plan_live) plan.deinit();
3356 const actual = plan.kernels.items[0];
3357 try std.testing.expectEqual(expected.argument_count, actual.compile.argument_count);
3358 try std.testing.expectEqualDeep(expected.static_arguments, actual.static_arguments);
3359 try std.testing.expectEqual(expected.argument_count, actual.artifact.argument_count);
3360 const job = try artifact.ArtifactJob.init(allocator, plan);
3361 plan_live = false;
3362 defer job.deinit();
3363 const executable = @import("../executable/root.zig");
3364 const compiled = try executable.CompiledFragment.init(allocator, job);
3365 const loaded = try executable.loadFragment(allocator, state.handle(), compiled, .{ .artifact_format = .cpu_object });
3366 defer loaded.deinit();
3367 const left = [_]f32{ 1, 2, 3, 4 };
3368 const right = [_]f32{ 5, -2, 0.5, 6 };
3369 const bindings = try loaded.prepareInvocationBindings(allocator, &.{
3370 std.mem.sliceAsBytes(&left), std.mem.sliceAsBytes(&right),
3371 });
3372 defer bindings.deinit();
3373 try loaded.submitInvocationWithOptions(allocator, bindings, .{});
3374 try loaded.completeInvocationWithOptions(.{});
3375 var output: [4]f32 = undefined;
3376 try loaded.readInvocationOutput(bindings, 0, std.mem.sliceAsBytes(&output));
3377 for (output, left, right) |actual_value, lhs, rhs| {
3378 try std.testing.expectEqual(lhs + rhs + rhs, actual_value);
3379 }
3380 }
3381
3382 test "Accy retained artifact rejects incompatible stage images before rebuilding pointer maps" {
3383 const allocator = std.testing.allocator;
3384 const artifact = @import("../artifact/root.zig");
3385 var fixture = try Fixture.init();
3386 defer fixture.deinit();
3387 var captured = try captureFixture(&fixture);
3388 defer captured.deinit();
3389 var chain = try Chain.init(fixture.source.choir_module, captured);
3390 defer chain.deinit();
3391 var other = try Fixture.initNames(&.{"different_stage_program"});
3392 defer other.deinit();
3393 const changed = try seal(chain.store, chain.kinds[6], other.source.choir_module, .target, captured.target, chain.products[5]);
3394 chain.products[6].release();
3395 chain.products[6] = changed;
3396 const prepared = try preparation.pipeline.BackendPreparedModule.create(allocator, preparationRevisions(&chain));
3397 defer prepared.deinit();
3398 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3399 defer allocator.free(workspace);
3400 if (artifact.InputJob.create(workspace, prepared, configuration)) |job| {
3401 job.destroy();
3402 return error.TestExpectedError;
3403 } else |err| try std.testing.expectEqual(error.IncompatibleStageImages, err);
3404 }
3405
3406 test "Accy retained executable compiles without retaining preparation or restoration workspace" {
3407 const allocator = std.testing.allocator;
3408 const executable = @import("../executable/root.zig");
3409 var state = gpu.recording.BackendState{
3410 .allocator = allocator,
3411 .kind = .cuda,
3412 .format = .cuda_ptx,
3413 };
3414 const prepared = try savedArtifactPreparation(.{
3415 .backend_kind = .cuda,
3416 .artifact_format = .cuda_ptx,
3417 .dtype_bits = gpu.DTypeSet.init(&.{.f32}).bits,
3418 });
3419 var prepared_live = true;
3420 defer if (prepared_live) prepared.deinit();
3421 try std.testing.expectError(error.WorkExhausted, executable.compileFragmentFromPreparedModule(
3422 allocator,
3423 state.handle(),
3424 prepared,
3425 .{},
3426 &.{},
3427 configuration,
3428 ));
3429 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3430 var workspace_live = true;
3431 defer if (workspace_live) allocator.free(workspace);
3432 const compiled = try executable.compileFragmentFromPreparedModule(
3433 allocator,
3434 state.handle(),
3435 prepared,
3436 .{},
3437 workspace,
3438 configuration,
3439 );
3440 var compiled_live = true;
3441 defer if (compiled_live) compiled.deinit();
3442 prepared.deinit();
3443 prepared_live = false;
3444 @memset(workspace, 0xa5);
3445 allocator.free(workspace);
3446 workspace_live = false;
3447 const plan = compiled.artifactPlan();
3448 try std.testing.expectEqual(@as(usize, 1), plan.kernelCount());
3449 try std.testing.expectEqual(@as(usize, 2), plan.input_slot_ids.len);
3450 try std.testing.expectEqual(@as(usize, 1), plan.output_slot_ids.len);
3451 compiled_live = false;
3452 const loaded = try executable.loadFragment(allocator, state.handle(), compiled, .{});
3453 defer loaded.deinit();
3454 try std.testing.expectEqual(@as(usize, 1), state.load_count);
3455 }
3456
3457 const RejectCompilationPhase = struct {
3458 phase: []const u8,
3459
3460 fn observe(raw: ?*anyopaque, phase: []const u8, _: u64) !void {
3461 const self: *const RejectCompilationPhase = @ptrCast(@alignCast(raw.?));
3462 if (std.mem.eql(u8, self.phase, phase)) return error.CompilationObserverRejected;
3463 }
3464 };
3465
3466 test "Accy retained executable releases compiled ownership after observer failure" {
3467 const allocator = std.testing.allocator;
3468 const executable = @import("../executable/root.zig");
3469 var state = gpu.recording.BackendState{
3470 .allocator = allocator,
3471 .kind = .cuda,
3472 .format = .cuda_ptx,
3473 };
3474 const prepared = try savedArtifactPreparation(.{
3475 .backend_kind = .cuda,
3476 .artifact_format = .cuda_ptx,
3477 .dtype_bits = gpu.DTypeSet.init(&.{.f32}).bits,
3478 });
3479 defer prepared.deinit();
3480 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3481 defer allocator.free(workspace);
3482 for ([_][]const u8{ "plan_create_backend_artifacts", "compile_fragment" }) |phase| {
3483 var observer = RejectCompilationPhase{ .phase = phase };
3484 try std.testing.expectError(error.CompilationObserverRejected, executable.compileFragmentFromPreparedModule(
3485 allocator,
3486 state.handle(),
3487 prepared,
3488 .{ .instrumentation = .{ .context = &observer, .observe = RejectCompilationPhase.observe } },
3489 workspace,
3490 configuration,
3491 ));
3492 }
3493 const retry = try executable.compileFragmentFromPreparedModule(
3494 allocator,
3495 state.handle(),
3496 prepared,
3497 .{},
3498 workspace,
3499 configuration,
3500 );
3501 defer retry.deinit();
3502 try std.testing.expectEqual(@as(usize, 1), retry.kernelCount());
3503 try std.testing.expectEqual(@as(usize, 0), state.load_count);
3504 }
3505
3506 const exec_compiler = @import("../executable/root.zig");
3507
3508 fn compilationRequest(workspace: []u8) exec_compiler.FragmentCompilationRequest {
3509 const request = preparationRequest();
3510 return .{
3511 .source = request.source,
3512 .work = request.work,
3513 .record_bytes = request.record_bytes,
3514 .artifact_workspace = workspace,
3515 };
3516 }
3517
3518 fn compileCacheFixture(
3519 cache: *exec_compiler.FragmentCompilerCache,
3520 name: []const u8,
3521 request: exec_compiler.FragmentCompilationRequest,
3522 report: *preparation.publication.PreparationReport,
3523 ) !exec_compiler.FragmentCompilerCacheUpdate {
3524 var fixture = try Fixture.initNames(&.{name});
3525 fixture.cache.deinit();
3526 return cache.refreshFromSemanticModule(fixture.source, .{}, request, report, configuration);
3527 }
3528
3529 const PreparationPhases = struct {
3530 seen: u8 = 0,
3531
3532 fn observe(raw: ?*anyopaque, phase: []const u8, _: u64) !void {
3533 if (!std.mem.startsWith(u8, phase, "run_")) return;
3534 const self: *PreparationPhases = @ptrCast(@alignCast(raw.?));
3535 const expected = [_][]const u8{
3536 "run_contract_pipeline", "run_tensor_pipeline", "run_dispatch_pipeline",
3537 "run_memory_pipeline", "run_kernel_pipeline", "run_target_pipeline",
3538 };
3539 try std.testing.expect(self.seen < expected.len);
3540 try std.testing.expectEqualStrings(expected[self.seen], phase);
3541 self.seen += 1;
3542 }
3543 };
3544
3545 test "Accy executable compiler admits warm stages and compiles a fresh native job" {
3546 const allocator = std.testing.allocator;
3547 var state = gpu.recording.BackendState{
3548 .allocator = allocator,
3549 .kind = .cuda,
3550 .format = .cuda_ptx,
3551 };
3552 var cache = exec_compiler.FragmentCompilerCache.init(allocator, state.handle());
3553 defer cache.deinit();
3554 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3555 defer allocator.free(workspace);
3556 var request = compilationRequest(workspace);
3557 var cold = preparation.publication.PreparationReport{};
3558 defer cold.deinit();
3559 var first = try compileCacheFixture(&cache, "compiler_transaction", request, &cold);
3560 defer first.deinit(allocator);
3561 try std.testing.expect(!first.preparation.reused);
3562 try std.testing.expectEqual(@as(usize, 1), state.load_count);
3563 request.work.allowance = try cold.charged();
3564 var warm = preparation.publication.PreparationReport{};
3565 defer warm.deinit();
3566 var phases = PreparationPhases{};
3567 var second = try cache.refreshFromSemanticRevision(
3568 first.preparation.prepared.stage(.semantic),
3569 .{ .instrumentation = .{ .context = &phases, .observe = PreparationPhases.observe } },
3570 request,
3571 &warm,
3572 configuration,
3573 );
3574 defer second.deinit(allocator);
3575 try std.testing.expect(second.preparation.reused);
3576 try std.testing.expectEqualDeep(try cold.charged(), try warm.charged());
3577 for (warm.stages[0..warm.completed]) |stage| try std.testing.expect(stage == .warm);
3578 try std.testing.expect(first.preparation.prepared.stage(.target).eql(second.preparation.prepared.stage(.target)));
3579 try std.testing.expectEqual(@as(usize, 7), second.preparation.refresh.productCount());
3580 try std.testing.expectEqual(@as(usize, 2), state.load_count);
3581 try std.testing.expectEqual(@as(usize, 1), state.destroy_count);
3582 try std.testing.expect(first.fragment != second.fragment);
3583 try std.testing.expectEqual(second.fragment, cache.currentFragment().?);
3584 try std.testing.expectEqual(@as(u8, 6), phases.seen);
3585 }
3586
3587 test "Accy executable compiler preserves both prior owners after native load failure" {
3588 const allocator = std.testing.allocator;
3589 var state = gpu.recording.BackendState{
3590 .allocator = allocator,
3591 .kind = .cuda,
3592 .format = .cuda_ptx,
3593 };
3594 var cache = exec_compiler.FragmentCompilerCache.init(allocator, state.handle());
3595 defer cache.deinit();
3596 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3597 defer allocator.free(workspace);
3598 const request = compilationRequest(workspace);
3599 var cold = preparation.publication.PreparationReport{};
3600 defer cold.deinit();
3601 var first = try compileCacheFixture(&cache, "original_compiler_source", request, &cold);
3602 defer first.deinit(allocator);
3603 const previous = cache.currentPrepared().?;
3604 state.fail_load_after_count = 1;
3605 var failed = preparation.publication.PreparationReport{};
3606 defer failed.deinit();
3607 try std.testing.expectError(error.RuntimeUnavailable, compileCacheFixture(
3608 &cache,
3609 "changed_compiler_source",
3610 request,
3611 &failed,
3612 ));
3613 try std.testing.expectEqual(@as(usize, 7), failed.completed);
3614 try std.testing.expect(!failed.stages[0].record().eql(first.preparation.prepared.stage(.semantic)));
3615 try std.testing.expectEqual(previous, cache.currentPrepared().?);
3616 try std.testing.expectEqual(first.fragment, cache.currentFragment().?);
3617 try std.testing.expectEqual(@as(usize, 0), state.destroy_count);
3618 state.fail_load_after_count = null;
3619 var retried = preparation.publication.PreparationReport{};
3620 defer retried.deinit();
3621 var second = try compileCacheFixture(&cache, "changed_compiler_source", request, &retried);
3622 defer second.deinit(allocator);
3623 try std.testing.expect(!second.preparation.reused);
3624 try std.testing.expectEqual(@as(usize, 2), state.load_count);
3625 try std.testing.expectEqual(@as(usize, 1), state.destroy_count);
3626 }
3627
3628 test "Accy executable compiler exhaustion stops before replacing or loading native state" {
3629 const allocator = std.testing.allocator;
3630 var state = gpu.recording.BackendState{
3631 .allocator = allocator,
3632 .kind = .cuda,
3633 .format = .cuda_ptx,
3634 };
3635 var cache = exec_compiler.FragmentCompilerCache.init(allocator, state.handle());
3636 defer cache.deinit();
3637 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3638 defer allocator.free(workspace);
3639 var request = compilationRequest(workspace);
3640 var cold = preparation.publication.PreparationReport{};
3641 defer cold.deinit();
3642 var first = try compileCacheFixture(&cache, "compiler_exhaustion", request, &cold);
3643 defer first.deinit(allocator);
3644 const previous = cache.currentPrepared().?;
3645 request.work.allowance = .{};
3646 var failed = preparation.publication.PreparationReport{};
3647 defer failed.deinit();
3648 try std.testing.expectError(error.WorkExhausted, cache.refreshFromSemanticRevision(
3649 previous.stage(.semantic),
3650 .{},
3651 request,
3652 &failed,
3653 configuration,
3654 ));
3655 try std.testing.expectEqual(previous, cache.currentPrepared().?);
3656 try std.testing.expectEqual(first.fragment, cache.currentFragment().?);
3657 try std.testing.expectEqual(@as(usize, 1), state.load_count);
3658 try std.testing.expectEqual(@as(usize, 0), state.destroy_count);
3659 }
3660
3661 const tensor_compiler = @import("../tensor/root.zig");
3662
3663 fn tensorCompilerProgram(name: []const u8) !tensor_compiler.Graph {
3664 var builder = try tensor_compiler.Builder.init(std.testing.allocator, name);
3665 errdefer builder.deinit();
3666 const left = try builder.input(.f32, .{ .lane = 4 });
3667 const right = try builder.input(.f32, .{ .lane = 4 });
3668 return builder.finish(&.{try left.add(right)});
3669 }
3670
3671 test "Accy tensor compiler admits exact stages from independent program drafts" {
3672 const allocator = std.testing.allocator;
3673 var state = gpu.recording.BackendState{
3674 .allocator = allocator,
3675 .kind = .cuda,
3676 .format = .cuda_ptx,
3677 };
3678 var cache = tensor_compiler.FragmentCompilerCache.init(allocator, state.handle());
3679 defer cache.deinit();
3680 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3681 defer allocator.free(workspace);
3682 const request = compilationRequest(workspace);
3683 var first_program = try tensorCompilerProgram("tensor_compiler");
3684 defer first_program.deinit();
3685 var first_report = preparation.publication.PreparationReport{};
3686 defer first_report.deinit();
3687 var first = try cache.refreshFromProgram(&first_program, .{}, request, &first_report, configuration);
3688 defer first.deinit(allocator);
3689 var second_program = try tensorCompilerProgram("tensor_compiler");
3690 defer second_program.deinit();
3691 var second_report = preparation.publication.PreparationReport{};
3692 defer second_report.deinit();
3693 var second = try cache.refreshFromProgram(&second_program, .{}, request, &second_report, configuration);
3694 defer second.deinit(allocator);
3695 try std.testing.expect(second.preparation.reused);
3696 try std.testing.expect(first.preparation.prepared.stage(.semantic).eql(second.preparation.prepared.stage(.semantic)));
3697 try std.testing.expectEqualDeep(try first_report.charged(), try second_report.charged());
3698 try std.testing.expectEqual(@as(usize, 7), second.preparation.refresh.productCount());
3699 try std.testing.expectEqual(@as(usize, 2), state.load_count);
3700 try std.testing.expectEqual(@as(usize, 1), state.destroy_count);
3701 try std.testing.expectEqual(second.fragment, cache.currentFragment().?);
3702 }
3703
3704 test "Accy tensor compiler respects lowering capacity on repeated input and preserves prior state" {
3705 const allocator = std.testing.allocator;
3706 var state = gpu.recording.BackendState{
3707 .allocator = allocator,
3708 .kind = .cuda,
3709 .format = .cuda_ptx,
3710 };
3711 var cache = tensor_compiler.FragmentCompilerCache.init(allocator, state.handle());
3712 defer cache.deinit();
3713 const workspace = try allocator.alloc(u8, 256 * 1024 * 1024);
3714 defer allocator.free(workspace);
3715 const request = compilationRequest(workspace);
3716 var program = try tensorCompilerProgram("tensor_compiler_bounds");
3717 defer program.deinit();
3718 var first_report = preparation.publication.PreparationReport{};
3719 defer first_report.deinit();
3720 var first = try cache.refreshFromProgram(&program, .{}, request, &first_report, configuration);
3721 defer first.deinit(allocator);
3722 const previous = cache.currentPrepared().?;
3723 var options = exec_compiler.FragmentCompilerOptions{};
3724 options.semantic_context_limits.operations.storage_bytes = 1;
3725 var failed = preparation.publication.PreparationReport{};
3726 defer failed.deinit();
3727 try std.testing.expectError(error.OutOfMemory, cache.refreshFromProgram(
3728 &program,
3729 options,
3730 request,
3731 &failed,
3732 configuration,
3733 ));
3734 try std.testing.expectEqual(@as(u8, 0), failed.completed);
3735 try std.testing.expectEqual(previous, cache.currentPrepared().?);
3736 try std.testing.expectEqual(first.fragment, cache.currentFragment().?);
3737 try std.testing.expectEqual(@as(usize, 1), state.load_count);
3738 var retried = preparation.publication.PreparationReport{};
3739 defer retried.deinit();
3740 var retry = try cache.refreshFromProgram(&program, .{}, request, &retried, configuration);
3741 defer retry.deinit(allocator);
3742 try std.testing.expect(retry.preparation.reused);
3743 try std.testing.expectEqual(@as(usize, 2), state.load_count);
3744 }
3745
3746 fn tensorScatterCompilerProgram() !tensor_compiler.Graph {
3747 var builder = try tensor_compiler.Builder.init(std.testing.allocator, "tensor_scatter_compiler");
3748 errdefer builder.deinit();
3749 const table = try builder.input(.f32, .{ .vocab = 16, .channel = 4 });
3750 const ids = try builder.input(.i32, .{ .token = 3 });
3751 const gathered = try table.gather(ids, .vocab);
3752 const zero = try builder.full(.f32, .{ .vocab = 16, .channel = 4 }, 0.0);
3753 return builder.finish(&.{try zero.scatterAdd(ids, gathered, .vocab)});
3754 }
3755
3756 fn expectTensorKernelCall(
3757 prepared: *const preparation.pipeline.BackendPreparedModule,
3758 expected: ?[]const u8,
3759 ) !void {
3760 const allocator = std.testing.allocator;
3761 const module = try publication.Module(.tensor).fromRevision(allocator, prepared.stage(.tensor));
3762 defer module.deinit();
3763 const image = try module.open(allocator, configuration.image);
3764 defer image.destroy();
3765 var context = try choir.ir.Context.init(allocator, configuration.context);
3766 defer context.deinit(allocator);
3767 try registerContext(&context);
3768 var decoded = try choir.bytecode.decodeModule(allocator, &context, image.root(0).?.bytes);
3769 defer decoded.deinit();
3770 const operation_name = @import("../choir/root.zig").dialect.AccyDialect.KernelCallOp.operation_name;
3771 try std.testing.expectEqual(@as(usize, if (expected != null) 1 else 0), choir.ir.inspection.countOperationsNamed(decoded.module, operation_name));
3772 if (expected) |name| {
3773 const call = choir.ir.inspection.findOperationNamed(decoded.module, operation_name).?;
3774 const target = call.getAttr("target").?.cast(choir.ir.Attribute.DialectAttr).?;
3775 try std.testing.expectEqualStrings(name, target.payload);
3776 }
3777 }
3778
3779 test "Accy tensor compiler captures scatter mode changes and current catalog targets" {
3780 const allocator = std.testing.allocator;
3781 const library = @import("../kernel/root.zig").library;
3782 var state = gpu.recording.BackendState{
3783 .allocator = allocator,
3784 .kind = .cuda,
3785 .format = .cuda_ptx,
3786 };
3787 var cache = tensor_compiler.FragmentCompilerCache.init(allocator, state.handle());
3788 defer cache.deinit();
3789 const workspace = try allocator.alloc(u8, 1024 * 1024 * 1024);
3790 defer allocator.free(workspace);
3791 var request = compilationRequest(workspace);
3792 request.work.workspace = 2 * 1024 * 1024 * 1024;
3793 var program = try tensorScatterCompilerProgram();
3794 defer program.deinit();
3795 var family4 = try library.indexing.createScatterAddFamilyArtifact(allocator, state.handle(), .{
3796 .axis_size = 16,
3797 .updates = 3,
3798 .inner = 4,
3799 .dtype = .f32,
3800 .threads = 4,
3801 }, .{ .limits = .testing, .format = .cuda_ptx });
3802 defer family4.deinit();
3803 var family8 = try library.indexing.createScatterAddFamilyArtifact(allocator, state.handle(), .{
3804 .axis_size = 16,
3805 .updates = 3,
3806 .inner = 4,
3807 .dtype = .f32,
3808 .threads = 8,
3809 }, .{ .limits = .testing, .format = .cuda_ptx });
3810 defer family8.deinit();
3811 const registry4 = family4.registry();
3812 const registry8 = family8.registry();
3813 for ([_]?u32{ null, 4, 4, 8, null }, 0..) |threads, index| {
3814 var options = exec_compiler.FragmentCompilerOptions{};
3815 if (threads) |count| {
3816 options.kernel_call_registry = if (count == 4) ®istry4 else ®istry8;
3817 options.scatter_add_schedule = .{ .thread_blocks = count };
3818 }
3819 var report = preparation.publication.PreparationReport{};
3820 defer report.deinit();
3821 var update = cache.refreshFromProgram(&program, options, request, &report, configuration) catch |err| {
3822 try reportPreparationTestFailure(err, &report);
3823 return err;
3824 };
3825 defer update.deinit(allocator);
3826 try std.testing.expectEqual(index == 2, update.preparation.reused);
3827 const target: ?[]const u8 = if (threads) |count|
3828 (if (count == 4) "accy.kernel.indexing.scatter_add_family_4_f32" else "accy.kernel.indexing.scatter_add_family_8_f32")
3829 else
3830 null;
3831 try expectTensorKernelCall(update.preparation.prepared, target);
3832 if (index < 2) try expectRestoredTensorLookups(update.preparation.prepared, workspace, index == 1);
3833 }
3834 }
3835
3836 fn expectRestoredTensorLookups(
3837 prepared: *const preparation.pipeline.BackendPreparedModule,
3838 workspace: []u8,
3839 aliases: bool,
3840 ) !void {
3841 const input = try @import("../artifact/root.zig").InputJob.create(workspace, prepared, configuration);
3842 defer input.destroy();
3843 const plans = input.plans();
3844 var reordered = false;
3845 for (plans.schedule.work_items.items, 0..) |work, index| {
3846 const found = plans.schedule.getWorkForRoot(work.root) orelse return error.MissingWorkLookup;
3847 try std.testing.expectEqual(work.id, found.id);
3848 reordered = reordered or work.id != index;
3849 }
3850 if (!aliases) try std.testing.expect(reordered);
3851 if (aliases) {
3852 const name = @import("../choir/root.zig").dialect.AccyDialect.KernelCallOp.operation_name;
3853 const call = choir.ir.inspection.findOperationNamed(plans.root, name).?;
3854 const value = call.getResult(0).?;
3855 const slot = plans.buffers.getSlot(value) orelse return error.MissingAliasLookup;
3856 try std.testing.expect(slot.value != value);
3857 try std.testing.expect(plans.buffers.value_to_slot.count() > plans.buffers.slots.items.len);
3858 }
3859 }
3860
3861 fn reportPreparationTestFailure(err: anyerror, report: *const preparation.publication.PreparationReport) !void {
3862 const pretty = @import("pretty");
3863 var arena = @import("alloc_arena").Arena.init(std.testing.allocator);
3864 defer arena.deinit();
3865 var output = try pretty.diagnostic.Report.init(arena.allocator(), "retained preparation failed");
3866 defer output.deinit();
3867 try output.field("error", "{s}", .{@errorName(err)});
3868 try output.field("completed stages", "{d}", .{report.completed});
3869 if (report.failure) |failure| {
3870 try output.field("workspace limit", "{d}", .{failure.work.limits.workspace});
3871 try output.field("maximum live storage", "{d}", .{failure.work.maximum_live_storage});
3872 try output.field("exceeded", "{any}", .{failure.work.exceeded});
3873 for (failure.work.events) |event| {
3874 if (event.outcome == .success) continue;
3875 try output.field(event.identity().name, "outcome={s}, admitted={}, workspace={d}, charged={any}", .{
3876 @tagName(event.outcome), event.admitted, event.workspace, event.charged,
3877 });
3878 }
3879 }
3880 pretty.diagnostic.writeStderr(&output, .{ .width = 100 });
3881 }
3882
3883 test "Accy tensor compiler Dispatch records preserve reordered IDs and reject counter drift" {
3884 const allocator = std.testing.allocator;
3885 var fixture = try Fixture.initNames(&.{ "first", "second" });
3886 defer fixture.deinit();
3887 var captured = try captureFixture(&fixture);
3888 defer captured.deinit();
3889 var decoded = try records.codec.decode(allocator, records.dispatch.Record, .dispatch, captured.dispatch);
3890 defer decoded.deinit();
3891 const items = @constCast(decoded.value.schedule.work_items);
3892 try std.testing.expectEqual(@as(usize, 2), items.len);
3893 std.mem.swap(records.dispatch.WorkItem, &items[0], &items[1]);
3894 try records.dispatch.validate(allocator, decoded.value);
3895 const id = items[1].id;
3896 items[1].id = items[0].id;
3897 try std.testing.expectError(error.InvalidStageRecord, records.dispatch.validate(allocator, decoded.value));
3898 items[1].id = items.len;
3899 try std.testing.expectError(error.InvalidStageRecord, records.dispatch.validate(allocator, decoded.value));
3900 items[1].id = id;
3901 decoded.value.schedule.total_static_elements += 1;
3902 try std.testing.expectError(error.InvalidStageRecord, records.dispatch.validate(allocator, decoded.value));
3903 decoded.value.schedule.total_static_elements -= 1;
3904 decoded.value.fusion.fused_op_count += 1;
3905 try std.testing.expectError(error.InvalidStageRecord, records.dispatch.validate(allocator, decoded.value));
3906 }
3907
3908 test "Accy tensor compiler Memory bindings require canonical order and complete primary values" {
3909 const allocator = std.testing.allocator;
3910 var fixture = try Fixture.init();
3911 defer fixture.deinit();
3912 var captured = try captureFixture(&fixture);
3913 defer captured.deinit();
3914 var memory = try records.codec.decode(allocator, records.memory.Record, .memory, captured.memory);
3915 defer memory.deinit();
3916 var dispatch = try records.codec.decode(allocator, records.dispatch.Record, .dispatch, captured.dispatch);
3917 defer dispatch.deinit();
3918 try records.memory.validate(allocator, memory.value, dispatch.value);
3919 const bindings = @constCast(memory.value.bindings);
3920 try std.testing.expect(bindings.len > 1);
3921 std.mem.swap(records.memory.Binding, &bindings[0], &bindings[1]);
3922 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, memory.value, dispatch.value));
3923 std.mem.swap(records.memory.Binding, &bindings[0], &bindings[1]);
3924 const saved = bindings[1];
3925 bindings[1] = bindings[0];
3926 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, memory.value, dispatch.value));
3927 bindings[1] = saved;
3928 bindings[1].slot_id = memory.value.buffers.slots.len;
3929 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, memory.value, dispatch.value));
3930 bindings[1] = saved;
3931 memory.value.bindings = bindings[0 .. bindings.len - 1];
3932 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, memory.value, dispatch.value));
3933 }
3934
3935 test "Accy record counters reject every Memory aggregate independently" {
3936 const allocator = std.testing.allocator;
3937 var fixture = try Fixture.init();
3938 defer fixture.deinit();
3939 var captured = try captureFixture(&fixture);
3940 defer captured.deinit();
3941 var memory = try records.codec.decode(allocator, records.memory.Record, .memory, captured.memory);
3942 defer memory.deinit();
3943 var dispatch = try records.codec.decode(allocator, records.dispatch.Record, .dispatch, captured.dispatch);
3944 defer dispatch.deinit();
3945 try records.memory.validate(allocator, memory.value, dispatch.value);
3946 inline for (.{
3947 "input_slot_count", "output_slot_count", "temporary_slot_count",
3948 "constant_slot_count", "dynamic_slot_count", "total_static_bytes",
3949 }) |field| {
3950 var changed = memory.value;
3951 @field(changed.buffers, field) += 1;
3952 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, changed, dispatch.value));
3953 }
3954 inline for (.{
3955 "host_slot_count", "device_global_slot_count", "device_constant_slot_count",
3956 "device_shared_slot_count", "unified_slot_count", "host_input_transfer_count",
3957 "host_output_transfer_count", "dynamic_slot_count", "elided_value_count",
3958 "total_static_bytes",
3959 }) |field| {
3960 var changed = memory.value;
3961 @field(changed.spaces, field) += 1;
3962 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, changed, dispatch.value));
3963 }
3964 inline for (.{
3965 "scalar_layout_count", "row_major_layout_count", "dynamic_row_major_layout_count",
3966 "host_slot_count", "device_global_slot_count", "device_constant_slot_count",
3967 "device_shared_slot_count", "unified_slot_count", "dynamic_slot_count",
3968 "elided_value_count", "total_static_bytes",
3969 }) |field| {
3970 var changed = memory.value;
3971 @field(changed.layouts, field) += 1;
3972 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, changed, dispatch.value));
3973 }
3974 }
3975
3976 test "Accy record counters reject Memory byte total overflow" {
3977 const allocator = std.testing.allocator;
3978 var fixture = try Fixture.init();
3979 defer fixture.deinit();
3980 var captured = try captureFixture(&fixture);
3981 defer captured.deinit();
3982 var memory = try records.codec.decode(allocator, records.memory.Record, .memory, captured.memory);
3983 defer memory.deinit();
3984 var dispatch = try records.codec.decode(allocator, records.dispatch.Record, .dispatch, captured.dispatch);
3985 defer dispatch.deinit();
3986 const slots = @constCast(memory.value.buffers.slots);
3987 try std.testing.expect(slots.len >= 2);
3988 slots[0].byte_size = std.math.maxInt(u64);
3989 slots[1].byte_size = 1;
3990 try std.testing.expectError(error.InvalidStageRecord, records.memory.validate(allocator, memory.value, dispatch.value));
3991 }
3992
3993 test "Accy record counters recompute Target runtime scalars from the captured parameters" {
3994 const allocator = std.testing.allocator;
3995 var fixture = try Fixture.init();
3996 defer fixture.deinit();
3997 var captured = try captureFixture(&fixture);
3998 defer captured.deinit();
3999 var target = try records.codec.decode(allocator, records.target.Record, .target, captured.target);
4000 defer target.deinit();
4001 var parent = try records.codec.decode(allocator, records.kernel.Record, .kernel, captured.kernel);
4002 defer parent.deinit();
4003 try records.target.validate(allocator, target.value, parent.value);
4004 const kernels = @constCast(target.value.kernels);
4005 try std.testing.expect(kernels.len != 0);
4006 kernels[0].runtime_scalar_argument_count += 1;
4007 try std.testing.expectError(error.InvalidStageRecord, records.target.validate(allocator, target.value, parent.value));
4008 }
4009
4010 fn registerTestContext(context: *choir.ir.Context) !void {
4011 try choir.dialects.registerChoirDialect(context);
4012 try @import("../choir/root.zig").registerAccyDialect(context);
4013 }
4014
4015 const test_configuration = operation.Configuration{
4016 .context = choir.ir.Context.Limits.testing,
4017 .register = registerTestContext,
4018 .registration = .{ .name = "accy-stage-test-context", .version = 1 },
4019 .codec = .{ .operations = 100, .entities = 1000, .fields = 1000, .depth = 32 },
4020 .image = .{ .bytes = 65536, .entities = 1024, .depth = 32 },
4021 .roots = 1,
4022 .gate_scratch = 64 * 1024 * 1024,
4023 .verify = choir.ir.verify.default_options,
4024 };
4025
4026 const test_limits = revision.store.Limits{
4027 .revisions = 8,
4028 .kinds = 4,
4029 .builders = 4,
4030 .compiler_manifests = 2,
4031 .record_bytes = 32 * 1024 * 1024,
4032 .gate_scratch_bytes = test_configuration.gate_scratch,
4033 .candidate_count = 8,
4034 .screening_bytes = 32 * 1024 * 1024,
4035 };
4036
4037 const TestKinds = struct {
4038 semantic: *const revision.Kind,
4039 contract: *const revision.Kind,
4040 tensor: *const revision.Kind,
4041 bare: *const revision.Kind,
4042 };
4043
4044 fn testKinds(store: *revision.Store) !TestKinds {
4045 const allocator = std.testing.allocator;
4046 const kinds = TestKinds{
4047 .semantic = try @import("root.zig").publication.registerKind(
4048 allocator,
4049 store,
4050 .semantic,
4051 test_configuration,
4052 ),
4053 .contract = try @import("root.zig").publication.registerKind(
4054 allocator,
4055 store,
4056 .contract,
4057 test_configuration,
4058 ),
4059 .tensor = try @import("root.zig").publication.registerKind(
4060 allocator,
4061 store,
4062 .tensor,
4063 test_configuration,
4064 ),
4065 .bare = try operation.registerKind(allocator, store, test_configuration, .{
4066 .name = "accy-bare-test",
4067 .version = 1,
4068 }, &.{}),
4069 };
4070 store.freeze();
4071 return kinds;
4072 }
4073
4074 fn testBegin(
4075 store: *revision.Store,
4076 kind: *const revision.Kind,
4077 comptime stage: publication.Stage,
4078 parent: ?*const revision.Revision,
4079 ) !*revision.Builder {
4080 const allocator = std.testing.allocator;
4081 const policy = try choir.product.recipe.encode(allocator, .{
4082 .arithmetic = .{},
4083 .policy = "capture-test",
4084 });
4085 defer allocator.free(policy);
4086 const dependencies: []const revision.store.Dependency = if (parent) |source|
4087 &.{.{ .role = "source", .revision = source }}
4088 else
4089 &.{};
4090 return store.begin(.{
4091 .kind = kind,
4092 .address = .{
4093 .producer = "accy",
4094 .source = "sealed-stage-test",
4095 .stage = stage.name(),
4096 .variant = "",
4097 },
4098 .inputs = .{
4099 .compiler_manifest = try choir.product.compiler.manifest(),
4100 .versions = &.{ stage.schema(), operation.schema_identity },
4101 .pipeline = &.{},
4102 .options = &.{},
4103 .policy = policy,
4104 },
4105 .dependencies = dependencies,
4106 .parent = parent,
4107 }, .{
4108 .allowance = revision.WorkVector.uniform(1024 * 1024 * 1024),
4109 .workspace = 128 * 1024 * 1024,
4110 .events = 64,
4111 });
4112 }
4113
4114 fn abortTest(builder: *revision.Builder) void {
4115 if (builder.abort(.rejected)) |failure| {
4116 var owned = failure;
4117 owned.deinit();
4118 }
4119 }
4120
4121 fn testSource(
4122 store: *revision.Store,
4123 kind: *const revision.Kind,
4124 foreign: bool,
4125 ) !operation.Product {
4126 const allocator = std.testing.allocator;
4127 var source = try semantic.Builder.init(allocator, .standard);
4128 defer source.deinit();
4129 const typ = try source.tensor(.f32, &.{4});
4130 var function = try source.beginFunction("sealed_stage", &.{ typ, typ }, &.{typ});
4131 const sum = try function.add(function.parameter(0), function.parameter(1));
4132 try function.return_(&.{sum});
4133 try function.finish();
4134 const module = try source.finish();
4135 defer module.deinit();
4136 if (foreign) try addForeignOperation(module);
4137 const builder = try testBegin(store, kind, .semantic, null);
4138 var open = true;
4139 errdefer if (open) abortTest(builder);
4140 try module.capture(builder, test_configuration);
4141 try module.choir_module.setAttr("after_capture", try module.context().getStringAttr("changed"));
4142 const product = try operation.Product.seal(builder, kind);
4143 open = false;
4144 return product;
4145 }
4146
4147 fn addForeignOperation(module: *@import("../choir/root.zig").semantic.SemanticModule) !void {
4148 const dialect = choir.dialects;
4149 const typ = try dialect.ArithDialect.getScalarType(module.context(), .f32);
4150 const memref = try dialect.MemrefDialect.getMemrefType1D(module.context(), 4, typ, .device);
4151 const alloc = try dialect.MemrefDialect.AllocOp.createStatic(
4152 module.context(),
4153 .unknown,
4154 memref,
4155 );
4156 try module.choir_module.getRegion(0).?.getEntryBlock().?.addOperation(alloc.op);
4157 }
4158
4159 fn testSuccessor(
4160 store: *revision.Store,
4161 kind: *const revision.Kind,
4162 comptime stage: publication.Stage,
4163 parent: operation.Product,
4164 bytes: []const u8,
4165 ) !operation.Product {
4166 const builder = try testBegin(store, kind, stage, parent.revision);
4167 var open = true;
4168 errdefer if (open) abortTest(builder);
4169 const job = try parent.successor(builder, test_configuration);
4170 defer job.destroy();
4171 try job.capture(builder, bytes, &.{}, test_configuration);
4172 const product = try operation.Product.seal(builder, kind);
4173 open = false;
4174 return product;
4175 }
4176
4177 test "sealed Accy IR stages retain immutable images after source destruction" {
4178 const allocator = std.testing.allocator;
4179 const store = try revision.Store.create(allocator, test_limits);
4180 defer store.release();
4181 const kinds = try testKinds(store);
4182 const source = try testSource(store, kinds.semantic, false);
4183 defer source.release();
4184 const contract = try testSuccessor(
4185 store,
4186 kinds.contract,
4187 .contract,
4188 source,
4189 &publication.irRecord(.contract),
4190 );
4191 defer contract.release();
4192 const tensor = try testSuccessor(store, kinds.tensor, .tensor, contract, &publication.irRecord(.tensor));
4193 defer tensor.release();
4194 const handle = try publication.Module(.tensor).fromRevision(allocator, tensor.revision);
4195 defer handle.deinit();
4196 const retained = try handle.retain(allocator);
4197 defer retained.deinit();
4198 try std.testing.expect(handle.eql(retained));
4199 const image = try retained.open(allocator, test_configuration.image);
4200 defer image.destroy();
4201 try std.testing.expectEqualStrings("builtin.module", image.root(0).?.operations[0].name);
4202 try std.testing.expectEqualSlices(u8, &publication.irRecord(.tensor), image.stage());
4203 try std.testing.expect(std.mem.indexOf(u8, image.root(0).?.bytes, "after_capture") == null);
4204 try std.testing.expectError(
4205 error.WrongStage,
4206 publication.Module(.contract).fromRevision(allocator, tensor.revision),
4207 );
4208 }
4209
4210 test "sealed Accy stages reject missing gates and wrong predecessor records" {
4211 const allocator = std.testing.allocator;
4212 const store = try revision.Store.create(allocator, test_limits);
4213 defer store.release();
4214 const kinds = try testKinds(store);
4215 const source = try testSource(store, kinds.semantic, false);
4216 defer source.release();
4217 const bare = try testSuccessor(store, kinds.bare, .contract, source, &publication.irRecord(.contract));
4218 defer bare.release();
4219 try std.testing.expectError(
4220 error.MissingGateEvidence,
4221 publication.Module(.contract).fromRevision(allocator, bare.revision),
4222 );
4223 try std.testing.expectError(
4224 error.InvalidStageDependency,
4225 testSuccessor(store, kinds.tensor, .tensor, source, &publication.irRecord(.tensor)),
4226 );
4227 try std.testing.expectError(
4228 error.InvalidStageRecord,
4229 testSuccessor(store, kinds.contract, .contract, source, &publication.irRecord(.tensor)),
4230 );
4231 }
4232
4233 test "sealed Accy Contract rejects unsupported dialects" {
4234 const allocator = std.testing.allocator;
4235 const store = try revision.Store.create(allocator, test_limits);
4236 defer store.release();
4237 const kinds = try testKinds(store);
4238 const source = try testSource(store, kinds.semantic, true);
4239 defer source.release();
4240 try std.testing.expectError(
4241 error.UnsupportedDialect,
4242 testSuccessor(store, kinds.contract, .contract, source, &publication.irRecord(.contract)),
4243 );
4244 }
4245
4246 fn checkHandleFailure(allocator: std.mem.Allocator, published: *const revision.Revision) !void {
4247 const handle = try publication.Module(.semantic).fromRevision(allocator, published);
4248 defer handle.deinit();
4249 const image = try handle.open(allocator, test_configuration.image);
4250 defer image.destroy();
4251 try std.testing.expectEqualStrings("builtin.module", image.root(0).?.operations[0].name);
4252 }
4253
4254 test "sealed Accy module handles clean allocation failures" {
4255 const allocator = std.testing.allocator;
4256 const store = try revision.Store.create(allocator, test_limits);
4257 defer store.release();
4258 const kinds = try testKinds(store);
4259 const source = try testSource(store, kinds.semantic, false);
4260 defer source.release();
4261 try std.testing.checkAllAllocationFailures(allocator, checkHandleFailure, .{source.revision});
4262 }
4263
4264 test "sealed Accy stages require predecessor gate evidence beyond its address" {
4265 const allocator = std.testing.allocator;
4266 const store = try revision.Store.create(allocator, test_limits);
4267 defer store.release();
4268 const kinds = try testKinds(store);
4269 const bare_source = try testSource(store, kinds.bare, false);
4270 defer bare_source.release();
4271 try std.testing.expectError(
4272 error.MissingGateEvidence,
4273 testSuccessor(store, kinds.contract, .contract, bare_source, &publication.irRecord(.contract)),
4274 );
4275 const source = try testSource(store, kinds.semantic, false);
4276 defer source.release();
4277 const bare_contract = try testSuccessor(
4278 store,
4279 kinds.bare,
4280 .contract,
4281 source,
4282 &publication.irRecord(.contract),
4283 );
4284 defer bare_contract.release();
4285 try std.testing.expectError(
4286 error.MissingGateEvidence,
4287 testSuccessor(store, kinds.tensor, .tensor, bare_contract, &publication.irRecord(.tensor)),
4288 );
4289 }