lib/accy/src/validation/conformance/harness.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const gpu = @import("gpu");
3 const choir_abi = @import("choir_abi");
4 const pretty = @import("pretty");
5 const accy = @import("accy");
6 const choir = @import("choir");
7 const sys = @import("sys");
8 const ptx = @import("accy_validation_ptx");
9 const conformance = @import("root.zig");
10
11 const records = conformance.records;
12 const kernel = accy.kernel;
13
14 pub const CudaState = gpu.cuda.State;
15 pub const MetalState = gpu.metal.State;
16 pub const VulkanState = gpu.vulkan.State;
17 pub const BackendHandle = gpu.BackendHandle;
18 pub const SemanticBuilder = accy.choir.SemanticBuilder;
19 pub const SemanticModule = accy.choir.SemanticModule;
20 pub const FunctionBuilder = accy.choir.semantic.FunctionBuilder;
21 pub const Value = choir.ir.Value;
22 pub const DType = choir_abi.DType;
23 pub const Bf16 = choir_abi.Bf16;
24
25 pub const Expectation = enum { verified, unsupported, invalid };
26
27 pub const Mode = enum { execute, assemble };
28
29 pub const Assembler = struct {
30 dir: []const u8,
31 target: []const u8,
32 };
33
34 pub const Gate = struct {
35 mode: Mode,
36 assembler: ?Assembler,
37 };
38
39 pub const Tensor = struct {
40 dtype: DType,
41 dims: []const i64,
42
43 pub fn elementCount(comptime self: Tensor) usize {
44 var count: usize = 1;
45 for (self.dims) |dim| count *= @as(usize, @intCast(dim));
46 return count;
47 }
48
49 pub fn byteCount(comptime self: Tensor) usize {
50 return self.elementCount() * self.dtype.sizeOf();
51 }
52 };
53
54 pub fn vec(comptime dtype: DType, comptime len: i64) Tensor {
55 return .{ .dtype = dtype, .dims = &.{len} };
56 }
57
58 pub fn mat(comptime dtype: DType, comptime rows: i64, comptime cols: i64) Tensor {
59 return .{ .dtype = dtype, .dims = &.{ rows, cols } };
60 }
61
62 pub fn requiredDTypes(comptime Spec: type) gpu.DTypeSet {
63 var set: gpu.DTypeSet = .{};
64 if (comptime @hasDecl(Spec, "buffers")) {
65 inline for (Spec.buffers) |buffer| set.insert(buffer.tensor.dtype);
66 } else {
67 inline for (Spec.inputs) |input| set.insert(input.dtype);
68 set.insert(Spec.output.dtype);
69 }
70 return set;
71 }
72
73 pub fn requiredFeatures(comptime Spec: type) choir_abi.Features {
74 if (comptime @hasDecl(Spec, "required_features")) return Spec.required_features;
75 return .{};
76 }
77
78 pub fn requiredSubgroup(comptime Spec: type) choir_abi.SubgroupRequirements {
79 if (comptime @hasDecl(Spec, "required_subgroup")) return Spec.required_subgroup;
80 return .{};
81 }
82
83 pub fn buildModule(comptime Spec: type, allocator: std.mem.Allocator) !*SemanticModule {
84 var builder = try SemanticBuilder.init(allocator, SemanticBuilder.ContextLimits.standard);
85 defer builder.deinit();
86
87 var param_types: [Spec.inputs.len]choir.ir.Type = undefined;
88 inline for (Spec.inputs, 0..) |input, index| {
89 param_types[index] = try builder.tensor(input.dtype, input.dims);
90 }
91 const result_type = try builder.tensor(Spec.output.dtype, Spec.output.dims);
92
93 var function = try builder.beginFunction("conformance_" ++ Spec.name, ¶m_types, &.{result_type});
94 const result = try Spec.body(&builder, &function);
95 try function.return_(&.{result});
96 try function.finish();
97 return try builder.finish();
98 }
99
100 pub fn runCase(
101 comptime Spec: type,
102 allocator: std.mem.Allocator,
103 handle: BackendHandle,
104 gate: Gate,
105 ) !records.Case {
106 var arena_state = std.heap.ArenaAllocator.init(allocator);
107 defer arena_state.deinit();
108 const arena = arena_state.allocator();
109
110 const compile_start = nowNs();
111 const module = try buildModule(Spec, allocator);
112 var prepared = try accy.executable.prepareFragmentFromSemanticModule(
113 allocator,
114 handle,
115 module,
116 .{},
117 );
118 defer prepared.deinit();
119 const artifact_module = try accy.executable.createArtifactJobFromPreparedJob(
120 allocator,
121 handle,
122 &prepared,
123 .{},
124 );
125 defer artifact_module.deinit();
126
127 const ptxas_status = acceptPlannedKernels(allocator, artifact_module, gate.assembler);
128 const ptxas_ok = !std.mem.eql(u8, ptxas_status, "fail");
129
130 if (gate.mode == .assemble) {
131 return .{
132 .name = Spec.name,
133 .status = if (ptxas_ok) "assembled" else "fail",
134 .detail = if (ptxas_ok) "" else "PtxasRejected",
135 .ptxas = ptxas_status,
136 .kernels = artifact_module.kernelCount(),
137 .compile_ns = elapsedNs(compile_start),
138 .launch_read_ns = 0,
139 .checksum = 0,
140 .max_abs_error = 0,
141 .tolerance = Spec.tolerance,
142 };
143 }
144
145 const compiled = try accy.executable.compileFragmentFromArtifactJob(allocator, artifact_module);
146 var fragment = try accy.executable.loadFragment(allocator, handle, compiled, .{});
147 const compile_ns = elapsedNs(compile_start);
148 defer fragment.deinit();
149
150 var input_views: [Spec.inputs.len][]const u8 = undefined;
151 inline for (Spec.inputs, 0..) |input, index| {
152 const buffer = try arena.alignedAlloc(u8, .@"16", input.byteCount());
153 fillInput(Spec, input, index, buffer);
154 input_views[index] = buffer;
155 }
156
157 const output_bytes = try arena.alignedAlloc(u8, .@"16", Spec.output.byteCount());
158 @memset(output_bytes, 0xA5);
159 var outputs = [_][]u8{output_bytes};
160
161 const launch_start = nowNs();
162 try accy.executable.invoke(fragment, allocator, allocator, &input_views, &outputs);
163 const launch_read_ns = elapsedNs(launch_start);
164
165 const expected = try arena.alignedAlloc(u8, .@"16", Spec.output.byteCount());
166 try expectedOutput(Spec, allocator, &prepared, artifact_module.artifactPlan(), &input_views, expected);
167
168 const max_error = maxAbsError(Spec.output.dtype, expected, output_bytes);
169 const numeric_ok = fragment.outputCount() == 1 and max_error <= Spec.tolerance;
170 return .{
171 .name = Spec.name,
172 .status = if (numeric_ok and ptxas_ok) "pass" else "fail",
173 .detail = if (ptxas_ok) "" else "PtxasRejected",
174 .ptxas = ptxas_status,
175 .kernels = fragment.kernelCount(),
176 .compile_ns = compile_ns,
177 .launch_read_ns = launch_read_ns,
178 .checksum = finiteOrMax(checksum(Spec.output.dtype, output_bytes)),
179 .max_abs_error = finiteOrMax(max_error),
180 .tolerance = Spec.tolerance,
181 };
182 }
183
184 fn acceptPlannedKernels(
185 allocator: std.mem.Allocator,
186 artifact_module: *accy.artifact.ArtifactJob,
187 assembler: ?Assembler,
188 ) []const u8 {
189 const active = assembler orelse return "skip";
190 const plan = artifact_module.artifactPlan();
191 for (plan.kernels.items) |planned| {
192 ptx.assemble.acceptArtifact(allocator, active.dir, active.target, planned.artifact) catch {
193 return "fail";
194 };
195 }
196 return "pass";
197 }
198
199 fn expectedOutput(
200 comptime Spec: type,
201 allocator: std.mem.Allocator,
202 prepared: *accy.preparation.BackendPreparedJob,
203 artifact_plan: *const accy.artifact.BackendArtifactPlan,
204 input_views: *const [Spec.inputs.len][]const u8,
205 output_bytes: []u8,
206 ) !void {
207 if (comptime Spec.expectation == .verified) {
208 try expectedOutputFromGeneratedKernel(Spec, allocator, prepared, artifact_plan, input_views, output_bytes);
209 return;
210 }
211 Spec.reference(input_views, output_bytes);
212 }
213
214 fn expectedOutputFromGeneratedKernel(
215 comptime Spec: type,
216 allocator: std.mem.Allocator,
217 prepared: *accy.preparation.BackendPreparedJob,
218 artifact_plan: *const accy.artifact.BackendArtifactPlan,
219 input_views: *const [Spec.inputs.len][]const u8,
220 output_bytes: []u8,
221 ) !void {
222 const product = @constCast(try prepared.kernelizationProduct());
223 if (product.kernelCount() != artifact_plan.kernelCount()) return error.UnsupportedOperation;
224 var slots = try OracleSlotTable.init(Spec, allocator, artifact_plan, input_views, output_bytes);
225 defer slots.deinit();
226
227 for (artifact_plan.kernels.items, 0..) |planned, kernel_index| {
228 if (planned.runtime_scalar_argument_count != 0 or planned.static_arguments.len != 0) return error.UnsupportedOperation;
229 var generated = &product.kernels.items[kernel_index].program;
230 const params = generated.params();
231 const has_count_argument = planned.element_count_argument != .none;
232 const expected_arg_count = 1 + planned.input_slot_ids.len + @intFromBool(has_count_argument);
233 if (params.len != expected_arg_count) return error.UnsupportedOperation;
234
235 const args = try allocator.alloc(kernel.Argument, params.len);
236 defer allocator.free(args);
237 var arg_index: usize = 0;
238 args[arg_index] = .{ .memref = try slots.bytesFor(planned.output_slot_id) };
239 arg_index += 1;
240
241 for (planned.input_slot_ids) |slot_id| {
242 args[arg_index] = .{ .memref = try slots.bytesFor(slot_id) };
243 arg_index += 1;
244 }
245
246 var count = [_]i32{std.math.cast(i32, planned.element_count) orelse return error.Overflow};
247 if (has_count_argument) {
248 switch (params[arg_index]) {
249 .buffer => args[arg_index] = kernel.argumentBuffer(i32, count[0..]),
250 .scalar => |dtype| args[arg_index] = switch (dtype) {
251 .i32 => kernel.argumentI32(count[0]),
252 .i64 => kernel.argumentI64(count[0]),
253 else => return error.UnsupportedOperation,
254 },
255 }
256 arg_index += 1;
257 }
258
259 if (arg_index != args.len) return error.UnsupportedOperation;
260 var executor = kernel.program.execution.Executor.init(generated);
261 defer executor.deinit();
262 try executor.runCpu(allocator, args);
263 }
264 }
265
266 const OracleSlot = struct {
267 slot_id: usize,
268 bytes: []u8,
269 owned: bool,
270 };
271
272 const OracleSlotTable = struct {
273 allocator: std.mem.Allocator,
274 items: []OracleSlot,
275
276 fn init(
277 comptime Spec: type,
278 allocator: std.mem.Allocator,
279 artifact_plan: *const accy.artifact.BackendArtifactPlan,
280 input_views: *const [Spec.inputs.len][]const u8,
281 output_bytes: []u8,
282 ) !OracleSlotTable {
283 if (artifact_plan.output_slot_ids.len != 1) return error.UnsupportedOperation;
284 const items = try allocator.alloc(OracleSlot, artifact_plan.slots.len);
285 var initialized: usize = 0;
286 errdefer {
287 for (items[0..initialized]) |item| {
288 if (item.owned) allocator.free(item.bytes);
289 }
290 allocator.free(items);
291 }
292
293 for (artifact_plan.slots) |slot| {
294 items[initialized] = try slotBytes(Spec, allocator, artifact_plan, input_views, output_bytes, slot);
295 initialized += 1;
296 }
297
298 return .{ .allocator = allocator, .items = items };
299 }
300
301 fn deinit(self: *OracleSlotTable) void {
302 for (self.items) |item| {
303 if (item.owned) self.allocator.free(item.bytes);
304 }
305 self.allocator.free(self.items);
306 self.* = undefined;
307 }
308
309 fn bytesFor(self: *const OracleSlotTable, slot_id: usize) ![]u8 {
310 for (self.items) |item| {
311 if (item.slot_id == slot_id) return item.bytes;
312 }
313 return error.InvalidArtifact;
314 }
315 };
316
317 fn slotBytes(
318 comptime Spec: type,
319 allocator: std.mem.Allocator,
320 artifact_plan: *const accy.artifact.BackendArtifactPlan,
321 input_views: *const [Spec.inputs.len][]const u8,
322 output_bytes: []u8,
323 slot: accy.artifact.PlannedSlot,
324 ) !OracleSlot {
325 if (slot.slot_id == artifact_plan.output_slot_ids[0]) {
326 return .{ .slot_id = slot.slot_id, .bytes = output_bytes, .owned = false };
327 }
328 if (boundaryInputBytes(Spec, artifact_plan, input_views, slot.slot_id)) |bytes| {
329 return .{ .slot_id = slot.slot_id, .bytes = @constCast(bytes), .owned = false };
330 }
331 if (slot.constantBytes()) |bytes| {
332 return .{ .slot_id = slot.slot_id, .bytes = @constCast(bytes), .owned = false };
333 }
334 const byte_size_u64 = slot.byte_size orelse return error.UnsupportedOperation;
335 const byte_size = std.math.cast(usize, byte_size_u64) orelse return error.InvalidArtifact;
336 const bytes = try allocator.alloc(u8, byte_size);
337 @memset(bytes, 0xA5);
338 return .{ .slot_id = slot.slot_id, .bytes = bytes, .owned = true };
339 }
340
341 fn boundaryInputBytes(
342 comptime Spec: type,
343 artifact_plan: *const accy.artifact.BackendArtifactPlan,
344 input_views: *const [Spec.inputs.len][]const u8,
345 slot_id: usize,
346 ) ?[]const u8 {
347 if (comptime Spec.inputs.len == 0) return null;
348 for (artifact_plan.input_slot_ids, 0..) |input_slot_id, index| {
349 if (input_slot_id == slot_id) return input_views[index];
350 }
351 return null;
352 }
353
354 pub fn finiteOrMax(value: f32) f32 {
355 return if (std.math.isFinite(value)) value else std.math.floatMax(f32);
356 }
357
358 fn fillInput(comptime Spec: type, comptime input: Tensor, comptime index: usize, buffer: []u8) void {
359 if (@hasDecl(Spec, "fill")) {
360 Spec.fill(index, buffer);
361 return;
362 }
363 defaultFill(input.dtype, index, buffer);
364 }
365
366 pub fn defaultFill(comptime dtype: DType, input_index: usize, buffer: []u8) void {
367 const T = dtype.ZigType();
368 const values = std.mem.bytesAsSlice(T, buffer);
369 for (values, 0..) |*value, element_index| {
370 value.* = defaultValue(T, input_index, element_index);
371 }
372 }
373
374 pub fn positiveFill(comptime dtype: DType, input_index: usize, buffer: []u8) void {
375 const T = dtype.ZigType();
376 const values = std.mem.bytesAsSlice(T, buffer);
377 for (values, 0..) |*value, element_index| {
378 value.* = positiveValue(T, input_index, element_index);
379 }
380 }
381
382 pub fn defaultValue(comptime T: type, input_index: usize, element_index: usize) T {
383 const pattern = (element_index * 7 + input_index * 13 + 5) % 31;
384 const alt = (element_index * 11 + input_index * 3 + 1) % 29;
385 if (comptime T == Bf16) return Bf16.fromF32(defaultValue(f32, input_index, element_index));
386 return switch (@typeInfo(T)) {
387 .bool => ((pattern + alt) & 1) == 0,
388 .float => blk: {
389 const base: f32 = -0.45 + @as(f32, @floatFromInt(pattern)) * 0.03125;
390 const tweak: f32 = @as(f32, @floatFromInt(alt)) * 0.001;
391 break :blk @floatCast(base + tweak);
392 },
393 .int => |info| if (info.signedness == .signed)
394 if (info.bits < 32)
395 @intCast(@as(i64, @intCast((pattern * 3 + alt) % 63)) - 31)
396 else
397 @intCast(@as(i64, @intCast(pattern * 7 + alt)) - 100)
398 else
399 @intCast(pattern * 7 + alt),
400 else => @compileError("unsupported conformance element type"),
401 };
402 }
403
404 pub fn positiveValue(comptime T: type, input_index: usize, element_index: usize) T {
405 const pattern = (element_index * 7 + input_index * 13 + 5) % 31;
406 if (comptime T == Bf16) return Bf16.fromF32(positiveValue(f32, input_index, element_index));
407 return switch (@typeInfo(T)) {
408 .bool => pattern != 0,
409 .float => @floatCast(0.0625 + @as(f32, @floatFromInt(pattern)) * 0.0625),
410 .int => @intCast(pattern + 1),
411 else => @compileError("unsupported conformance element type"),
412 };
413 }
414
415 pub fn maxAbsError(comptime dtype: DType, expected: []const u8, actual: []const u8) f32 {
416 const T = dtype.ZigType();
417 const expected_values = std.mem.bytesAsSlice(T, expected);
418 const actual_values = std.mem.bytesAsSlice(T, actual);
419 var max: f32 = 0;
420 for (expected_values, actual_values) |want, got| {
421 max = @max(max, errorOf(T, want, got));
422 }
423 return max;
424 }
425
426 fn errorOf(comptime T: type, want: T, got: T) f32 {
427 if (comptime T == Bf16) {
428 const w = want.toF32();
429 const g = got.toF32();
430 if (std.math.isNan(w) and std.math.isNan(g)) return 0;
431 if (std.math.isNan(w) or std.math.isNan(g)) return std.math.inf(f32);
432 return @abs(w - g);
433 }
434 switch (@typeInfo(T)) {
435 .bool => return if (want == got) 0 else 1,
436 .float => {
437 const w: f32 = @floatCast(want);
438 const g: f32 = @floatCast(got);
439 if (std.math.isNan(w) and std.math.isNan(g)) return 0;
440 if (std.math.isNan(w) or std.math.isNan(g)) return std.math.inf(f32);
441 return @abs(w - g);
442 },
443 .int => {
444 if (comptime @typeInfo(T).int.signedness == .unsigned) {
445 const w: u128 = @intCast(want);
446 const g: u128 = @intCast(got);
447 return @floatFromInt(if (w > g) w - g else g - w);
448 }
449 const w: i128 = @intCast(want);
450 const g: i128 = @intCast(got);
451 return @floatFromInt(if (w > g) w - g else g - w);
452 },
453 else => @compileError("unsupported conformance element type"),
454 }
455 }
456
457 pub fn checksum(comptime dtype: DType, bytes: []const u8) f32 {
458 const T = dtype.ZigType();
459 const values = std.mem.bytesAsSlice(T, bytes);
460 var sum: f64 = 0;
461 for (values) |value| {
462 if (comptime T == Bf16) {
463 sum += value.toF32();
464 continue;
465 }
466 sum += switch (@typeInfo(T)) {
467 .bool => if (value) 1 else 0,
468 .float => @as(f64, @floatCast(value)),
469 .int => @as(f64, @floatFromInt(value)),
470 else => @compileError("unsupported conformance element type"),
471 };
472 }
473 return @floatCast(sum);
474 }
475
476 pub fn numericToF32(comptime T: type, value: T) f32 {
477 if (comptime T == Bf16) return value.toF32();
478 return switch (@typeInfo(T)) {
479 .bool => if (value) 1 else 0,
480 .float => @floatCast(value),
481 .int => @floatFromInt(value),
482 else => @compileError("unsupported conformance element type"),
483 };
484 }
485
486 pub fn nowNs() i128 {
487 return sys.time.nanoTimestamp();
488 }
489
490 pub fn elapsedNs(start: i128) u64 {
491 const elapsed = nowNs() - start;
492 if (elapsed <= 0) return 0;
493 return @intCast(elapsed);
494 }
495
496 test "default fill is deterministic and mixes input positions" {
497 var first: [64]u8 = undefined;
498 var second: [64]u8 = undefined;
499 var other: [64]u8 = undefined;
500 defaultFill(.f32, 0, first[0..]);
501 defaultFill(.f32, 0, second[0..]);
502 defaultFill(.f32, 1, other[0..]);
503 try std.testing.expectEqualSlices(u8, first[0..], second[0..]);
504 try std.testing.expect(!std.mem.eql(u8, first[0..], other[0..]));
505 }
506
507 test "positive fill stays strictly positive" {
508 var buffer: [128]u8 = undefined;
509 positiveFill(.f32, 0, buffer[0..]);
510 const values = std.mem.bytesAsSlice(f32, buffer[0..]);
511 for (values) |value| try std.testing.expect(value > 0);
512 }
513
514 test "max abs error treats mismatched nan as infinite" {
515 const want = [_]f32{ 1.0, std.math.nan(f32) };
516 const got_match = [_]f32{ 1.0, std.math.nan(f32) };
517 const got_bad = [_]f32{ 1.0, 0.0 };
518 try std.testing.expectEqual(
519 @as(f32, 0),
520 maxAbsError(.f32, std.mem.sliceAsBytes(want[0..]), std.mem.sliceAsBytes(got_match[0..])),
521 );
522 try std.testing.expect(std.math.isInf(
523 maxAbsError(.f32, std.mem.sliceAsBytes(want[0..]), std.mem.sliceAsBytes(got_bad[0..])),
524 ));
525 }
526
527 test "integer error is the absolute difference" {
528 const want = [_]i32{ 5, -7 };
529 const got = [_]i32{ 5, -10 };
530 try std.testing.expectEqual(
531 @as(f32, 3),
532 maxAbsError(.i32, std.mem.sliceAsBytes(want[0..]), std.mem.sliceAsBytes(got[0..])),
533 );
534 }
535
536 test "conformance semantic row required dtypes include inputs and output" {
537 const Spec = struct {
538 pub const name = "dtype_requirements_semantic";
539 pub const expectation: Expectation = .verified;
540 pub const tolerance: f32 = 0;
541 pub const inputs = [_]Tensor{
542 vec(.f16, 4),
543 vec(.i32, 4),
544 };
545 pub const output = vec(.f32, 4);
546 };
547
548 const set = requiredDTypes(Spec);
549 try std.testing.expect(set.contains(.f16));
550 try std.testing.expect(set.contains(.i32));
551 try std.testing.expect(set.contains(.f32));
552 try std.testing.expect(!set.contains(.bf16));
553 }
554
555 test "conformance family row required dtypes include every buffer" {
556 const Spec = struct {
557 pub const name = "dtype_requirements_family";
558 pub const expectation: Expectation = .verified;
559 pub const tolerance: f32 = 0;
560 pub const buffers = [_]FamilyBuffer{
561 .{ .tensor = vec(.bf16, 4), .access = .inout },
562 .{ .tensor = vec(.u32, 4) },
563 };
564 };
565
566 const set = requiredDTypes(Spec);
567 try std.testing.expect(set.contains(.bf16));
568 try std.testing.expect(set.contains(.u32));
569 try std.testing.expect(!set.contains(.f32));
570 }
571
572 test "conformance row backend requirements default to empty" {
573 const Spec = struct {
574 pub const name = "empty_backend_requirements";
575 pub const expectation: Expectation = .verified;
576 pub const tolerance: f32 = 0;
577 pub const inputs = [_]Tensor{vec(.f32, 4)};
578 pub const output = vec(.f32, 4);
579 };
580
581 try std.testing.expect(std.meta.eql(choir_abi.Features{}, requiredFeatures(Spec)));
582 try std.testing.expect(std.meta.eql(choir_abi.SubgroupRequirements{}, requiredSubgroup(Spec)));
583 }
584
585 test "conformance row backend requirements read optional declarations" {
586 const Spec = struct {
587 pub const name = "declared_backend_requirements";
588 pub const expectation: Expectation = .verified;
589 pub const tolerance: f32 = 0;
590 pub const inputs = [_]Tensor{vec(.f32, 4)};
591 pub const output = vec(.f32, 4);
592 pub const required_features = choir_abi.Features{ .atomic_f32_add_device = true };
593 pub const required_subgroup = choir_abi.SubgroupRequirements{ .supported = true, .scan = true };
594 };
595
596 try std.testing.expect(requiredFeatures(Spec).atomic_f32_add_device);
597 try std.testing.expect(requiredSubgroup(Spec).supported);
598 try std.testing.expect(requiredSubgroup(Spec).scan);
599 }
600
601 fn expectGeneratedOracleMatchesReference(comptime Spec: type) !void {
602 const allocator = std.testing.allocator;
603 var state = gpu.recording.BackendState{
604 .allocator = allocator,
605 .kind = .vulkan,
606 .format = .vulkan_spirv,
607 };
608
609 const module = try buildModule(Spec, allocator);
610 var prepared = try accy.executable.prepareFragmentFromSemanticModule(
611 allocator,
612 state.handle(),
613 module,
614 .{},
615 );
616 defer prepared.deinit();
617
618 var input_views: [Spec.inputs.len][]const u8 = undefined;
619 var input_buffers: [Spec.inputs.len][]align(16) u8 = undefined;
620 inline for (Spec.inputs, 0..) |input, index| {
621 const buffer = try allocator.alignedAlloc(u8, .@"16", input.byteCount());
622 fillInput(Spec, input, index, buffer);
623 input_views[index] = buffer;
624 input_buffers[index] = buffer;
625 }
626 defer {
627 inline for (0..Spec.inputs.len) |index| allocator.free(input_buffers[index]);
628 }
629
630 const oracle = try allocator.alignedAlloc(u8, .@"16", Spec.output.byteCount());
631 defer allocator.free(oracle);
632 const reference = try allocator.alignedAlloc(u8, .@"16", Spec.output.byteCount());
633 defer allocator.free(reference);
634
635 const artifact_module = try accy.executable.createArtifactJobFromPreparedJob(
636 allocator,
637 state.handle(),
638 &prepared,
639 .{},
640 );
641 defer artifact_module.deinit();
642
643 try expectedOutput(Spec, allocator, &prepared, artifact_module.artifactPlan(), &input_views, oracle);
644 Spec.reference(&input_views, reference);
645 try std.testing.expect(maxAbsError(Spec.output.dtype, reference, oracle) <= Spec.tolerance);
646 }
647
648 test "conformance verified references match the generated CPU oracle" {
649 @setEvalBranchQuota(200_000);
650 inline for (conformance.cases.all) |Spec| {
651 if (comptime std.mem.eql(u8, Spec.name, "add_f32_256") or
652 std.mem.eql(u8, Spec.name, "add_f16_256") or
653 std.mem.eql(u8, Spec.name, "reshape_add_f32_4x32") or
654 std.mem.eql(u8, Spec.name, "reduce_sum_f32_16x16") or
655 std.mem.eql(u8, Spec.name, "dot_general_f32_16x16") or
656 std.mem.eql(u8, Spec.name, "pad_f32_6x6") or
657 std.mem.eql(u8, Spec.name, "pad_i1_6x6"))
658 {
659 try expectGeneratedOracleMatchesReference(Spec);
660 }
661 }
662 }
663
664 test "conformance harness executes selected cases on live Metal" {
665 @setEvalBranchQuota(200_000);
666 const allocator = std.testing.allocator;
667 var state = try initMetalStateOrSkip(allocator);
668 defer state.deinit();
669 const gate = Gate{ .mode = .execute, .assembler = null };
670
671 var semantic_count: usize = 0;
672 var family_count: usize = 0;
673 inline for (conformance.cases.all) |Spec| {
674 if (comptime selectedMetalConformanceCase(Spec)) {
675 const result = if (comptime @hasDecl(Spec, "buildArtifact"))
676 runFamilyCase(Spec, allocator, state.handle(), gate) catch |err| {
677 pretty.diagnostic.writeStderrText(
678 "metal conformance case errored: name={s} error={s}\n",
679 .{ Spec.name, @errorName(err) },
680 );
681 return err;
682 }
683 else
684 runCase(Spec, allocator, state.handle(), gate) catch |err| {
685 pretty.diagnostic.writeStderrText(
686 "metal conformance case errored: name={s} error={s}\n",
687 .{ Spec.name, @errorName(err) },
688 );
689 return err;
690 };
691 try expectConformanceCasePass(result);
692 if (comptime @hasDecl(Spec, "buildArtifact")) {
693 family_count += 1;
694 } else {
695 semantic_count += 1;
696 }
697 }
698 }
699
700 try std.testing.expectEqual(@as(usize, 94), semantic_count);
701 try std.testing.expectEqual(@as(usize, 40), family_count);
702 }
703
704 const selected_metal_family_conformance_cases = [_][]const u8{
705 "gather_family_f32_2x5x6x3",
706 "gather_family_f16_2x5x6x3",
707 "scatter_family_f32_2x5x6x3",
708 "scatter_family_f16_2x5x6x3",
709 "scatter_add_family_direct_i32_16x64",
710 "scatter_add_family_shared_i32_16x64",
711 "scatter_add_family_direct_f32_16x64",
712 "prefix_sum_family_inclusive_f32_64",
713 "prefix_sum_family_exclusive_f32_64",
714 "segment_sum_family_thread_f32_8x80",
715 "segment_sum_family_warp_f32_8x80",
716 "filter_family_nonzero_f32_70x32",
717 "filter_family_greater_i32_40x32",
718 "device_scan_block_scan_family_dst_f32_3x32",
719 "device_scan_block_scan_family_sums_f32_3x32",
720 "device_scan_block_scan_family_dst_f16_3x32",
721 "device_scan_block_scan_family_sums_f16_3x32",
722 "device_scan_add_base_family_f32_3x32",
723 "device_scan_add_base_family_f16_3x32",
724 "batched_cholesky_family_f32_48x3",
725 "batched_cholesky_family_f32_48x3_interleaved",
726 "batched_cholesky_solve_family_f32_48x3",
727 "batched_cholesky_solve_family_f32_48x3_interleaved",
728 "batched_inverse_family_f32_48x3",
729 "batched_inverse_family_f32_48x3_interleaved",
730 "grid_cells_family_f32_96points",
731 "grid_neighbor_count_family_f32_96points",
732 "spmv_csr_row_warp_family_f32_48rows",
733 "spmv_csr_row_thread_family_f32_48rows",
734 "spmv_csr_row_warp_family_f16_48rows",
735 "spmv_coo_element_thread_family_f32_48x96",
736 "spmv_ell_row_thread_family_f32_48x6",
737 "spmv_sell_row_thread_slice8_family_f32_48x8",
738 "spmm_csr_row_column_thread_family_f32_32x9",
739 "philox_fill_family_i32_64",
740 "threefry_fill_family_i32_65",
741 "squares_fill_family_i32_63",
742 "histogram_family_direct_f32_16x96",
743 "histogram_family_shared_f32_16x96",
744 };
745
746 fn selectedMetalConformanceCase(comptime Spec: type) bool {
747 if (comptime @hasDecl(Spec, "buildArtifact")) {
748 inline for (selected_metal_family_conformance_cases) |selected| {
749 if (std.mem.eql(u8, Spec.name, selected)) return true;
750 }
751 return false;
752 }
753 if (comptime Spec.expectation != .verified) return false;
754 return switch (comptime Spec.output.dtype) {
755 .f32, .f16, .i32 => true,
756 else => false,
757 };
758 }
759
760 fn expectConformanceCasePass(result: records.Case) !void {
761 if (std.mem.eql(u8, result.status, "pass")) return;
762 pretty.diagnostic.writeStderrText(
763 "metal conformance case failed: name={s} status={s} detail={s} max_abs_error={d} tolerance={d}\n",
764 .{ result.name, result.status, result.detail, result.max_abs_error, result.tolerance },
765 );
766 return error.MetalConformanceCaseFailed;
767 }
768
769 fn initMetalStateOrSkip(allocator: std.mem.Allocator) !gpu.metal.State {
770 try accy.validation.gating.skipIfBuildFlagDisabled(.metal);
771 if (!accy.validation.gating.appleMetalPlatform()) return accy.validation.gating.skip(.metal, .unsupported_platform);
772 return gpu.metal.State.initDevice(allocator) catch |err| switch (err) {
773 error.RuntimeUnavailable => return accy.validation.gating.skip(.metal, .metal_device_missing),
774 else => return err,
775 };
776 }
777
778 pub const FamilyAccess = enum { input, inout };
779
780 pub const FamilyBuffer = struct {
781 tensor: Tensor,
782 access: FamilyAccess = .input,
783 };
784
785 pub fn runFamilyCase(
786 comptime Spec: type,
787 allocator: std.mem.Allocator,
788 handle: BackendHandle,
789 gate: Gate,
790 ) !records.Case {
791 var arena_state = std.heap.ArenaAllocator.init(allocator);
792 defer arena_state.deinit();
793 const arena = arena_state.allocator();
794
795 const compile_start = nowNs();
796 var artifact = try Spec.buildArtifact(allocator, handle);
797 defer artifact.deinit();
798 const ptxas_status = acceptSingleArtifact(allocator, artifact, gate.assembler);
799 const ptxas_ok = !std.mem.eql(u8, ptxas_status, "fail");
800
801 if (gate.mode == .assemble) {
802 return .{
803 .name = Spec.name,
804 .status = if (ptxas_ok) "assembled" else "fail",
805 .detail = if (ptxas_ok) "" else "PtxasRejected",
806 .ptxas = ptxas_status,
807 .kernels = 1,
808 .compile_ns = elapsedNs(compile_start),
809 .launch_read_ns = 0,
810 .checksum = 0,
811 .max_abs_error = 0,
812 .tolerance = Spec.tolerance,
813 };
814 }
815
816 const loaded = try handle.loadArtifact(&artifact);
817 const compile_ns = elapsedNs(compile_start);
818
819 var seeded: [Spec.buffers.len][]const u8 = undefined;
820 var bindings: [Spec.buffers.len]gpu.BufferBinding = undefined;
821 inline for (Spec.buffers, 0..) |family_buffer, index| {
822 const bytes = try arena.alignedAlloc(u8, .@"16", family_buffer.tensor.byteCount());
823 Spec.fillBuffer(index, bytes);
824 seeded[index] = bytes;
825 const device_buffer = try handle.allocateBuffer(.{
826 .byte_size = bytes.len,
827 .alignment = 256,
828 .dtype = family_buffer.tensor.dtype,
829 .element_count = family_buffer.tensor.elementCount(),
830 });
831 try handle.writeBuffer(.{ .handle = device_buffer, .bytes = bytes });
832 bindings[index] = .{
833 .handle = device_buffer,
834 .access = switch (family_buffer.access) {
835 .input => .read_only,
836 .inout => .read_write,
837 },
838 .ownership = device_buffer.ownership,
839 .byte_size = device_buffer.byte_size,
840 };
841 }
842
843 const launch_start = nowNs();
844 const runtime_arguments = try Spec.runtimeArguments();
845 try handle.launch(.{
846 .artifact = &artifact,
847 .loaded_artifact = loaded,
848 .buffers = bindings[0..],
849 .scalar_arguments = runtime_arguments[0..],
850 .geometry = Spec.geometry,
851 });
852 try handle.synchronize(.{ .scope = .device });
853
854 const observed_tensor = Spec.buffers[Spec.observed].tensor;
855 const observed = try arena.alignedAlloc(u8, .@"16", observed_tensor.byteCount());
856 try handle.readBuffer(.{ .handle = bindings[Spec.observed].handle, .bytes = observed });
857 const launch_read_ns = elapsedNs(launch_start);
858
859 const expected = try arena.alignedAlloc(u8, .@"16", observed_tensor.byteCount());
860 Spec.reference(seeded[0..], expected);
861
862 const max_error = maxAbsError(observed_tensor.dtype, expected, observed);
863 const numeric_ok = max_error <= Spec.tolerance;
864 return .{
865 .name = Spec.name,
866 .status = if (numeric_ok and ptxas_ok) "pass" else "fail",
867 .detail = if (ptxas_ok) "" else "PtxasRejected",
868 .ptxas = ptxas_status,
869 .kernels = 1,
870 .compile_ns = compile_ns,
871 .launch_read_ns = launch_read_ns,
872 .checksum = finiteOrMax(checksum(observed_tensor.dtype, observed)),
873 .max_abs_error = finiteOrMax(max_error),
874 .tolerance = Spec.tolerance,
875 };
876 }
877
878 fn acceptSingleArtifact(
879 allocator: std.mem.Allocator,
880 artifact: gpu.KernelArtifact,
881 assembler: ?Assembler,
882 ) []const u8 {
883 const active = assembler orelse return "skip";
884 ptx.assemble.acceptArtifact(allocator, active.dir, active.target, artifact) catch {
885 return "fail";
886 };
887 return "pass";
888 }