lib/accy/src/validation/conformance/cases.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const gpu = @import("gpu");
3 const choir_abi = @import("choir_abi");
4 const accy = @import("accy");
5 const conformance = @import("root.zig");
6
7 const harness = conformance.harness;
8 const SemanticBuilder = harness.SemanticBuilder;
9 const FunctionBuilder = harness.FunctionBuilder;
10 const Value = harness.Value;
11 const DType = harness.DType;
12 const Bf16 = harness.Bf16;
13 const kernel_limits = accy.kernel.Limits.standard;
14
15 const element_count = 256;
16
17 const BinaryOp = enum { add, sub, mul, div, min, max, pow };
18 const UnaryOp = enum { neg, abs, exp, log, sqrt, sin, cos, tanh, floor, round, trunc };
19 const FillKind = enum { default, positive, lhs_positive, rhs_positive };
20
21 fn Binary(
22 comptime case_name: []const u8,
23 comptime dtype: DType,
24 comptime op: BinaryOp,
25 comptime fill_kind: FillKind,
26 comptime case_tolerance: f32,
27 comptime case_expectation: harness.Expectation,
28 ) type {
29 return struct {
30 pub const name = case_name;
31 pub const expectation = case_expectation;
32 pub const inputs = [_]harness.Tensor{
33 harness.vec(dtype, element_count),
34 harness.vec(dtype, element_count),
35 };
36 pub const output = harness.vec(dtype, element_count);
37 pub const tolerance: f32 = case_tolerance;
38
39 pub fn body(_: *SemanticBuilder, function: *FunctionBuilder) !*Value {
40 const lhs = function.parameter(0);
41 const rhs = function.parameter(1);
42 return switch (op) {
43 .add => try function.add(lhs, rhs),
44 .sub => try function.sub(lhs, rhs),
45 .mul => try function.mul(lhs, rhs),
46 .div => try function.div(lhs, rhs),
47 .min => try function.min(lhs, rhs),
48 .max => try function.max(lhs, rhs),
49 .pow => try function.pow(lhs, rhs),
50 };
51 }
52
53 pub fn fill(input_index: usize, buffer: []u8) void {
54 fillByKind(dtype, fill_kind, input_index, buffer);
55 }
56
57 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
58 const T = dtype.ZigType();
59 const lhs = std.mem.bytesAsSlice(T, views[0]);
60 const rhs = std.mem.bytesAsSlice(T, views[1]);
61 const out = std.mem.bytesAsSlice(T, output_bytes);
62 for (out, lhs, rhs) |*value, a, b| value.* = binaryReference(T, op, a, b);
63 }
64 };
65 }
66
67 fn Unary(
68 comptime case_name: []const u8,
69 comptime dtype: DType,
70 comptime op: UnaryOp,
71 comptime fill_kind: FillKind,
72 comptime case_tolerance: f32,
73 comptime case_expectation: harness.Expectation,
74 ) type {
75 return struct {
76 pub const name = case_name;
77 pub const expectation = case_expectation;
78 pub const inputs = [_]harness.Tensor{harness.vec(dtype, element_count)};
79 pub const output = harness.vec(dtype, element_count);
80 pub const tolerance: f32 = case_tolerance;
81
82 pub fn body(_: *SemanticBuilder, function: *FunctionBuilder) !*Value {
83 const input = function.parameter(0);
84 return switch (op) {
85 .neg => try function.neg(input),
86 .abs => try function.abs(input),
87 .exp => try function.exp(input),
88 .log => try function.log(input),
89 .sqrt => try function.sqrt(input),
90 .sin => try function.sin(input),
91 .cos => try function.cos(input),
92 .tanh => try function.tanh(input),
93 .floor => try function.floor(input),
94 .round => try function.round(input),
95 .trunc => try function.trunc(input),
96 };
97 }
98
99 pub fn fill(input_index: usize, buffer: []u8) void {
100 fillByKind(dtype, fill_kind, input_index, buffer);
101 }
102
103 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
104 const T = dtype.ZigType();
105 const input = std.mem.bytesAsSlice(T, views[0]);
106 const out = std.mem.bytesAsSlice(T, output_bytes);
107 for (out, input) |*value, a| value.* = unaryReference(T, op, a);
108 }
109 };
110 }
111
112 fn fillByKind(comptime dtype: DType, comptime fill_kind: FillKind, input_index: usize, buffer: []u8) void {
113 const positive = switch (fill_kind) {
114 .default => false,
115 .positive => true,
116 .lhs_positive => input_index == 0,
117 .rhs_positive => input_index == 1,
118 };
119 if (positive) {
120 harness.positiveFill(dtype, input_index, buffer);
121 } else {
122 harness.defaultFill(dtype, input_index, buffer);
123 }
124 }
125
126 fn isUnsignedType(comptime T: type) bool {
127 return @typeInfo(T) == .int and @typeInfo(T).int.signedness == .unsigned;
128 }
129
130 fn isIntType(comptime T: type) bool {
131 return @typeInfo(T) == .int;
132 }
133
134 fn floatLikeFromF32(comptime T: type, value: f32) T {
135 if (comptime T == Bf16) return Bf16.fromF32(value);
136 return @floatCast(value);
137 }
138
139 fn numericLikeFromF32(comptime T: type, value: f32) T {
140 if (comptime T == Bf16) return Bf16.fromF32(value);
141 return switch (@typeInfo(T)) {
142 .float => @floatCast(value),
143 .int => @intFromFloat(value),
144 else => @compileError("unsupported conformance element type"),
145 };
146 }
147
148 fn literalForDType(comptime dtype: DType, comptime float_value: f32, comptime int_value: comptime_int) dtype.ZigType() {
149 const T = dtype.ZigType();
150 if (comptime T == Bf16) return Bf16.fromF32(float_value);
151 return if (comptime dtype.isFloat()) @floatCast(float_value) else @intCast(int_value);
152 }
153
154 fn iotaValue(comptime dtype: DType, index: usize) dtype.ZigType() {
155 const T = dtype.ZigType();
156 if (comptime T == Bf16) return Bf16.fromF32(@floatFromInt(index));
157 return if (comptime dtype.isFloat()) @floatFromInt(index) else @intCast(index);
158 }
159
160 fn addReference(comptime T: type, a: T, b: T) T {
161 return binaryReference(T, .add, a, b);
162 }
163
164 fn reduceInitValue(comptime dtype: DType, comptime kind: ReducerKind) dtype.ZigType() {
165 const T = dtype.ZigType();
166 if (comptime T == Bf16) {
167 return Bf16.fromF32(switch (kind) {
168 .sum => 0,
169 .max => -std.math.inf(f32),
170 .min => std.math.inf(f32),
171 });
172 }
173 return switch (kind) {
174 .sum => 0,
175 .max => if (comptime dtype.isFloat()) -std.math.inf(T) else std.math.minInt(T),
176 .min => if (comptime dtype.isFloat()) std.math.inf(T) else std.math.maxInt(T),
177 };
178 }
179
180 fn binaryReference(comptime T: type, comptime op: BinaryOp, a: T, b: T) T {
181 if (comptime T == Bf16) {
182 const lhs = a.toF32();
183 const rhs = b.toF32();
184 const result = switch (op) {
185 .add => lhs + rhs,
186 .sub => lhs - rhs,
187 .mul => lhs * rhs,
188 .div => lhs / rhs,
189 .min => @min(lhs, rhs),
190 .max => @max(lhs, rhs),
191 .pow => std.math.pow(f32, lhs, rhs),
192 };
193 return Bf16.fromF32(result);
194 }
195 return switch (op) {
196 .add => if (comptime isIntType(T)) a +% b else a + b,
197 .sub => if (comptime isIntType(T)) a -% b else a - b,
198 .mul => if (comptime isIntType(T)) a *% b else a * b,
199 .div => switch (@typeInfo(T)) {
200 .float => a / b,
201 .int => |info| if (info.signedness == .signed) @divTrunc(a, b) else a / b,
202 else => @compileError("unsupported conformance element type"),
203 },
204 .min => @min(a, b),
205 .max => @max(a, b),
206 .pow => powReference(T, a, b),
207 };
208 }
209
210 fn powReference(comptime T: type, a: T, b: T) T {
211 if (comptime T == Bf16) return Bf16.fromF32(std.math.pow(f32, a.toF32(), b.toF32()));
212 if (comptime @typeInfo(T) != .float) unreachable;
213 if (comptime T == f16) {
214 return @floatCast(std.math.pow(f32, @floatCast(a), @floatCast(b)));
215 }
216 return std.math.pow(T, a, b);
217 }
218
219 fn unaryReference(comptime T: type, comptime op: UnaryOp, a: T) T {
220 if (comptime T == Bf16) {
221 return switch (op) {
222 .neg => Bf16.fromF32(-a.toF32()),
223 .abs => Bf16.fromF32(@abs(a.toF32())),
224 .exp, .log, .sqrt, .sin, .cos, .tanh, .floor, .round, .trunc => floatUnaryReference(T, op, a),
225 };
226 }
227 return switch (op) {
228 .neg => if (comptime isUnsignedType(T)) 0 -% a else -a,
229 .abs => switch (@typeInfo(T)) {
230 .float => @abs(a),
231 .int => @intCast(@abs(a)),
232 else => @compileError("unsupported conformance element type"),
233 },
234 .exp => floatUnaryReference(T, .exp, a),
235 .log => floatUnaryReference(T, .log, a),
236 .sqrt => floatUnaryReference(T, .sqrt, a),
237 .sin => floatUnaryReference(T, .sin, a),
238 .cos => floatUnaryReference(T, .cos, a),
239 .tanh => floatUnaryReference(T, .tanh, a),
240 .floor => floatUnaryReference(T, .floor, a),
241 .round => floatUnaryReference(T, .round, a),
242 .trunc => floatUnaryReference(T, .trunc, a),
243 };
244 }
245
246 fn floatUnaryReference(comptime T: type, comptime op: UnaryOp, a: T) T {
247 if (comptime T == Bf16) return Bf16.fromF32(floatUnaryF32(op, a.toF32()));
248 if (comptime @typeInfo(T) != .float) unreachable;
249 return switch (op) {
250 .exp => @exp(a),
251 .log => @log(a),
252 .sqrt => @sqrt(a),
253 .sin => @sin(a),
254 .cos => @cos(a),
255 .tanh => @floatCast(std.math.tanh(@as(f32, @floatCast(a)))),
256 .floor => @floor(a),
257 .round => @round(a),
258 .trunc => @trunc(a),
259 .neg, .abs => unreachable,
260 };
261 }
262
263 fn floatUnaryF32(comptime op: UnaryOp, a: f32) f32 {
264 return switch (op) {
265 .exp => @exp(a),
266 .log => @log(a),
267 .sqrt => @sqrt(a),
268 .sin => @sin(a),
269 .cos => @cos(a),
270 .tanh => std.math.tanh(a),
271 .floor => @floor(a),
272 .round => @round(a),
273 .trunc => @trunc(a),
274 .neg, .abs => unreachable,
275 };
276 }
277
278 const TanhAddF32 = struct {
279 pub const name = "tanh_add_f32_256";
280 pub const expectation: harness.Expectation = .verified;
281 pub const inputs = [_]harness.Tensor{
282 harness.vec(.f32, element_count),
283 harness.vec(.f32, element_count),
284 };
285 pub const output = harness.vec(.f32, element_count);
286 pub const tolerance: f32 = 0.00001;
287
288 pub fn body(_: *SemanticBuilder, function: *FunctionBuilder) !*Value {
289 const sum = try function.add(function.parameter(0), function.parameter(1));
290 return try function.tanh(sum);
291 }
292
293 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
294 const lhs = std.mem.bytesAsSlice(f32, views[0]);
295 const rhs = std.mem.bytesAsSlice(f32, views[1]);
296 const out = std.mem.bytesAsSlice(f32, output_bytes);
297 for (out, lhs, rhs) |*value, a, b| value.* = std.math.tanh(a + b);
298 }
299 };
300
301 const BiasBroadcastF32 = struct {
302 pub const name = "bias_broadcast_f32_8x32";
303 pub const expectation: harness.Expectation = .verified;
304 pub const inputs = [_]harness.Tensor{
305 harness.mat(.f32, 8, 32),
306 harness.vec(.f32, 32),
307 };
308 pub const output = harness.mat(.f32, 8, 32);
309 pub const tolerance: f32 = 0.0;
310
311 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
312 const result_type = try builder.tensor(.f32, &.{ 8, 32 });
313 const bias = try function.broadcastInDim(function.parameter(1), result_type, &.{ 8, 32 }, &.{1});
314 return try function.add(function.parameter(0), bias);
315 }
316
317 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
318 const matrix = std.mem.bytesAsSlice(f32, views[0]);
319 const bias = std.mem.bytesAsSlice(f32, views[1]);
320 const out = std.mem.bytesAsSlice(f32, output_bytes);
321 for (out, 0..) |*value, index| value.* = matrix[index] + bias[index % 32];
322 }
323 };
324
325 const DotRectF32 = struct {
326 pub const name = "dot_general_f32_8x24x12";
327 pub const expectation: harness.Expectation = .verified;
328 pub const inputs = [_]harness.Tensor{
329 harness.mat(.f32, 8, 24),
330 harness.mat(.f32, 24, 12),
331 };
332 pub const output = harness.mat(.f32, 8, 12);
333 pub const tolerance: f32 = 0.0001;
334
335 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
336 const result_type = try builder.tensor(.f32, &.{ 8, 12 });
337 return try function.dotGeneral(
338 function.parameter(0),
339 function.parameter(1),
340 result_type,
341 &.{1},
342 &.{0},
343 &.{},
344 &.{},
345 );
346 }
347
348 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
349 dotReference(f32, views, output_bytes, 8, 24, 12);
350 }
351 };
352
353 fn dotReference(
354 comptime T: type,
355 views: []const []const u8,
356 output_bytes: []u8,
357 comptime rows: usize,
358 comptime inner: usize,
359 comptime cols: usize,
360 ) void {
361 if (comptime T == f16 or T == Bf16) {
362 const lhs = std.mem.bytesAsSlice(T, views[0]);
363 const rhs = std.mem.bytesAsSlice(T, views[1]);
364 const out = std.mem.bytesAsSlice(T, output_bytes);
365 for (0..rows) |row| {
366 for (0..cols) |col| {
367 var acc: f32 = 0;
368 for (0..inner) |axis| {
369 acc += harness.numericToF32(T, lhs[row * inner + axis]) * harness.numericToF32(T, rhs[axis * cols + col]);
370 }
371 out[row * cols + col] = floatLikeFromF32(T, acc);
372 }
373 }
374 return;
375 }
376
377 const lhs = std.mem.bytesAsSlice(T, views[0]);
378 const rhs = std.mem.bytesAsSlice(T, views[1]);
379 const out = std.mem.bytesAsSlice(T, output_bytes);
380 for (0..rows) |row| {
381 for (0..cols) |col| {
382 var acc: T = 0;
383 for (0..inner) |axis| {
384 acc += lhs[row * inner + axis] * rhs[axis * cols + col];
385 }
386 out[row * cols + col] = acc;
387 }
388 }
389 }
390
391 const EinsumMatmulF32 = struct {
392 pub const name = "einsum_matmul_f32_8x16x12";
393 pub const expectation: harness.Expectation = .verified;
394 pub const inputs = [_]harness.Tensor{
395 harness.mat(.f32, 8, 16),
396 harness.mat(.f32, 16, 12),
397 };
398 pub const output = harness.mat(.f32, 8, 12);
399 pub const tolerance: f32 = 0.0001;
400
401 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
402 const result_type = try builder.tensor(.f32, &.{ 8, 12 });
403 return try function.einsum(
404 &.{ function.parameter(0), function.parameter(1) },
405 result_type,
406 "ij,jk->ik",
407 );
408 }
409
410 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
411 dotReference(f32, views, output_bytes, 8, 16, 12);
412 }
413 };
414
415 const EinsumPermuteF32 = struct {
416 pub const name = "einsum_permute_f32_8x16";
417 pub const expectation: harness.Expectation = .verified;
418 pub const inputs = [_]harness.Tensor{harness.mat(.f32, 8, 16)};
419 pub const output = harness.mat(.f32, 16, 8);
420 pub const tolerance: f32 = 0.0;
421
422 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
423 const result_type = try builder.tensor(.f32, &.{ 16, 8 });
424 return try function.einsum(&.{function.parameter(0)}, result_type, "ij->ji");
425 }
426
427 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
428 const input = std.mem.bytesAsSlice(f32, views[0]);
429 const out = std.mem.bytesAsSlice(f32, output_bytes);
430 for (0..16) |row| {
431 for (0..8) |col| {
432 out[row * 8 + col] = input[col * 16 + row];
433 }
434 }
435 }
436 };
437
438 const ReduceSumAxisZeroF32 = struct {
439 pub const name = "reduce_sum_axis0_f32_16x16";
440 pub const expectation: harness.Expectation = .verified;
441 pub const inputs = [_]harness.Tensor{harness.mat(.f32, 16, 16)};
442 pub const output = harness.vec(.f32, 16);
443 pub const tolerance: f32 = 0.0001;
444 const init_value: f32 = 0.0;
445
446 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
447 const scalar_type = try builder.tensor(.f32, &.{});
448 const result_type = try builder.tensor(.f32, &.{16});
449 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
450 return try function.reduce(function.parameter(0), init, result_type, "sum", &.{0});
451 }
452
453 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
454 const input = std.mem.bytesAsSlice(f32, views[0]);
455 const out = std.mem.bytesAsSlice(f32, output_bytes);
456 for (0..16) |col| {
457 var acc: f32 = init_value;
458 for (0..16) |row| acc += input[row * 16 + col];
459 out[col] = acc;
460 }
461 }
462 };
463
464 const ReduceSumAllF32 = struct {
465 pub const name = "reduce_sum_all_f32_16x16";
466 pub const expectation: harness.Expectation = .verified;
467 pub const inputs = [_]harness.Tensor{harness.mat(.f32, 16, 16)};
468 pub const output = harness.Tensor{ .dtype = .f32, .dims = &.{} };
469 pub const tolerance: f32 = 0.001;
470 const init_value: f32 = 0.0;
471
472 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
473 const scalar_type = try builder.tensor(.f32, &.{});
474 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
475 return try function.reduce(function.parameter(0), init, scalar_type, "sum", &.{ 0, 1 });
476 }
477
478 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
479 const input = std.mem.bytesAsSlice(f32, views[0]);
480 const out = std.mem.bytesAsSlice(f32, output_bytes);
481 var acc: f32 = init_value;
482 for (input) |value| acc += value;
483 out[0] = acc;
484 }
485 };
486
487 const AddRankThreeF32 = struct {
488 pub const name = "add_f32_4x8x8";
489 pub const expectation: harness.Expectation = .verified;
490 pub const inputs = [_]harness.Tensor{
491 .{ .dtype = .f32, .dims = &.{ 4, 8, 8 } },
492 .{ .dtype = .f32, .dims = &.{ 4, 8, 8 } },
493 };
494 pub const output = harness.Tensor{ .dtype = .f32, .dims = &.{ 4, 8, 8 } };
495 pub const tolerance: f32 = 0.0;
496
497 pub fn body(_: *SemanticBuilder, function: *FunctionBuilder) !*Value {
498 return try function.add(function.parameter(0), function.parameter(1));
499 }
500
501 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
502 const lhs = std.mem.bytesAsSlice(f32, views[0]);
503 const rhs = std.mem.bytesAsSlice(f32, views[1]);
504 const out = std.mem.bytesAsSlice(f32, output_bytes);
505 for (out, lhs, rhs) |*value, a, b| value.* = a + b;
506 }
507 };
508
509 const ReduceSumRankThreeTrailingF32 = struct {
510 pub const name = "reduce_sum_axis2_f32_4x8x8";
511 pub const expectation: harness.Expectation = .verified;
512 pub const inputs = [_]harness.Tensor{.{ .dtype = .f32, .dims = &.{ 4, 8, 8 } }};
513 pub const output = harness.mat(.f32, 4, 8);
514 pub const tolerance: f32 = 0.0001;
515 const init_value: f32 = 0.0;
516
517 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
518 const scalar_type = try builder.tensor(.f32, &.{});
519 const result_type = try builder.tensor(.f32, &.{ 4, 8 });
520 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
521 return try function.reduce(function.parameter(0), init, result_type, "sum", &.{2});
522 }
523
524 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
525 const input = std.mem.bytesAsSlice(f32, views[0]);
526 const out = std.mem.bytesAsSlice(f32, output_bytes);
527 for (0..4) |outer| {
528 for (0..8) |middle| {
529 var acc: f32 = init_value;
530 for (0..8) |inner| acc += input[outer * 64 + middle * 8 + inner];
531 out[outer * 8 + middle] = acc;
532 }
533 }
534 }
535 };
536
537 const ReduceSumRankThreeF32 = struct {
538 pub const name = "reduce_sum_axis1_f32_4x8x8";
539 pub const expectation: harness.Expectation = .verified;
540 pub const inputs = [_]harness.Tensor{.{ .dtype = .f32, .dims = &.{ 4, 8, 8 } }};
541 pub const output = harness.mat(.f32, 4, 8);
542 pub const tolerance: f32 = 0.0001;
543 const init_value: f32 = 0.0;
544
545 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
546 const scalar_type = try builder.tensor(.f32, &.{});
547 const result_type = try builder.tensor(.f32, &.{ 4, 8 });
548 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
549 return try function.reduce(function.parameter(0), init, result_type, "sum", &.{1});
550 }
551
552 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
553 const input = std.mem.bytesAsSlice(f32, views[0]);
554 const out = std.mem.bytesAsSlice(f32, output_bytes);
555 for (0..4) |outer| {
556 for (0..8) |inner| {
557 var acc: f32 = init_value;
558 for (0..8) |middle| acc += input[outer * 64 + middle * 8 + inner];
559 out[outer * 8 + inner] = acc;
560 }
561 }
562 }
563 };
564
565 const ReduceMaxRankThreeF32 = struct {
566 pub const name = "reduce_max_axis1_f32_4x8x8";
567 pub const expectation: harness.Expectation = .verified;
568 pub const inputs = [_]harness.Tensor{.{ .dtype = .f32, .dims = &.{ 4, 8, 8 } }};
569 pub const output = harness.mat(.f32, 4, 8);
570 pub const tolerance: f32 = 0.0;
571 const init_value: f32 = -std.math.inf(f32);
572
573 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
574 const scalar_type = try builder.tensor(.f32, &.{});
575 const result_type = try builder.tensor(.f32, &.{ 4, 8 });
576 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
577 return try function.reduce(function.parameter(0), init, result_type, "max", &.{1});
578 }
579
580 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
581 const input = std.mem.bytesAsSlice(f32, views[0]);
582 const out = std.mem.bytesAsSlice(f32, output_bytes);
583 for (0..4) |outer| {
584 for (0..8) |inner| {
585 var acc: f32 = init_value;
586 for (0..8) |middle| acc = @max(acc, input[outer * 64 + middle * 8 + inner]);
587 out[outer * 8 + inner] = acc;
588 }
589 }
590 }
591 };
592
593 const ReduceSumSplitAxesF32 = struct {
594 pub const name = "reduce_sum_axes02_f32_4x8x8";
595 pub const expectation: harness.Expectation = .unsupported;
596 pub const inputs = [_]harness.Tensor{.{ .dtype = .f32, .dims = &.{ 4, 8, 8 } }};
597 pub const output = harness.vec(.f32, 8);
598 pub const tolerance: f32 = 0.0001;
599 const init_value: f32 = 0.0;
600
601 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
602 const scalar_type = try builder.tensor(.f32, &.{});
603 const result_type = try builder.tensor(.f32, &.{8});
604 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
605 return try function.reduce(function.parameter(0), init, result_type, "sum", &.{ 0, 2 });
606 }
607
608 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
609 const input = std.mem.bytesAsSlice(f32, views[0]);
610 const out = std.mem.bytesAsSlice(f32, output_bytes);
611 for (0..8) |middle| {
612 var acc: f32 = init_value;
613 for (0..4) |outer| {
614 for (0..8) |inner| acc += input[outer * 64 + middle * 8 + inner];
615 }
616 out[middle] = acc;
617 }
618 }
619 };
620
621 const GatherClampF32 = struct {
622 pub const name = "gather_clamp_f32_6x8_axis0";
623 pub const expectation: harness.Expectation = .verified;
624 pub const inputs = [_]harness.Tensor{
625 harness.mat(.f32, 6, 8),
626 harness.vec(.i32, 4),
627 };
628 pub const output = harness.mat(.f32, 4, 8);
629 pub const tolerance: f32 = 0.0;
630 const index_values = [_]i32{ -3, 9, 5, 0 };
631
632 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
633 const result_type = try builder.tensor(.f32, &.{ 4, 8 });
634 return try function.gather(function.parameter(0), function.parameter(1), result_type, 0);
635 }
636
637 pub fn fill(input_index: usize, buffer: []u8) void {
638 if (input_index == 1) {
639 const values = std.mem.bytesAsSlice(i32, buffer);
640 for (values, 0..) |*value, position| value.* = index_values[position];
641 return;
642 }
643 harness.defaultFill(.f32, input_index, buffer);
644 }
645
646 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
647 const data = std.mem.bytesAsSlice(f32, views[0]);
648 const indices = std.mem.bytesAsSlice(i32, views[1]);
649 const out = std.mem.bytesAsSlice(f32, output_bytes);
650 for (0..4) |row| {
651 const clamped = std.math.clamp(indices[row], 0, 5);
652 const source: usize = @intCast(clamped);
653 for (0..8) |col| out[row * 8 + col] = data[source * 8 + col];
654 }
655 }
656 };
657
658 fn ScatterCase(
659 comptime case_name: []const u8,
660 comptime dtype: DType,
661 comptime case_expectation: harness.Expectation,
662 comptime index_values: [4]i32,
663 ) type {
664 return struct {
665 pub const name = case_name;
666 pub const expectation: harness.Expectation = case_expectation;
667 pub const inputs = [_]harness.Tensor{
668 harness.mat(dtype, 6, 8),
669 harness.vec(.i32, 4),
670 harness.mat(dtype, 4, 8),
671 };
672 pub const output = harness.mat(dtype, 6, 8);
673 pub const tolerance: f32 = 0.0;
674
675 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
676 const result_type = try builder.tensor(dtype, &.{ 6, 8 });
677 return try function.scatter(
678 function.parameter(0),
679 function.parameter(1),
680 function.parameter(2),
681 result_type,
682 0,
683 );
684 }
685
686 pub fn fill(input_index: usize, buffer: []u8) void {
687 if (input_index == 1) {
688 const values = std.mem.bytesAsSlice(i32, buffer);
689 for (values, 0..) |*value, position| value.* = index_values[position];
690 return;
691 }
692 harness.defaultFill(dtype, input_index, buffer);
693 }
694
695 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
696 const T = dtype.ZigType();
697 const data = std.mem.bytesAsSlice(T, views[0]);
698 const indices = std.mem.bytesAsSlice(i32, views[1]);
699 const updates = std.mem.bytesAsSlice(T, views[2]);
700 const out = std.mem.bytesAsSlice(T, output_bytes);
701 for (out, data) |*value, source| value.* = source;
702 for (0..4) |position| {
703 const target = indices[position];
704 if (target < 0 or target >= 6) continue;
705 const row: usize = @intCast(target);
706 for (0..8) |col| out[row * 8 + col] = updates[position * 8 + col];
707 }
708 }
709 };
710 }
711
712 const BoolMovementOp = enum { transpose, slice, concatenate, pad, gather, scatter };
713
714 fn BoolMovement(comptime op: BoolMovementOp) type {
715 return struct {
716 pub const name = switch (op) {
717 .transpose => "transpose_i1_8x16",
718 .slice => "slice_i1_16x16",
719 .concatenate => "concatenate_i1_64x2",
720 .pad => "pad_i1_6x6",
721 .gather => "gather_i1_6x8_axis0",
722 .scatter => "scatter_i1_6x8_axis0",
723 };
724 pub const expectation: harness.Expectation = .verified;
725 pub const inputs = switch (op) {
726 .transpose => [_]harness.Tensor{harness.mat(.i1, 8, 16)},
727 .slice => [_]harness.Tensor{harness.mat(.i1, 16, 16)},
728 .concatenate => [_]harness.Tensor{ harness.vec(.i1, 64), harness.vec(.i1, 64) },
729 .pad => [_]harness.Tensor{harness.mat(.i1, 6, 6)},
730 .gather => [_]harness.Tensor{ harness.mat(.i1, 6, 8), harness.vec(.i32, 4) },
731 .scatter => [_]harness.Tensor{
732 harness.mat(.i1, 6, 8),
733 harness.vec(.i32, 4),
734 harness.mat(.i1, 4, 8),
735 },
736 };
737 pub const output = switch (op) {
738 .transpose => harness.mat(.i1, 16, 8),
739 .slice => harness.mat(.i1, 8, 8),
740 .concatenate => harness.vec(.i1, 128),
741 .pad => harness.mat(.i1, 8, 8),
742 .gather => harness.mat(.i1, 4, 8),
743 .scatter => harness.mat(.i1, 6, 8),
744 };
745 pub const tolerance: f32 = 0.0;
746 const gather_indices = [_]i32{ 5, 0, 3, 2 };
747 const scatter_indices = [_]i32{ 5, 0, 3, 2 };
748 const pad_value = true;
749
750 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
751 return switch (op) {
752 .transpose => blk: {
753 const result_type = try builder.tensor(.i1, &.{ 16, 8 });
754 break :blk try function.transpose(function.parameter(0), result_type, &.{ 1, 0 });
755 },
756 .slice => blk: {
757 const result_type = try builder.tensor(.i1, &.{ 8, 8 });
758 break :blk try function.slice(function.parameter(0), result_type, &.{ 2, 4 }, &.{ 10, 12 }, &.{ 1, 1 });
759 },
760 .concatenate => blk: {
761 const result_type = try builder.tensor(.i1, &.{128});
762 break :blk try function.concatenate(
763 &.{ function.parameter(0), function.parameter(1) },
764 result_type,
765 0,
766 );
767 },
768 .pad => blk: {
769 const scalar_type = try builder.tensor(.i1, &.{});
770 const result_type = try builder.tensor(.i1, &.{ 8, 8 });
771 const padding = try function.constant(scalar_type, std.mem.asBytes(&pad_value));
772 break :blk try function.pad(
773 function.parameter(0),
774 padding,
775 result_type,
776 &.{ 1, 1 },
777 &.{ 1, 1 },
778 &.{ 0, 0 },
779 );
780 },
781 .gather => blk: {
782 const result_type = try builder.tensor(.i1, &.{ 4, 8 });
783 break :blk try function.gather(function.parameter(0), function.parameter(1), result_type, 0);
784 },
785 .scatter => blk: {
786 const result_type = try builder.tensor(.i1, &.{ 6, 8 });
787 break :blk try function.scatter(
788 function.parameter(0),
789 function.parameter(1),
790 function.parameter(2),
791 result_type,
792 0,
793 );
794 },
795 };
796 }
797
798 pub fn fill(input_index: usize, buffer: []u8) void {
799 if ((op == .gather or op == .scatter) and input_index == 1) {
800 const values = std.mem.bytesAsSlice(i32, buffer);
801 const selected = if (op == .gather) gather_indices else scatter_indices;
802 for (values, 0..) |*value, position| value.* = selected[position];
803 return;
804 }
805 harness.defaultFill(.i1, input_index, buffer);
806 }
807
808 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
809 switch (op) {
810 .transpose => {
811 const input = std.mem.bytesAsSlice(bool, views[0]);
812 const out = std.mem.bytesAsSlice(bool, output_bytes);
813 for (0..16) |row| {
814 for (0..8) |col| out[row * 8 + col] = input[col * 16 + row];
815 }
816 },
817 .slice => {
818 const input = std.mem.bytesAsSlice(bool, views[0]);
819 const out = std.mem.bytesAsSlice(bool, output_bytes);
820 for (0..8) |row| {
821 for (0..8) |col| out[row * 8 + col] = input[(row + 2) * 16 + (col + 4)];
822 }
823 },
824 .concatenate => {
825 const first = std.mem.bytesAsSlice(bool, views[0]);
826 const second = std.mem.bytesAsSlice(bool, views[1]);
827 const out = std.mem.bytesAsSlice(bool, output_bytes);
828 for (first, 0..) |value, index| out[index] = value;
829 for (second, 0..) |value, index| out[64 + index] = value;
830 },
831 .pad => {
832 const input = std.mem.bytesAsSlice(bool, views[0]);
833 const out = std.mem.bytesAsSlice(bool, output_bytes);
834 for (0..8) |row| {
835 for (0..8) |col| {
836 const interior = row >= 1 and row < 7 and col >= 1 and col < 7;
837 out[row * 8 + col] = if (interior) input[(row - 1) * 6 + (col - 1)] else pad_value;
838 }
839 }
840 },
841 .gather => {
842 const data = std.mem.bytesAsSlice(bool, views[0]);
843 const indices = std.mem.bytesAsSlice(i32, views[1]);
844 const out = std.mem.bytesAsSlice(bool, output_bytes);
845 for (0..4) |row| {
846 const source: usize = @intCast(indices[row]);
847 for (0..8) |col| out[row * 8 + col] = data[source * 8 + col];
848 }
849 },
850 .scatter => {
851 const data = std.mem.bytesAsSlice(bool, views[0]);
852 const indices = std.mem.bytesAsSlice(i32, views[1]);
853 const updates = std.mem.bytesAsSlice(bool, views[2]);
854 const out = std.mem.bytesAsSlice(bool, output_bytes);
855 for (out, data) |*value, source| value.* = source;
856 for (0..4) |position| {
857 const target = indices[position];
858 if (target < 0 or target >= 6) continue;
859 const row: usize = @intCast(target);
860 for (0..8) |col| out[row * 8 + col] = updates[position * 8 + col];
861 }
862 },
863 }
864 }
865 };
866 }
867
868 fn boolMovementRows() [6]type {
869 return .{
870 BoolMovement(.transpose),
871 BoolMovement(.slice),
872 BoolMovement(.concatenate),
873 BoolMovement(.pad),
874 BoolMovement(.gather),
875 BoolMovement(.scatter),
876 };
877 }
878
879 const ReducerKind = enum { sum, max, min };
880
881 fn subgroupReduceRequirement() choir_abi.SubgroupRequirements {
882 return .{ .supported = true, .arithmetic = true };
883 }
884
885 fn subgroupScanRequirement() choir_abi.SubgroupRequirements {
886 return .{ .supported = true, .arithmetic = true, .scan = true };
887 }
888
889 fn scatterAddRequiredFeatures(
890 comptime dtype: DType,
891 comptime variant: accy.kernel.library.indexing.ScatterAddVariant,
892 ) choir_abi.Features {
893 return switch (dtype) {
894 .i32 => .{ .atomic_i32 = true },
895 .f32 => switch (variant) {
896 .direct => .{ .atomic_f32_add_device = true },
897 .shared_bins => .{
898 .atomic_f32_add_device = true,
899 .atomic_f32_add_shared = true,
900 },
901 },
902 else => .{ .unsupported_atomic = true },
903 };
904 }
905
906 fn segmentSumRequiredSubgroup(
907 comptime granularity: accy.kernel.library.segmented.SegmentSumGranularity,
908 ) choir_abi.SubgroupRequirements {
909 return switch (granularity) {
910 .thread => .{},
911 .warp => subgroupReduceRequirement(),
912 };
913 }
914
915 fn spmvCsrRequiredSubgroup(
916 comptime structure: accy.kernel.library.sparse.SpmvCsrStructure,
917 ) choir_abi.SubgroupRequirements {
918 return switch (structure) {
919 .row_thread => .{},
920 .row_warp => subgroupReduceRequirement(),
921 };
922 }
923
924 fn spmvCooRequiredFeatures(
925 comptime structure: accy.kernel.library.sparse.SpmvCooStructure,
926 comptime dtype: DType,
927 ) choir_abi.Features {
928 if (comptime structure != .element_thread) return .{};
929 return switch (dtype) {
930 .f32 => .{ .atomic_f32_add_device = true },
931 else => .{ .unsupported_atomic = true },
932 };
933 }
934
935 fn caseName(comptime op_name: []const u8, comptime dtype: DType, comptime suffix: []const u8) []const u8 {
936 @setEvalBranchQuota(200_000);
937 return std.fmt.comptimePrint("{s}_{s}_{s}", .{ op_name, @tagName(dtype), suffix });
938 }
939
940 fn arithmeticExpectation(comptime dtype: DType) harness.Expectation {
941 return switch (dtype) {
942 .f32, .i8, .i16, .i32, .u8, .u16, .u32, .i64, .u64, .f16, .bf16, .f64 => .verified,
943 else => .unsupported,
944 };
945 }
946
947 fn floatOnlyExpectation(comptime dtype: DType) harness.Expectation {
948 if (comptime !dtype.isFloat()) return .invalid;
949 return switch (dtype) {
950 .f32, .f16 => .verified,
951 else => .unsupported,
952 };
953 }
954
955 fn shapeExpectation(comptime dtype: DType) harness.Expectation {
956 return switch (dtype) {
957 .f32, .i8, .i16, .i32, .u8, .u16, .u32, .i64, .u64, .f16, .bf16, .f64 => .verified,
958 else => .unsupported,
959 };
960 }
961
962 fn computeExpectation(comptime dtype: DType) harness.Expectation {
963 return switch (dtype) {
964 .f32, .i32, .u32, .f16 => .verified,
965 else => .unsupported,
966 };
967 }
968
969 fn exactTolerance(comptime dtype: DType) f32 {
970 return switch (dtype) {
971 .f16 => 0.001,
972 .bf16 => 0.01,
973 else => 0.0,
974 };
975 }
976
977 fn divTolerance(comptime dtype: DType) f32 {
978 return switch (dtype) {
979 .f32, .f64 => 0.00001,
980 .f16 => 0.001,
981 .bf16 => 0.01,
982 else => 0.0,
983 };
984 }
985
986 fn powTolerance(comptime dtype: DType) f32 {
987 return switch (dtype) {
988 .f32, .f64 => 0.0001,
989 .f16 => 0.005,
990 .bf16 => 0.05,
991 else => 0.0,
992 };
993 }
994
995 fn transcendentalTolerance(comptime dtype: DType) f32 {
996 return switch (dtype) {
997 .f32, .f64 => 0.00001,
998 .f16 => 0.005,
999 .bf16 => 0.05,
1000 else => 0.0,
1001 };
1002 }
1003
1004 fn binaryRowsFor(comptime dtype: DType) [7]type {
1005 return .{
1006 Binary(caseName("add", dtype, "256"), dtype, .add, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1007 Binary(caseName("sub", dtype, "256"), dtype, .sub, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1008 Binary(caseName("mul", dtype, "256"), dtype, .mul, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1009 Binary(caseName("div", dtype, "256"), dtype, .div, .rhs_positive, divTolerance(dtype), arithmeticExpectation(dtype)),
1010 Binary(caseName("min", dtype, "256"), dtype, .min, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1011 Binary(caseName("max", dtype, "256"), dtype, .max, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1012 Binary(caseName("pow", dtype, "256"), dtype, .pow, .lhs_positive, powTolerance(dtype), floatOnlyExpectation(dtype)),
1013 };
1014 }
1015
1016 fn unaryRowsFor(comptime dtype: DType) [11]type {
1017 return .{
1018 Unary(caseName("neg", dtype, "256"), dtype, .neg, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1019 Unary(caseName("abs", dtype, "256"), dtype, .abs, .default, exactTolerance(dtype), arithmeticExpectation(dtype)),
1020 Unary(caseName("exp", dtype, "256"), dtype, .exp, .default, transcendentalTolerance(dtype), floatOnlyExpectation(dtype)),
1021 Unary(caseName("log", dtype, "256"), dtype, .log, .positive, transcendentalTolerance(dtype), floatOnlyExpectation(dtype)),
1022 Unary(caseName("sqrt", dtype, "256"), dtype, .sqrt, .positive, transcendentalTolerance(dtype), floatOnlyExpectation(dtype)),
1023 Unary(caseName("sin", dtype, "256"), dtype, .sin, .default, transcendentalTolerance(dtype), floatOnlyExpectation(dtype)),
1024 Unary(caseName("cos", dtype, "256"), dtype, .cos, .default, transcendentalTolerance(dtype), floatOnlyExpectation(dtype)),
1025 Unary(caseName("tanh", dtype, "256"), dtype, .tanh, .default, transcendentalTolerance(dtype), floatOnlyExpectation(dtype)),
1026 Unary(caseName("floor", dtype, "256"), dtype, .floor, .default, exactTolerance(dtype), floatOnlyExpectation(dtype)),
1027 Unary(caseName("round", dtype, "256"), dtype, .round, .default, exactTolerance(dtype), floatOnlyExpectation(dtype)),
1028 Unary(caseName("trunc", dtype, "256"), dtype, .trunc, .default, exactTolerance(dtype), floatOnlyExpectation(dtype)),
1029 };
1030 }
1031
1032 fn Iota(comptime dtype: DType) type {
1033 return struct {
1034 pub const name = caseName("iota", dtype, "64");
1035 pub const expectation = shapeExpectation(dtype);
1036 pub const inputs = [_]harness.Tensor{};
1037 pub const output = harness.vec(dtype, 64);
1038 pub const tolerance: f32 = 0.0;
1039
1040 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1041 const result_type = try builder.tensor(dtype, &.{64});
1042 return try function.iota(result_type, 0);
1043 }
1044
1045 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1046 _ = views;
1047 const T = dtype.ZigType();
1048 const out = std.mem.bytesAsSlice(T, output_bytes);
1049 for (out, 0..) |*value, index| {
1050 value.* = iotaValue(dtype, index);
1051 }
1052 }
1053 };
1054 }
1055
1056 fn Transpose(comptime dtype: DType) type {
1057 return struct {
1058 pub const name = caseName("transpose", dtype, "8x16");
1059 pub const expectation = shapeExpectation(dtype);
1060 pub const inputs = [_]harness.Tensor{harness.mat(dtype, 8, 16)};
1061 pub const output = harness.mat(dtype, 16, 8);
1062 pub const tolerance: f32 = 0.0;
1063
1064 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1065 const result_type = try builder.tensor(dtype, &.{ 16, 8 });
1066 return try function.transpose(function.parameter(0), result_type, &.{ 1, 0 });
1067 }
1068
1069 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1070 const T = dtype.ZigType();
1071 const input = std.mem.bytesAsSlice(T, views[0]);
1072 const out = std.mem.bytesAsSlice(T, output_bytes);
1073 for (0..16) |row| {
1074 for (0..8) |col| {
1075 out[row * 8 + col] = input[col * 16 + row];
1076 }
1077 }
1078 }
1079 };
1080 }
1081
1082 fn ReshapeAdd(comptime dtype: DType) type {
1083 return struct {
1084 pub const name = caseName("reshape_add", dtype, "4x32");
1085 pub const expectation = shapeExpectation(dtype);
1086 pub const inputs = [_]harness.Tensor{
1087 harness.mat(dtype, 4, 32),
1088 harness.vec(dtype, 128),
1089 };
1090 pub const output = harness.vec(dtype, 128);
1091 pub const tolerance: f32 = exactTolerance(dtype);
1092
1093 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1094 const result_type = try builder.tensor(dtype, &.{128});
1095 const flat = try function.reshape(function.parameter(0), result_type, &.{128});
1096 return try function.add(flat, function.parameter(1));
1097 }
1098
1099 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1100 const T = dtype.ZigType();
1101 const matrix = std.mem.bytesAsSlice(T, views[0]);
1102 const addend = std.mem.bytesAsSlice(T, views[1]);
1103 const out = std.mem.bytesAsSlice(T, output_bytes);
1104 for (out, matrix, addend) |*value, a, b| value.* = addReference(T, a, b);
1105 }
1106 };
1107 }
1108
1109 fn Slice(comptime dtype: DType) type {
1110 return struct {
1111 pub const name = caseName("slice", dtype, "16x16");
1112 pub const expectation = shapeExpectation(dtype);
1113 pub const inputs = [_]harness.Tensor{harness.mat(dtype, 16, 16)};
1114 pub const output = harness.mat(dtype, 8, 8);
1115 pub const tolerance: f32 = 0.0;
1116
1117 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1118 const result_type = try builder.tensor(dtype, &.{ 8, 8 });
1119 return try function.slice(function.parameter(0), result_type, &.{ 2, 4 }, &.{ 10, 12 }, &.{ 1, 1 });
1120 }
1121
1122 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1123 const T = dtype.ZigType();
1124 const input = std.mem.bytesAsSlice(T, views[0]);
1125 const out = std.mem.bytesAsSlice(T, output_bytes);
1126 for (0..8) |row| {
1127 for (0..8) |col| {
1128 out[row * 8 + col] = input[(row + 2) * 16 + (col + 4)];
1129 }
1130 }
1131 }
1132 };
1133 }
1134
1135 fn Concatenate(comptime dtype: DType) type {
1136 return struct {
1137 pub const name = caseName("concatenate", dtype, "64x2");
1138 pub const expectation = shapeExpectation(dtype);
1139 pub const inputs = [_]harness.Tensor{
1140 harness.vec(dtype, 64),
1141 harness.vec(dtype, 64),
1142 };
1143 pub const output = harness.vec(dtype, 128);
1144 pub const tolerance: f32 = 0.0;
1145
1146 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1147 const result_type = try builder.tensor(dtype, &.{128});
1148 return try function.concatenate(
1149 &.{ function.parameter(0), function.parameter(1) },
1150 result_type,
1151 0,
1152 );
1153 }
1154
1155 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1156 const T = dtype.ZigType();
1157 const first = std.mem.bytesAsSlice(T, views[0]);
1158 const second = std.mem.bytesAsSlice(T, views[1]);
1159 const out = std.mem.bytesAsSlice(T, output_bytes);
1160 for (first, 0..) |value, index| out[index] = value;
1161 for (second, 0..) |value, index| out[64 + index] = value;
1162 }
1163 };
1164 }
1165
1166 fn Pad(comptime dtype: DType) type {
1167 return struct {
1168 pub const name = caseName("pad", dtype, "6x6");
1169 pub const expectation = shapeExpectation(dtype);
1170 pub const inputs = [_]harness.Tensor{harness.mat(dtype, 6, 6)};
1171 pub const output = harness.mat(dtype, 8, 8);
1172 pub const tolerance: f32 = 0.0;
1173 const T: type = dtype.ZigType();
1174 const pad_value: T = literalForDType(dtype, 0.5, 7);
1175
1176 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1177 const scalar_type = try builder.tensor(dtype, &.{});
1178 const result_type = try builder.tensor(dtype, &.{ 8, 8 });
1179 const padding = try function.constant(scalar_type, std.mem.asBytes(&pad_value));
1180 return try function.pad(
1181 function.parameter(0),
1182 padding,
1183 result_type,
1184 &.{ 1, 1 },
1185 &.{ 1, 1 },
1186 &.{ 0, 0 },
1187 );
1188 }
1189
1190 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1191 const input = std.mem.bytesAsSlice(T, views[0]);
1192 const out = std.mem.bytesAsSlice(T, output_bytes);
1193 for (0..8) |row| {
1194 for (0..8) |col| {
1195 const interior = row >= 1 and row < 7 and col >= 1 and col < 7;
1196 out[row * 8 + col] = if (interior) input[(row - 1) * 6 + (col - 1)] else pad_value;
1197 }
1198 }
1199 }
1200 };
1201 }
1202
1203 fn Gather(comptime dtype: DType) type {
1204 return struct {
1205 pub const name = caseName("gather", dtype, "6x8_axis0");
1206 pub const expectation = shapeExpectation(dtype);
1207 pub const inputs = [_]harness.Tensor{
1208 harness.mat(dtype, 6, 8),
1209 harness.vec(.i32, 4),
1210 };
1211 pub const output = harness.mat(dtype, 4, 8);
1212 pub const tolerance: f32 = 0.0;
1213 const index_values = [_]i32{ 5, 0, 3, 2 };
1214
1215 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1216 const result_type = try builder.tensor(dtype, &.{ 4, 8 });
1217 return try function.gather(function.parameter(0), function.parameter(1), result_type, 0);
1218 }
1219
1220 pub fn fill(input_index: usize, buffer: []u8) void {
1221 if (input_index == 1) {
1222 const values = std.mem.bytesAsSlice(i32, buffer);
1223 for (values, 0..) |*value, position| value.* = index_values[position];
1224 return;
1225 }
1226 harness.defaultFill(dtype, input_index, buffer);
1227 }
1228
1229 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1230 const T = dtype.ZigType();
1231 const data = std.mem.bytesAsSlice(T, views[0]);
1232 const indices = std.mem.bytesAsSlice(i32, views[1]);
1233 const out = std.mem.bytesAsSlice(T, output_bytes);
1234 for (0..4) |row| {
1235 const source: usize = @intCast(indices[row]);
1236 for (0..8) |col| out[row * 8 + col] = data[source * 8 + col];
1237 }
1238 }
1239 };
1240 }
1241
1242 fn Reduce(comptime kind: ReducerKind, comptime dtype: DType) type {
1243 return struct {
1244 pub const name = caseName("reduce_" ++ @tagName(kind), dtype, "16x16");
1245 pub const expectation = computeExpectation(dtype);
1246 pub const inputs = [_]harness.Tensor{harness.mat(dtype, 16, 16)};
1247 pub const output = harness.vec(dtype, 16);
1248 pub const tolerance: f32 = switch (kind) {
1249 .sum => if (dtype == .f32 or dtype == .f64) 0.0001 else exactTolerance(dtype),
1250 .max, .min => exactTolerance(dtype),
1251 };
1252 const T: type = dtype.ZigType();
1253 const init_value: T = reduceInitValue(dtype, kind);
1254
1255 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1256 const scalar_type = try builder.tensor(dtype, &.{});
1257 const result_type = try builder.tensor(dtype, &.{16});
1258 const init = try function.constant(scalar_type, std.mem.asBytes(&init_value));
1259 return try function.reduce(function.parameter(0), init, result_type, @tagName(kind), &.{1});
1260 }
1261
1262 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1263 if (comptime T == f16 or T == Bf16) {
1264 const input = std.mem.bytesAsSlice(T, views[0]);
1265 const out = std.mem.bytesAsSlice(T, output_bytes);
1266 for (0..16) |row| {
1267 var acc: f32 = switch (kind) {
1268 .sum => 0,
1269 .max => -std.math.inf(f32),
1270 .min => std.math.inf(f32),
1271 };
1272 for (0..16) |col| {
1273 const value = harness.numericToF32(T, input[row * 16 + col]);
1274 acc = switch (kind) {
1275 .sum => acc + value,
1276 .max => @max(acc, value),
1277 .min => @min(acc, value),
1278 };
1279 }
1280 out[row] = floatLikeFromF32(T, acc);
1281 }
1282 return;
1283 }
1284
1285 const input = std.mem.bytesAsSlice(T, views[0]);
1286 const out = std.mem.bytesAsSlice(T, output_bytes);
1287 for (0..16) |row| {
1288 var acc: T = init_value;
1289 for (0..16) |col| {
1290 const value = input[row * 16 + col];
1291 acc = switch (kind) {
1292 .sum => acc + value,
1293 .max => @max(acc, value),
1294 .min => @min(acc, value),
1295 };
1296 }
1297 out[row] = acc;
1298 }
1299 }
1300 };
1301 }
1302
1303 fn Dot(comptime dtype: DType) type {
1304 return struct {
1305 pub const name = caseName("dot_general", dtype, "16x16");
1306 pub const expectation = computeExpectation(dtype);
1307 pub const inputs = [_]harness.Tensor{
1308 harness.mat(dtype, 16, 16),
1309 harness.mat(dtype, 16, 16),
1310 };
1311 pub const output = harness.mat(dtype, 16, 16);
1312 pub const tolerance: f32 = if (dtype == .f32 or dtype == .f64) 0.0001 else exactTolerance(dtype);
1313
1314 pub fn body(builder: *SemanticBuilder, function: *FunctionBuilder) !*Value {
1315 const result_type = try builder.tensor(dtype, &.{ 16, 16 });
1316 return try function.dotGeneral(
1317 function.parameter(0),
1318 function.parameter(1),
1319 result_type,
1320 &.{1},
1321 &.{0},
1322 &.{},
1323 &.{},
1324 );
1325 }
1326
1327 pub fn reference(views: []const []const u8, output_bytes: []u8) void {
1328 dotReference(dtype.ZigType(), views, output_bytes, 16, 16, 16);
1329 }
1330 };
1331 }
1332
1333 fn shapeRowsFor(comptime dtype: DType) [8]type {
1334 return .{
1335 Iota(dtype),
1336 Transpose(dtype),
1337 ReshapeAdd(dtype),
1338 Slice(dtype),
1339 Concatenate(dtype),
1340 Pad(dtype),
1341 Gather(dtype),
1342 ScatterCase(caseName("scatter", dtype, "6x8_axis0"), dtype, shapeExpectation(dtype), .{ 5, 0, 3, 2 }),
1343 };
1344 }
1345
1346 fn computeRowsFor(comptime dtype: DType) [4]type {
1347 return .{
1348 Dot(dtype),
1349 Reduce(.sum, dtype),
1350 Reduce(.max, dtype),
1351 Reduce(.min, dtype),
1352 };
1353 }
1354
1355 fn productRowsFor(comptime dtype: DType) [30]type {
1356 return binaryRowsFor(dtype) ++ unaryRowsFor(dtype) ++ shapeRowsFor(dtype) ++ computeRowsFor(dtype);
1357 }
1358
1359 fn linalgFamilyBlockCount(extent: u64, threads: u32) u32 {
1360 return @intCast((extent + threads - 1) / threads);
1361 }
1362
1363 fn linalgFamilySignedPattern(element: usize, comptime modulus: usize, comptime shift: i32) f32 {
1364 return @floatFromInt(@as(i32, @intCast(element % modulus)) - shift);
1365 }
1366
1367 const MatrixProductFamilyCase = struct {
1368 const linalg = accy.kernel.library.linalg;
1369 const instance = linalg.MatrixProduct{
1370 .m = 5,
1371 .n = 7,
1372 .k = 3,
1373 .threads = .{ .x = 4, .y = 2 },
1374 };
1375
1376 pub const name = "matrix_product_family_f32_5x7x3";
1377 pub const expectation: harness.Expectation = .verified;
1378 pub const tolerance: f32 = 0;
1379 pub const buffers = [_]harness.FamilyBuffer{
1380 .{ .tensor = harness.vec(.f32, instance.m * instance.n), .access = .inout },
1381 .{ .tensor = harness.vec(.f32, instance.m * instance.k) },
1382 .{ .tensor = harness.vec(.f32, instance.k * instance.n) },
1383 };
1384 pub const observed: usize = 0;
1385 pub const geometry = choir_abi.LaunchGeometry{
1386 .grid = .{ linalgFamilyBlockCount(instance.n, instance.threads.x), linalgFamilyBlockCount(instance.m, instance.threads.y), 1 },
1387 .threadgroup = .{ instance.threads.x, instance.threads.y, 1 },
1388 };
1389
1390 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1391 switch (index) {
1392 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
1393 value.* = 0;
1394 },
1395 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1396 value.* = linalgFamilySignedPattern(element, 7, 3);
1397 },
1398 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1399 value.* = linalgFamilySignedPattern(element * 2 + 1, 9, 4);
1400 },
1401 else => unreachable,
1402 }
1403 }
1404
1405 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
1406 return linalg.matrixProductRuntimeArguments(instance);
1407 }
1408
1409 pub fn buildArtifact(
1410 allocator: std.mem.Allocator,
1411 handle: harness.BackendHandle,
1412 ) !gpu.KernelArtifact {
1413 const entry_name = try linalg.matrixProductFamilyEntryName(allocator, instance);
1414 defer allocator.free(entry_name);
1415 var graph = try linalg.MatrixProductRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
1416 defer graph.deinit();
1417 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1418 .authored_kernel_diagnostic_id = "conformance/matrix-product-family",
1419 });
1420 }
1421
1422 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1423 const lhs = std.mem.bytesAsSlice(f32, seeded[1]);
1424 const rhs = std.mem.bytesAsSlice(f32, seeded[2]);
1425 const out = std.mem.bytesAsSlice(f32, expected);
1426 for (0..instance.m) |row| {
1427 for (0..instance.n) |col| {
1428 var sum: f32 = 0;
1429 for (0..instance.k) |offset| {
1430 sum += lhs[row * instance.k + offset] * rhs[offset * instance.n + col];
1431 }
1432 out[row * instance.n + col] = sum;
1433 }
1434 }
1435 }
1436 };
1437
1438 const BatchedMatrixProductFamilyCase = struct {
1439 const linalg = accy.kernel.library.linalg;
1440 const instance = linalg.BatchedMatrixProduct{
1441 .batch = 3,
1442 .m = 5,
1443 .n = 6,
1444 .k = 4,
1445 .threads = .{ .x = 4, .y = 2, .z = 2 },
1446 };
1447
1448 pub const name = "batched_matrix_product_family_f32_3x5x6x4";
1449 pub const expectation: harness.Expectation = .verified;
1450 pub const tolerance: f32 = 0;
1451 pub const buffers = [_]harness.FamilyBuffer{
1452 .{ .tensor = harness.vec(.f32, instance.batch * instance.m * instance.n), .access = .inout },
1453 .{ .tensor = harness.vec(.f32, instance.batch * instance.m * instance.k) },
1454 .{ .tensor = harness.vec(.f32, instance.batch * instance.k * instance.n) },
1455 };
1456 pub const observed: usize = 0;
1457 pub const geometry = choir_abi.LaunchGeometry{
1458 .grid = .{
1459 linalgFamilyBlockCount(instance.n, instance.threads.x),
1460 linalgFamilyBlockCount(instance.m, instance.threads.y),
1461 linalgFamilyBlockCount(instance.batch, instance.threads.z),
1462 },
1463 .threadgroup = .{ instance.threads.x, instance.threads.y, instance.threads.z },
1464 };
1465
1466 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1467 switch (index) {
1468 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
1469 value.* = 0;
1470 },
1471 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1472 value.* = linalgFamilySignedPattern(element + 3, 11, 5);
1473 },
1474 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1475 value.* = linalgFamilySignedPattern(element * 3 + 1, 13, 6);
1476 },
1477 else => unreachable,
1478 }
1479 }
1480
1481 pub fn runtimeArguments() ![4]choir_abi.ScalarArgument {
1482 return linalg.batchedMatrixProductRuntimeArguments(instance);
1483 }
1484
1485 pub fn buildArtifact(
1486 allocator: std.mem.Allocator,
1487 handle: harness.BackendHandle,
1488 ) !gpu.KernelArtifact {
1489 const entry_name = try linalg.batchedMatrixProductFamilyEntryName(allocator, instance);
1490 defer allocator.free(entry_name);
1491 var graph = try linalg.BatchedMatrixProductRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
1492 defer graph.deinit();
1493 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1494 .authored_kernel_diagnostic_id = "conformance/batched-matrix-product-family",
1495 });
1496 }
1497
1498 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1499 const lhs = std.mem.bytesAsSlice(f32, seeded[1]);
1500 const rhs = std.mem.bytesAsSlice(f32, seeded[2]);
1501 const out = std.mem.bytesAsSlice(f32, expected);
1502 for (0..instance.batch) |batch| {
1503 for (0..instance.m) |row| {
1504 for (0..instance.n) |col| {
1505 var sum: f32 = 0;
1506 for (0..instance.k) |offset| {
1507 const lhs_index = batch * instance.m * instance.k + row * instance.k + offset;
1508 const rhs_index = batch * instance.k * instance.n + offset * instance.n + col;
1509 sum += lhs[lhs_index] * rhs[rhs_index];
1510 }
1511 out[batch * instance.m * instance.n + row * instance.n + col] = sum;
1512 }
1513 }
1514 }
1515 }
1516 };
1517
1518 const MatrixVectorProductFamilyCase = struct {
1519 const linalg = accy.kernel.library.linalg;
1520 const instance = linalg.MatrixVectorProduct{
1521 .m = 5,
1522 .k = 3,
1523 .threads = 4,
1524 };
1525
1526 pub const name = "matrix_vector_product_family_f32_5x3";
1527 pub const expectation: harness.Expectation = .verified;
1528 pub const tolerance: f32 = 0;
1529 pub const buffers = [_]harness.FamilyBuffer{
1530 .{ .tensor = harness.vec(.f32, instance.m), .access = .inout },
1531 .{ .tensor = harness.vec(.f32, instance.m * instance.k) },
1532 .{ .tensor = harness.vec(.f32, instance.k) },
1533 };
1534 pub const observed: usize = 0;
1535 pub const geometry = choir_abi.LaunchGeometry{
1536 .grid = .{ linalgFamilyBlockCount(instance.m, instance.threads), 1, 1 },
1537 .threadgroup = .{ instance.threads, 1, 1 },
1538 };
1539
1540 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1541 switch (index) {
1542 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
1543 value.* = 0;
1544 },
1545 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1546 value.* = linalgFamilySignedPattern(element, 9, 4);
1547 },
1548 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1549 value.* = linalgFamilySignedPattern(element * 2, 7, 3);
1550 },
1551 else => unreachable,
1552 }
1553 }
1554
1555 pub fn runtimeArguments() ![2]choir_abi.ScalarArgument {
1556 return linalg.matrixVectorProductRuntimeArguments(instance);
1557 }
1558
1559 pub fn buildArtifact(
1560 allocator: std.mem.Allocator,
1561 handle: harness.BackendHandle,
1562 ) !gpu.KernelArtifact {
1563 const entry_name = try linalg.matrixVectorProductFamilyEntryName(allocator, instance);
1564 defer allocator.free(entry_name);
1565 var graph = try linalg.MatrixVectorProductRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
1566 defer graph.deinit();
1567 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1568 .authored_kernel_diagnostic_id = "conformance/matrix-vector-product-family",
1569 });
1570 }
1571
1572 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1573 const matrix = std.mem.bytesAsSlice(f32, seeded[1]);
1574 const vector = std.mem.bytesAsSlice(f32, seeded[2]);
1575 const out = std.mem.bytesAsSlice(f32, expected);
1576 for (0..instance.m) |row| {
1577 var sum: f32 = 0;
1578 for (0..instance.k) |offset| {
1579 sum += matrix[row * instance.k + offset] * vector[offset];
1580 }
1581 out[row] = sum;
1582 }
1583 }
1584 };
1585
1586 const OuterProductFamilyCase = struct {
1587 const linalg = accy.kernel.library.linalg;
1588 const instance = linalg.OuterProduct{
1589 .m = 5,
1590 .n = 6,
1591 .threads = .{ .x = 4, .y = 2 },
1592 };
1593
1594 pub const name = "outer_product_family_f32_5x6";
1595 pub const expectation: harness.Expectation = .verified;
1596 pub const tolerance: f32 = 0;
1597 pub const buffers = [_]harness.FamilyBuffer{
1598 .{ .tensor = harness.vec(.f32, instance.m * instance.n), .access = .inout },
1599 .{ .tensor = harness.vec(.f32, instance.m) },
1600 .{ .tensor = harness.vec(.f32, instance.n) },
1601 };
1602 pub const observed: usize = 0;
1603 pub const geometry = choir_abi.LaunchGeometry{
1604 .grid = .{ linalgFamilyBlockCount(instance.n, instance.threads.x), linalgFamilyBlockCount(instance.m, instance.threads.y), 1 },
1605 .threadgroup = .{ instance.threads.x, instance.threads.y, 1 },
1606 };
1607
1608 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1609 switch (index) {
1610 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
1611 value.* = 0;
1612 },
1613 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1614 value.* = linalgFamilySignedPattern(element, 7, 3);
1615 },
1616 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
1617 value.* = linalgFamilySignedPattern(element * 2 + 1, 9, 4);
1618 },
1619 else => unreachable,
1620 }
1621 }
1622
1623 pub fn runtimeArguments() ![2]choir_abi.ScalarArgument {
1624 return linalg.outerProductRuntimeArguments(instance);
1625 }
1626
1627 pub fn buildArtifact(
1628 allocator: std.mem.Allocator,
1629 handle: harness.BackendHandle,
1630 ) !gpu.KernelArtifact {
1631 const entry_name = try linalg.outerProductFamilyEntryName(allocator, instance);
1632 defer allocator.free(entry_name);
1633 var graph = try linalg.OuterProductRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
1634 defer graph.deinit();
1635 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1636 .authored_kernel_diagnostic_id = "conformance/outer-product-family",
1637 });
1638 }
1639
1640 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1641 const lhs = std.mem.bytesAsSlice(f32, seeded[1]);
1642 const rhs = std.mem.bytesAsSlice(f32, seeded[2]);
1643 const out = std.mem.bytesAsSlice(f32, expected);
1644 for (0..instance.m) |row| {
1645 for (0..instance.n) |col| {
1646 out[row * instance.n + col] = lhs[row] * rhs[col];
1647 }
1648 }
1649 }
1650 };
1651
1652 fn GatherFamilyCase(
1653 comptime case_name: []const u8,
1654 comptime case_dtype: DType,
1655 ) type {
1656 return struct {
1657 const indexing = accy.kernel.library.indexing;
1658 const T: type = case_dtype.ZigType();
1659 const outer = 2;
1660 const axis_size = 5;
1661 const gathered = 6;
1662 const inner = 3;
1663 const threads = 32;
1664 const instance = indexing.Gather{
1665 .outer = outer,
1666 .axis_size = axis_size,
1667 .gathered = gathered,
1668 .inner = inner,
1669 .dtype = case_dtype,
1670 .threads = threads,
1671 };
1672
1673 pub const name = case_name;
1674 pub const expectation: harness.Expectation = .verified;
1675 pub const tolerance: f32 = 0;
1676 pub const buffers = [_]harness.FamilyBuffer{
1677 .{ .tensor = harness.vec(case_dtype, instance.total()), .access = .inout },
1678 .{ .tensor = harness.vec(case_dtype, outer * axis_size * inner) },
1679 .{ .tensor = harness.vec(.i32, gathered) },
1680 };
1681 pub const observed: usize = 0;
1682 pub const geometry = choir_abi.LaunchGeometry{
1683 .grid = .{ blockCount(), 1, 1 },
1684 .threadgroup = .{ threads, 1, 1 },
1685 };
1686
1687 fn blockCount() u32 {
1688 return @intCast((instance.total() + instance.threads - 1) / instance.threads);
1689 }
1690
1691 fn dataAt(element: usize) T {
1692 return @floatFromInt(element * 2 + 1);
1693 }
1694
1695 fn indexAt(position: usize) i32 {
1696 const indices = [_]i32{ 4, 1, -2, 3, 7, 0 };
1697 return indices[position];
1698 }
1699
1700 fn clampIndex(index: i32) usize {
1701 if (index <= 0) return 0;
1702 return @min(@as(usize, @intCast(index)), axis_size - 1);
1703 }
1704
1705 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1706 switch (index) {
1707 0 => for (std.mem.bytesAsSlice(T, buffer)) |*value| {
1708 value.* = @floatFromInt(0);
1709 },
1710 1 => for (std.mem.bytesAsSlice(T, buffer), 0..) |*value, element| {
1711 value.* = dataAt(element);
1712 },
1713 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, position| {
1714 value.* = indexAt(position);
1715 },
1716 else => unreachable,
1717 }
1718 }
1719
1720 pub fn runtimeArguments() ![5]choir_abi.ScalarArgument {
1721 return indexing.gatherRuntimeArguments(instance);
1722 }
1723
1724 pub fn buildArtifact(
1725 allocator: std.mem.Allocator,
1726 handle: harness.BackendHandle,
1727 ) !gpu.KernelArtifact {
1728 const entry_name = try indexing.gatherFamilyEntryName(allocator, instance);
1729 defer allocator.free(entry_name);
1730 var graph = switch (case_dtype) {
1731 .f32 => try indexing.GatherRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
1732 .f16 => try indexing.GatherRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
1733 else => unreachable,
1734 };
1735 defer graph.deinit();
1736 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1737 .authored_kernel_diagnostic_id = "conformance/gather-family",
1738 });
1739 }
1740
1741 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1742 _ = seeded[0];
1743 const data = std.mem.bytesAsSlice(T, seeded[1]);
1744 const indices = std.mem.bytesAsSlice(i32, seeded[2]);
1745 const out = std.mem.bytesAsSlice(T, expected);
1746 for (0..outer) |outer_index| {
1747 for (0..gathered) |position| {
1748 const source_axis = clampIndex(indices[position]);
1749 for (0..inner) |within| {
1750 const output = outer_index * gathered * inner + position * inner + within;
1751 const source = outer_index * axis_size * inner + source_axis * inner + within;
1752 out[output] = data[source];
1753 }
1754 }
1755 }
1756 }
1757 };
1758 }
1759
1760 fn ScatterFamilyCase(
1761 comptime case_name: []const u8,
1762 comptime case_dtype: DType,
1763 ) type {
1764 return struct {
1765 const indexing = accy.kernel.library.indexing;
1766 const T: type = case_dtype.ZigType();
1767 const outer = 2;
1768 const axis_size = 5;
1769 const updates = 6;
1770 const inner = 3;
1771 const threads = 32;
1772 const instance = indexing.Scatter{
1773 .outer = outer,
1774 .axis_size = axis_size,
1775 .updates = updates,
1776 .inner = inner,
1777 .dtype = case_dtype,
1778 .threads = threads,
1779 };
1780
1781 pub const name = case_name;
1782 pub const expectation: harness.Expectation = .verified;
1783 pub const tolerance: f32 = 0;
1784 pub const buffers = [_]harness.FamilyBuffer{
1785 .{ .tensor = harness.vec(case_dtype, instance.total()), .access = .inout },
1786 .{ .tensor = harness.vec(case_dtype, instance.total()) },
1787 .{ .tensor = harness.vec(.i32, updates) },
1788 .{ .tensor = harness.vec(case_dtype, outer * updates * inner) },
1789 };
1790 pub const observed: usize = 0;
1791 pub const geometry = choir_abi.LaunchGeometry{
1792 .grid = .{ blockCount(), 1, 1 },
1793 .threadgroup = .{ threads, 1, 1 },
1794 };
1795
1796 fn blockCount() u32 {
1797 return @intCast((instance.total() + instance.threads - 1) / instance.threads);
1798 }
1799
1800 fn dataAt(element: usize) T {
1801 return @floatFromInt(element + 10);
1802 }
1803
1804 fn updateAt(element: usize) T {
1805 return @floatFromInt(element + 100);
1806 }
1807
1808 fn indexAt(position: usize) i32 {
1809 const indices = [_]i32{ 4, 1, -2, 1, 6, 0 };
1810 return indices[position];
1811 }
1812
1813 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1814 switch (index) {
1815 0 => for (std.mem.bytesAsSlice(T, buffer)) |*value| {
1816 value.* = @floatFromInt(0);
1817 },
1818 1 => for (std.mem.bytesAsSlice(T, buffer), 0..) |*value, element| {
1819 value.* = dataAt(element);
1820 },
1821 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, position| {
1822 value.* = indexAt(position);
1823 },
1824 3 => for (std.mem.bytesAsSlice(T, buffer), 0..) |*value, element| {
1825 value.* = updateAt(element);
1826 },
1827 else => unreachable,
1828 }
1829 }
1830
1831 pub fn runtimeArguments() ![5]choir_abi.ScalarArgument {
1832 return indexing.scatterRuntimeArguments(instance);
1833 }
1834
1835 pub fn buildArtifact(
1836 allocator: std.mem.Allocator,
1837 handle: harness.BackendHandle,
1838 ) !gpu.KernelArtifact {
1839 const entry_name = try indexing.scatterFamilyEntryName(allocator, instance);
1840 defer allocator.free(entry_name);
1841 var graph = switch (case_dtype) {
1842 .f32 => try indexing.ScatterRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
1843 .f16 => try indexing.ScatterRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
1844 else => unreachable,
1845 };
1846 defer graph.deinit();
1847 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1848 .authored_kernel_diagnostic_id = "conformance/scatter-family",
1849 });
1850 }
1851
1852 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1853 _ = seeded[0];
1854 const data = std.mem.bytesAsSlice(T, seeded[1]);
1855 const indices = std.mem.bytesAsSlice(i32, seeded[2]);
1856 const update_values = std.mem.bytesAsSlice(T, seeded[3]);
1857 const out = std.mem.bytesAsSlice(T, expected);
1858 for (out, data) |*value, initial| value.* = initial;
1859 for (0..outer) |outer_index| {
1860 for (0..updates) |update_position| {
1861 const target = indices[update_position];
1862 if (target < 0 or target >= axis_size) continue;
1863 const target_axis: usize = @intCast(target);
1864 for (0..inner) |within| {
1865 const output = outer_index * axis_size * inner + target_axis * inner + within;
1866 const update = outer_index * updates * inner + update_position * inner + within;
1867 out[output] = update_values[update];
1868 }
1869 }
1870 }
1871 }
1872 };
1873 }
1874
1875 fn ScatterAddFamilyCase(
1876 comptime case_name: []const u8,
1877 comptime case_dtype: DType,
1878 comptime case_variant: accy.kernel.library.indexing.ScatterAddVariant,
1879 comptime case_tolerance: f32,
1880 ) type {
1881 return struct {
1882 const indexing = accy.kernel.library.indexing;
1883 const bin_count = 16;
1884 const update_count = 64;
1885 const instance = indexing.ScatterAdd{
1886 .axis_size = bin_count,
1887 .updates = update_count,
1888 .dtype = case_dtype,
1889 .variant = case_variant,
1890 .threads = 32,
1891 };
1892
1893 pub const name = case_name;
1894 pub const expectation: harness.Expectation = .verified;
1895 pub const tolerance: f32 = case_tolerance;
1896 pub const required_features = scatterAddRequiredFeatures(case_dtype, case_variant);
1897 pub const buffers = [_]harness.FamilyBuffer{
1898 .{ .tensor = harness.vec(case_dtype, bin_count), .access = .inout },
1899 .{ .tensor = harness.vec(case_dtype, bin_count) },
1900 .{ .tensor = harness.vec(.i32, update_count) },
1901 .{ .tensor = harness.vec(case_dtype, update_count) },
1902 };
1903 pub const observed: usize = 0;
1904 pub const geometry = choir_abi.LaunchGeometry{
1905 .grid = .{ update_count / 32, 1, 1 },
1906 .threadgroup = .{ 32, 1, 1 },
1907 };
1908
1909 fn indexAt(update: usize) i32 {
1910 if (update == 7) return -3;
1911 if (update == 13) return bin_count + 4;
1912 return @intCast((update * 5 + 3) % bin_count);
1913 }
1914
1915 fn updateAt(comptime T: type, update: usize) T {
1916 return switch (@typeInfo(T)) {
1917 .int => @intCast((update % 11) + 1),
1918 else => 0.25 + @as(T, @floatFromInt(update % 11)) * 0.5,
1919 };
1920 }
1921
1922 fn seedAt(comptime T: type, bin: usize) T {
1923 return switch (@typeInfo(T)) {
1924 .int => @intCast(bin % 3),
1925 else => @as(T, @floatFromInt(bin % 3)) * 0.5,
1926 };
1927 }
1928
1929 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
1930 switch (index) {
1931 0, 1 => switch (case_dtype) {
1932 .i32 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, bin| {
1933 value.* = seedAt(i32, bin);
1934 },
1935 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, bin| {
1936 value.* = seedAt(f32, bin);
1937 },
1938 else => unreachable,
1939 },
1940 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, update| {
1941 value.* = indexAt(update);
1942 },
1943 3 => switch (case_dtype) {
1944 .i32 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, update| {
1945 value.* = updateAt(i32, update);
1946 },
1947 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, update| {
1948 value.* = updateAt(f32, update);
1949 },
1950 else => unreachable,
1951 },
1952 else => unreachable,
1953 }
1954 }
1955
1956 pub fn runtimeArguments() ![5]choir_abi.ScalarArgument {
1957 return indexing.scatterAddRuntimeArguments(instance);
1958 }
1959
1960 pub fn buildArtifact(
1961 allocator: std.mem.Allocator,
1962 handle: harness.BackendHandle,
1963 ) !gpu.KernelArtifact {
1964 const entry_name = try indexing.scatterAddFamilyEntryName(allocator, instance);
1965 defer allocator.free(entry_name);
1966 var graph = switch (case_dtype) {
1967 .i32 => try indexing.ScatterAddRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance),
1968 .f32 => try indexing.ScatterAddRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
1969 else => unreachable,
1970 };
1971 defer graph.deinit();
1972 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
1973 .authored_kernel_diagnostic_id = "conformance/scatter-add-family",
1974 });
1975 }
1976
1977 pub fn reference(seeded: []const []const u8, expected: []u8) void {
1978 switch (case_dtype) {
1979 .i32 => referenceTyped(i32, seeded, expected),
1980 .f32 => referenceTyped(f32, seeded, expected),
1981 else => unreachable,
1982 }
1983 }
1984
1985 fn referenceTyped(comptime T: type, seeded: []const []const u8, expected: []u8) void {
1986 const dst = std.mem.bytesAsSlice(T, expected);
1987 const seed = std.mem.bytesAsSlice(T, seeded[0]);
1988 for (dst, seed) |*value, initial| value.* = initial;
1989 const indices = std.mem.bytesAsSlice(i32, seeded[2]);
1990 const updates = std.mem.bytesAsSlice(T, seeded[3]);
1991 for (indices, updates) |index, update| {
1992 if (index < 0 or index >= bin_count) continue;
1993 switch (@typeInfo(T)) {
1994 .int => dst[@intCast(index)] +%= update,
1995 else => dst[@intCast(index)] += update,
1996 }
1997 }
1998 }
1999 };
2000 }
2001
2002 fn PrefixSumFamilyCase(
2003 comptime case_name: []const u8,
2004 comptime dtype: DType,
2005 comptime case_mode: accy.kernel.library.scan.PrefixSumMode,
2006 ) type {
2007 return struct {
2008 const scan = accy.kernel.library.scan;
2009 const extent = 64;
2010 const instance = scan.PrefixSum{
2011 .extent = extent,
2012 .dtype = dtype,
2013 .mode = case_mode,
2014 .threads = 64,
2015 };
2016 const Data: type = dtype.ZigType();
2017
2018 pub const name = case_name;
2019 pub const expectation: harness.Expectation = .verified;
2020 pub const tolerance: f32 = 0;
2021 pub const required_subgroup = subgroupScanRequirement();
2022 pub const buffers = [_]harness.FamilyBuffer{
2023 .{ .tensor = harness.vec(dtype, extent), .access = .inout },
2024 .{ .tensor = harness.vec(dtype, extent) },
2025 };
2026 pub const observed: usize = 0;
2027 pub const geometry = choir_abi.LaunchGeometry{
2028 .grid = .{ 1, 1, 1 },
2029 .threadgroup = .{ instance.threads, 1, 1 },
2030 };
2031
2032 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2033 const values = std.mem.bytesAsSlice(Data, buffer);
2034 switch (index) {
2035 0 => for (values) |*value| {
2036 value.* = numericLikeFromF32(Data, 0);
2037 },
2038 1 => for (values, 0..) |*value, element| {
2039 value.* = numericLikeFromF32(Data, @floatFromInt((element % 7) + 1));
2040 },
2041 else => unreachable,
2042 }
2043 }
2044
2045 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
2046 return scan.prefixSumRuntimeArguments(instance);
2047 }
2048
2049 pub fn buildArtifact(
2050 allocator: std.mem.Allocator,
2051 handle: harness.BackendHandle,
2052 ) !gpu.KernelArtifact {
2053 const entry_name = try scan.prefixSumFamilyEntryName(allocator, instance);
2054 defer allocator.free(entry_name);
2055 var graph = switch (dtype) {
2056 .f32 => try scan.PrefixSumRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
2057 .f16 => try scan.PrefixSumRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
2058 .u32 => try scan.PrefixSumRuntimeFamilyU32.buildNamed(allocator, kernel_limits, entry_name, instance),
2059 else => @compileError("unsupported prefix sum conformance dtype"),
2060 };
2061 defer graph.deinit();
2062 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2063 .authored_kernel_diagnostic_id = "conformance/prefix-sum-family",
2064 });
2065 }
2066
2067 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2068 const data = std.mem.bytesAsSlice(Data, seeded[1]);
2069 const out = std.mem.bytesAsSlice(Data, expected);
2070 var running: Data = numericLikeFromF32(Data, 0);
2071 for (out, data) |*value, element| {
2072 switch (case_mode) {
2073 .inclusive => {
2074 running += element;
2075 value.* = running;
2076 },
2077 .exclusive => {
2078 value.* = running;
2079 running += element;
2080 },
2081 }
2082 }
2083 }
2084 };
2085 }
2086
2087 fn SegmentSumFamilyCase(
2088 comptime case_name: []const u8,
2089 comptime case_granularity: accy.kernel.library.segmented.SegmentSumGranularity,
2090 ) type {
2091 return struct {
2092 const segmented = accy.kernel.library.segmented;
2093 const segments = 8;
2094 const total = 80;
2095 const threads = 64;
2096 const instance = segmented.SegmentSum{
2097 .segments = segments,
2098 .total = total,
2099 .granularity = case_granularity,
2100 .threads = threads,
2101 };
2102
2103 pub const name = case_name;
2104 pub const expectation: harness.Expectation = .verified;
2105 pub const tolerance: f32 = 0;
2106 pub const required_subgroup = segmentSumRequiredSubgroup(case_granularity);
2107 pub const buffers = [_]harness.FamilyBuffer{
2108 .{ .tensor = harness.vec(.f32, segments), .access = .inout },
2109 .{ .tensor = harness.vec(.f32, total) },
2110 .{ .tensor = harness.vec(.i32, segments + 1) },
2111 };
2112 pub const observed: usize = 0;
2113 pub const geometry = choir_abi.LaunchGeometry{
2114 .grid = .{ blockCount(), 1, 1 },
2115 .threadgroup = .{ instance.threads, 1, 1 },
2116 };
2117
2118 fn blockCount() u32 {
2119 const extent = instance.launchExtent();
2120 return @intCast((extent + instance.threads - 1) / instance.threads);
2121 }
2122
2123 fn dataAt(element: usize) f32 {
2124 const magnitude: f32 = @floatFromInt((element % 11) + 1);
2125 return if (element % 2 == 0) magnitude else -magnitude;
2126 }
2127
2128 fn offsetAt(segment_boundary: usize) i32 {
2129 const offsets = [_]i32{ -3, 4, 4, 17, 65, 90, 72, 80, 120 };
2130 return offsets[segment_boundary];
2131 }
2132
2133 fn clampOffset(offset: i32, upper: usize) usize {
2134 if (offset <= 0) return 0;
2135 return @min(@as(usize, @intCast(offset)), upper);
2136 }
2137
2138 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2139 switch (index) {
2140 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
2141 value.* = -999;
2142 },
2143 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
2144 value.* = dataAt(element);
2145 },
2146 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, segment_boundary| {
2147 value.* = offsetAt(segment_boundary);
2148 },
2149 else => unreachable,
2150 }
2151 }
2152
2153 pub fn runtimeArguments() ![2]choir_abi.ScalarArgument {
2154 return segmented.segmentSumRuntimeArguments(instance);
2155 }
2156
2157 pub fn buildArtifact(
2158 allocator: std.mem.Allocator,
2159 handle: harness.BackendHandle,
2160 ) !gpu.KernelArtifact {
2161 const entry_name = try segmented.segmentSumFamilyEntryName(allocator, instance);
2162 defer allocator.free(entry_name);
2163 var graph = try segmented.SegmentSumRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
2164 defer graph.deinit();
2165 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2166 .authored_kernel_diagnostic_id = "conformance/segment-sum-family",
2167 });
2168 }
2169
2170 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2171 const data = std.mem.bytesAsSlice(f32, seeded[1]);
2172 const offsets = std.mem.bytesAsSlice(i32, seeded[2]);
2173 const out = std.mem.bytesAsSlice(f32, expected);
2174 for (out, 0..) |*value, segment| {
2175 const end = clampOffset(offsets[segment + 1], total);
2176 const begin = @min(clampOffset(offsets[segment], total), end);
2177 var sum: f32 = 0;
2178 for (begin..end) |element| sum += data[element];
2179 value.* = sum;
2180 }
2181 }
2182 };
2183 }
2184
2185 fn FilterFamilyCase(
2186 comptime case_name: []const u8,
2187 comptime case_dtype: DType,
2188 comptime case_predicate: accy.kernel.library.entry.CompactionPredicate,
2189 ) type {
2190 return struct {
2191 const compaction = accy.kernel.library.compaction;
2192 const extent = if (case_dtype == .f32) 70 else 40;
2193 const threads = 32;
2194 const blocks = (extent + threads - 1) / threads;
2195 const padded = extent + blocks;
2196 const instance = compaction.Filter{
2197 .extent = extent,
2198 .dtype = case_dtype,
2199 .predicate = case_predicate,
2200 .threads = threads,
2201 };
2202 const RuntimeArguments: type = if (case_predicate == .nonzero)
2203 [1]choir_abi.ScalarArgument
2204 else
2205 [2]choir_abi.ScalarArgument;
2206
2207 pub const name = case_name;
2208 pub const expectation: harness.Expectation = .verified;
2209 pub const tolerance: f32 = 0;
2210 pub const required_subgroup = subgroupScanRequirement();
2211 pub const buffers = [_]harness.FamilyBuffer{
2212 .{ .tensor = harness.vec(case_dtype, padded), .access = .inout },
2213 .{ .tensor = harness.vec(case_dtype, extent) },
2214 };
2215 pub const observed: usize = 0;
2216 pub const geometry = choir_abi.LaunchGeometry{
2217 .grid = .{ blocks, 1, 1 },
2218 .threadgroup = .{ threads, 1, 1 },
2219 };
2220
2221 fn dataAtF32(element: usize) f32 {
2222 if (element % 9 == 0) return 0;
2223 const magnitude: f32 = @floatFromInt((element % 7) + 1);
2224 return if (element % 2 == 0) magnitude else -magnitude * 0.5;
2225 }
2226
2227 fn dataAtI32(element: usize) i32 {
2228 if (element % 5 == 0) return 0;
2229 const raw: i32 = @intCast((element * 7 + 5) % 17);
2230 return raw - 6;
2231 }
2232
2233 fn seedAtF32(element: usize) f32 {
2234 return -777.0 - @as(f32, @floatFromInt(element % 5)) * 0.125;
2235 }
2236
2237 fn seedAtI32(element: usize) i32 {
2238 return -777 - @as(i32, @intCast(element % 5));
2239 }
2240
2241 fn thresholdArgument() choir_abi.ScalarArgument {
2242 return switch (case_dtype) {
2243 .f32 => .{ .f32 = 0.5 },
2244 .i32 => .{ .i32 = 3 },
2245 else => unreachable,
2246 };
2247 }
2248
2249 fn seededSlice(comptime T: type, bytes: []const u8) []const T {
2250 const aligned: []align(@alignOf(T)) const u8 = @alignCast(bytes);
2251 return std.mem.bytesAsSlice(T, aligned);
2252 }
2253
2254 fn expectedSlice(comptime T: type, bytes: []u8) []T {
2255 const aligned: []align(@alignOf(T)) u8 = @alignCast(bytes);
2256 return std.mem.bytesAsSlice(T, aligned);
2257 }
2258
2259 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2260 switch (index) {
2261 0 => switch (case_dtype) {
2262 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
2263 value.* = seedAtF32(element);
2264 },
2265 .i32 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2266 value.* = seedAtI32(element);
2267 },
2268 else => unreachable,
2269 },
2270 1 => switch (case_dtype) {
2271 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
2272 value.* = dataAtF32(element);
2273 },
2274 .i32 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2275 value.* = dataAtI32(element);
2276 },
2277 else => unreachable,
2278 },
2279 else => unreachable,
2280 }
2281 }
2282
2283 pub fn runtimeArguments() !RuntimeArguments {
2284 if (case_predicate == .nonzero) return compaction.filterRuntimeArguments(instance);
2285 return compaction.filterGreaterRuntimeArguments(instance, thresholdArgument());
2286 }
2287
2288 pub fn buildArtifact(
2289 allocator: std.mem.Allocator,
2290 handle: harness.BackendHandle,
2291 ) !gpu.KernelArtifact {
2292 const entry_name = try compaction.filterFamilyEntryName(allocator, instance);
2293 defer allocator.free(entry_name);
2294 var graph = switch (case_predicate) {
2295 .nonzero => switch (case_dtype) {
2296 .f32 => try compaction.FilterRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
2297 .i32 => try compaction.FilterRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance),
2298 else => unreachable,
2299 },
2300 .greater_than => switch (case_dtype) {
2301 .f32 => try compaction.FilterGreaterRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
2302 .i32 => try compaction.FilterGreaterRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance),
2303 else => unreachable,
2304 },
2305 };
2306 defer graph.deinit();
2307 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2308 .authored_kernel_diagnostic_id = "conformance/filter-family",
2309 });
2310 }
2311
2312 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2313 @memcpy(expected, seeded[0]);
2314 switch (case_predicate) {
2315 .nonzero => switch (case_dtype) {
2316 .f32 => compaction.filterBlocksExpectedF32(
2317 seededSlice(f32, seeded[1]),
2318 threads,
2319 expectedSlice(f32, expected),
2320 ),
2321 .i32 => compaction.filterBlocksExpectedI32(
2322 seededSlice(i32, seeded[1]),
2323 threads,
2324 expectedSlice(i32, expected),
2325 ),
2326 else => unreachable,
2327 },
2328 .greater_than => switch (case_dtype) {
2329 .f32 => compaction.filterBlocksGreaterExpectedF32(
2330 seededSlice(f32, seeded[1]),
2331 threads,
2332 thresholdArgument().f32,
2333 expectedSlice(f32, expected),
2334 ),
2335 .i32 => compaction.filterBlocksGreaterExpectedI32(
2336 seededSlice(i32, seeded[1]),
2337 threads,
2338 thresholdArgument().i32,
2339 expectedSlice(i32, expected),
2340 ),
2341 else => unreachable,
2342 },
2343 }
2344 }
2345 };
2346 }
2347
2348 const BitonicBlockFamilyCase = struct {
2349 const sort = accy.kernel.library.sort;
2350 const extent = 45;
2351 const threads = 64;
2352 const instance = sort.BitonicBlock{
2353 .extent = extent,
2354 .threads = threads,
2355 };
2356
2357 pub const name = "bitonic_block_family_i32_45x64";
2358 pub const expectation: harness.Expectation = .verified;
2359 pub const tolerance: f32 = 0;
2360 pub const buffers = [_]harness.FamilyBuffer{
2361 .{ .tensor = harness.vec(.i32, extent), .access = .inout },
2362 .{ .tensor = harness.vec(.i32, extent) },
2363 };
2364 pub const observed: usize = 0;
2365 pub const geometry = choir_abi.LaunchGeometry{
2366 .grid = .{ 1, 1, 1 },
2367 .threadgroup = .{ threads, 1, 1 },
2368 };
2369
2370 fn keyAt(element: usize) i32 {
2371 const raw: i32 = @intCast((element * 37 + 11) % 53);
2372 return if (element % 3 == 0) -raw else raw - 19;
2373 }
2374
2375 fn seededSlice(comptime T: type, bytes: []const u8) []const T {
2376 const aligned: []align(@alignOf(T)) const u8 = @alignCast(bytes);
2377 return std.mem.bytesAsSlice(T, aligned);
2378 }
2379
2380 fn expectedSlice(comptime T: type, bytes: []u8) []T {
2381 const aligned: []align(@alignOf(T)) u8 = @alignCast(bytes);
2382 return std.mem.bytesAsSlice(T, aligned);
2383 }
2384
2385 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2386 switch (index) {
2387 0 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2388 value.* = -5000 - @as(i32, @intCast(element));
2389 },
2390 1 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2391 value.* = keyAt(element);
2392 },
2393 else => unreachable,
2394 }
2395 }
2396
2397 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
2398 return sort.bitonicBlockRuntimeArguments(instance);
2399 }
2400
2401 pub fn buildArtifact(
2402 allocator: std.mem.Allocator,
2403 handle: harness.BackendHandle,
2404 ) !gpu.KernelArtifact {
2405 const entry_name = try sort.bitonicBlockFamilyEntryName(allocator, instance);
2406 defer allocator.free(entry_name);
2407 var graph = try sort.BitonicBlockRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
2408 defer graph.deinit();
2409 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2410 .authored_kernel_diagnostic_id = "conformance/bitonic-block-family",
2411 });
2412 }
2413
2414 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2415 @memcpy(expected, seeded[0]);
2416 const keys = seededSlice(i32, seeded[1]);
2417 const out = expectedSlice(i32, expected);
2418 @memcpy(out, keys);
2419 std.mem.sort(i32, out, {}, std.sort.asc(i32));
2420 }
2421 };
2422
2423 const TopKBlockFamilyCase = struct {
2424 const sort = accy.kernel.library.sort;
2425 const extent = 45;
2426 const top_count = 8;
2427 const threads = 64;
2428 const instance = sort.TopKBlock{
2429 .extent = extent,
2430 .k = top_count,
2431 .threads = threads,
2432 };
2433
2434 pub const name = "top_k_block_family_i32_8of45x64";
2435 pub const expectation: harness.Expectation = .verified;
2436 pub const tolerance: f32 = 0;
2437 pub const buffers = [_]harness.FamilyBuffer{
2438 .{ .tensor = harness.vec(.i32, top_count), .access = .inout },
2439 .{ .tensor = harness.vec(.i32, extent) },
2440 };
2441 pub const observed: usize = 0;
2442 pub const geometry = choir_abi.LaunchGeometry{
2443 .grid = .{ 1, 1, 1 },
2444 .threadgroup = .{ threads, 1, 1 },
2445 };
2446
2447 fn keyAt(element: usize) i32 {
2448 const raw: i32 = @intCast((element * 41 + 5) % 67);
2449 return if (element % 4 == 0) -raw else raw - 23;
2450 }
2451
2452 fn seededSlice(comptime T: type, bytes: []const u8) []const T {
2453 const aligned: []align(@alignOf(T)) const u8 = @alignCast(bytes);
2454 return std.mem.bytesAsSlice(T, aligned);
2455 }
2456
2457 fn expectedSlice(comptime T: type, bytes: []u8) []T {
2458 const aligned: []align(@alignOf(T)) u8 = @alignCast(bytes);
2459 return std.mem.bytesAsSlice(T, aligned);
2460 }
2461
2462 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2463 switch (index) {
2464 0 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2465 value.* = -9000 - @as(i32, @intCast(element));
2466 },
2467 1 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2468 value.* = keyAt(element);
2469 },
2470 else => unreachable,
2471 }
2472 }
2473
2474 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
2475 return sort.topKBlockRuntimeArguments(instance);
2476 }
2477
2478 pub fn buildArtifact(
2479 allocator: std.mem.Allocator,
2480 handle: harness.BackendHandle,
2481 ) !gpu.KernelArtifact {
2482 const entry_name = try sort.topKBlockFamilyEntryName(allocator, instance);
2483 defer allocator.free(entry_name);
2484 var graph = try sort.TopKBlockRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
2485 defer graph.deinit();
2486 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2487 .authored_kernel_diagnostic_id = "conformance/top-k-block-family",
2488 });
2489 }
2490
2491 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2492 @memcpy(expected, seeded[0]);
2493 const keys = seededSlice(i32, seeded[1]);
2494 const out = expectedSlice(i32, expected);
2495 var sorted: [extent]i32 = undefined;
2496 @memcpy(sorted[0..], keys);
2497 std.mem.sort(i32, sorted[0..], {}, std.sort.asc(i32));
2498 @memcpy(out, sorted[0..top_count]);
2499 }
2500 };
2501
2502 fn TopKBlockPairsFamilyCase(
2503 comptime case_name: []const u8,
2504 comptime case_observed: usize,
2505 ) type {
2506 return struct {
2507 const sort = accy.kernel.library.sort;
2508 const extent = 45;
2509 const top_count = 8;
2510 const threads = 64;
2511 const instance = sort.TopKBlockPairs{
2512 .extent = extent,
2513 .k = top_count,
2514 .threads = threads,
2515 };
2516
2517 pub const name = case_name;
2518 pub const expectation: harness.Expectation = .verified;
2519 pub const tolerance: f32 = 0;
2520 pub const buffers = [_]harness.FamilyBuffer{
2521 .{ .tensor = harness.vec(.i32, top_count), .access = .inout },
2522 .{ .tensor = harness.vec(.i32, top_count), .access = .inout },
2523 .{ .tensor = harness.vec(.i32, extent) },
2524 .{ .tensor = harness.vec(.i32, extent) },
2525 };
2526 pub const observed: usize = case_observed;
2527 pub const geometry = choir_abi.LaunchGeometry{
2528 .grid = .{ 1, 1, 1 },
2529 .threadgroup = .{ threads, 1, 1 },
2530 };
2531
2532 fn keyAt(element: usize) i32 {
2533 const raw: i32 = @intCast((element * 41 + 5) % 23);
2534 return if (element % 4 == 0) -raw else raw - 11;
2535 }
2536
2537 fn seededSlice(comptime T: type, bytes: []const u8) []const T {
2538 const aligned: []align(@alignOf(T)) const u8 = @alignCast(bytes);
2539 return std.mem.bytesAsSlice(T, aligned);
2540 }
2541
2542 fn expectedSlice(comptime T: type, bytes: []u8) []T {
2543 const aligned: []align(@alignOf(T)) u8 = @alignCast(bytes);
2544 return std.mem.bytesAsSlice(T, aligned);
2545 }
2546
2547 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2548 switch (index) {
2549 0, 1 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2550 value.* = -9100 - @as(i32, @intCast(index * 100 + element));
2551 },
2552 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2553 value.* = keyAt(element);
2554 },
2555 3 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2556 value.* = @intCast(element);
2557 },
2558 else => unreachable,
2559 }
2560 if (index == 2) {
2561 const values = std.mem.bytesAsSlice(i32, buffer);
2562 values[7] = values[4];
2563 values[13] = values[4];
2564 values[29] = std.math.minInt(i32);
2565 }
2566 }
2567
2568 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
2569 return sort.topKBlockPairsRuntimeArguments(instance);
2570 }
2571
2572 pub fn buildArtifact(
2573 allocator: std.mem.Allocator,
2574 handle: harness.BackendHandle,
2575 ) !gpu.KernelArtifact {
2576 const entry_name = try sort.topKBlockPairsFamilyEntryName(allocator, instance);
2577 defer allocator.free(entry_name);
2578 var graph = try sort.TopKBlockPairsRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
2579 defer graph.deinit();
2580 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2581 .authored_kernel_diagnostic_id = "conformance/top-k-block-pairs-family",
2582 });
2583 }
2584
2585 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2586 @memcpy(expected, seeded[case_observed]);
2587 const keys = seededSlice(i32, seeded[2]);
2588 const values = seededSlice(i32, seeded[3]);
2589 const out = expectedSlice(i32, expected);
2590 const Pair = struct {
2591 key: i32,
2592 value: i32,
2593
2594 fn lessThan(_: void, lhs: @This(), rhs: @This()) bool {
2595 return lhs.key < rhs.key or (lhs.key == rhs.key and lhs.value < rhs.value);
2596 }
2597 };
2598 var sorted: [extent]Pair = undefined;
2599 for (&sorted, keys, values) |*pair, key, value| pair.* = .{ .key = key, .value = value };
2600 std.mem.sort(Pair, sorted[0..], {}, Pair.lessThan);
2601 for (out, sorted[0..top_count]) |*slot, pair| {
2602 slot.* = if (case_observed == 0) pair.key else pair.value;
2603 }
2604 }
2605 };
2606 }
2607
2608 fn deviceScanConformanceAccumulatorDType(comptime dtype: DType) DType {
2609 return switch (dtype) {
2610 .f16, .f32 => .f32,
2611 .u32 => .u32,
2612 else => @compileError("unsupported device scan conformance dtype"),
2613 };
2614 }
2615
2616 fn DeviceScanBlockScanFamilyCase(
2617 comptime case_name: []const u8,
2618 comptime dtype: DType,
2619 comptime case_observed: usize,
2620 ) type {
2621 return struct {
2622 const scan = accy.kernel.library.scan;
2623 const extent = 90;
2624 const threads = 32;
2625 const blocks = 3;
2626 const instance = scan.DeviceScan{ .extent = extent, .dtype = dtype, .threads = threads };
2627 const Out: type = deviceScanBlockScanConformanceOutputDType(dtype).ZigType();
2628 const Data: type = dtype.ZigType();
2629 const Sum: type = deviceScanConformanceAccumulatorDType(dtype).ZigType();
2630
2631 pub const name = case_name;
2632 pub const expectation: harness.Expectation = .verified;
2633 pub const tolerance: f32 = if (dtype == .f16) 0.001 else 0;
2634 pub const required_subgroup = subgroupScanRequirement();
2635 pub const buffers = [_]harness.FamilyBuffer{
2636 .{ .tensor = harness.vec(deviceScanBlockScanConformanceOutputDType(dtype), extent), .access = .inout },
2637 .{ .tensor = harness.vec(dtype, extent) },
2638 .{ .tensor = harness.vec(deviceScanConformanceAccumulatorDType(dtype), blocks), .access = .inout },
2639 };
2640 pub const observed: usize = case_observed;
2641 pub const geometry = choir_abi.LaunchGeometry{
2642 .grid = .{ blocks, 1, 1 },
2643 .threadgroup = .{ threads, 1, 1 },
2644 };
2645
2646 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2647 switch (index) {
2648 0 => {
2649 const values = std.mem.bytesAsSlice(Out, buffer);
2650 for (values) |*value| value.* = numericLikeFromF32(Out, 0);
2651 },
2652 1 => {
2653 const values = std.mem.bytesAsSlice(Data, buffer);
2654 for (values, 0..) |*value, element| {
2655 value.* = numericLikeFromF32(Data, @floatFromInt((element % 7) + 1));
2656 }
2657 },
2658 2 => {
2659 const values = std.mem.bytesAsSlice(Sum, buffer);
2660 for (values) |*value| value.* = numericLikeFromF32(Sum, 0);
2661 },
2662 else => unreachable,
2663 }
2664 }
2665
2666 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
2667 return scan.deviceScanRuntimeArguments(instance);
2668 }
2669
2670 pub fn buildArtifact(
2671 allocator: std.mem.Allocator,
2672 handle: harness.BackendHandle,
2673 ) !gpu.KernelArtifact {
2674 const entry_name = try scan.deviceScanBlockScanFamilyEntryName(allocator, instance);
2675 defer allocator.free(entry_name);
2676 var graph = switch (dtype) {
2677 .f32 => try scan.DeviceScanBlockScanRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
2678 .f16 => try scan.DeviceScanBlockScanRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
2679 .u32 => try scan.DeviceScanBlockScanRuntimeFamilyU32.buildNamed(allocator, kernel_limits, entry_name, instance),
2680 else => @compileError("unsupported device scan conformance dtype"),
2681 };
2682 defer graph.deinit();
2683 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2684 .authored_kernel_diagnostic_id = "conformance/device-scan-block-scan-family",
2685 });
2686 }
2687
2688 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2689 const data = std.mem.bytesAsSlice(Data, seeded[1]);
2690 const out = std.mem.bytesAsSlice(Out, expected);
2691 switch (case_observed) {
2692 0 => {
2693 var running: f32 = 0;
2694 for (out, data, 0..) |*value, element, index| {
2695 if (index % threads == 0) running = 0;
2696 running += harness.numericToF32(Data, element);
2697 value.* = numericLikeFromF32(Out, running);
2698 }
2699 },
2700 2 => for (out, 0..) |*value, block| {
2701 var total: f32 = 0;
2702 const begin = block * threads;
2703 const end = @min(begin + threads, extent);
2704 for (data[begin..end]) |element| total += harness.numericToF32(Data, element);
2705 value.* = numericLikeFromF32(Out, total);
2706 },
2707 else => unreachable,
2708 }
2709 }
2710 };
2711 }
2712
2713 fn deviceScanBlockScanConformanceOutputDType(comptime dtype: DType) DType {
2714 return deviceScanConformanceAccumulatorDType(dtype);
2715 }
2716
2717 fn DeviceScanAddBaseFamilyCase(
2718 comptime case_name: []const u8,
2719 comptime dtype: DType,
2720 ) type {
2721 return struct {
2722 const scan = accy.kernel.library.scan;
2723 const extent = 90;
2724 const threads = 32;
2725 const blocks = 3;
2726 const instance = scan.DeviceScan{ .extent = extent, .dtype = dtype, .threads = threads };
2727 const Out: type = dtype.ZigType();
2728 const Base: type = deviceScanConformanceAccumulatorDType(dtype).ZigType();
2729
2730 pub const name = case_name;
2731 pub const expectation: harness.Expectation = .verified;
2732 pub const tolerance: f32 = if (dtype == .f16) 0.001 else 0;
2733 pub const buffers = switch (dtype) {
2734 .f32, .u32 => [_]harness.FamilyBuffer{
2735 .{ .tensor = harness.vec(dtype, extent), .access = .inout },
2736 .{ .tensor = harness.vec(deviceScanConformanceAccumulatorDType(dtype), blocks) },
2737 },
2738 .f16 => [_]harness.FamilyBuffer{
2739 .{ .tensor = harness.vec(.f16, extent), .access = .inout },
2740 .{ .tensor = harness.vec(.f32, extent) },
2741 .{ .tensor = harness.vec(.f32, blocks) },
2742 },
2743 else => @compileError("unsupported device scan conformance dtype"),
2744 };
2745 pub const observed: usize = 0;
2746 pub const geometry = choir_abi.LaunchGeometry{
2747 .grid = .{ blocks, 1, 1 },
2748 .threadgroup = .{ threads, 1, 1 },
2749 };
2750
2751 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2752 switch (dtype) {
2753 .f32, .u32 => {
2754 const T = if (index == 0) Out else Base;
2755 const values = std.mem.bytesAsSlice(T, buffer);
2756 switch (index) {
2757 0 => for (values, 0..) |*value, element| {
2758 value.* = numericLikeFromF32(T, @floatFromInt((element % 5) + 1));
2759 },
2760 1 => for (values, 0..) |*value, block| {
2761 value.* = numericLikeFromF32(T, @floatFromInt(block * 10));
2762 },
2763 else => unreachable,
2764 }
2765 },
2766 .f16 => switch (index) {
2767 0 => {
2768 const values = std.mem.bytesAsSlice(f16, buffer);
2769 for (values) |*value| value.* = 0;
2770 },
2771 1 => {
2772 const values = std.mem.bytesAsSlice(f32, buffer);
2773 for (values, 0..) |*value, element| {
2774 value.* = @floatFromInt((element % 5) + 1);
2775 }
2776 },
2777 2 => {
2778 const values = std.mem.bytesAsSlice(f32, buffer);
2779 for (values, 0..) |*value, block| {
2780 value.* = @floatFromInt(block * 10);
2781 }
2782 },
2783 else => unreachable,
2784 },
2785 else => @compileError("unsupported device scan conformance dtype"),
2786 }
2787 }
2788
2789 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
2790 return scan.deviceScanRuntimeArguments(instance);
2791 }
2792
2793 pub fn buildArtifact(
2794 allocator: std.mem.Allocator,
2795 handle: harness.BackendHandle,
2796 ) !gpu.KernelArtifact {
2797 const entry_name = try scan.deviceScanAddBaseFamilyEntryName(allocator, instance);
2798 defer allocator.free(entry_name);
2799 var graph = switch (dtype) {
2800 .f32 => try scan.DeviceScanAddBaseRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
2801 .f16 => try scan.DeviceScanAddBaseRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
2802 .u32 => try scan.DeviceScanAddBaseRuntimeFamilyU32.buildNamed(allocator, kernel_limits, entry_name, instance),
2803 else => @compileError("unsupported device scan conformance dtype"),
2804 };
2805 defer graph.deinit();
2806 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2807 .authored_kernel_diagnostic_id = "conformance/device-scan-add-base-family",
2808 });
2809 }
2810
2811 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2812 const out = std.mem.bytesAsSlice(Out, expected);
2813 switch (dtype) {
2814 .f32, .u32 => {
2815 const seed_dst = std.mem.bytesAsSlice(Out, seeded[0]);
2816 const base = std.mem.bytesAsSlice(Base, seeded[1]);
2817 for (out, seed_dst, 0..) |*value, initial, index| {
2818 value.* = numericLikeFromF32(
2819 Out,
2820 harness.numericToF32(Out, initial) + harness.numericToF32(Base, base[index / threads]),
2821 );
2822 }
2823 },
2824 .f16 => {
2825 const local = std.mem.bytesAsSlice(f32, seeded[1]);
2826 const base = std.mem.bytesAsSlice(f32, seeded[2]);
2827 for (out, local, 0..) |*value, prefix, index| {
2828 value.* = floatLikeFromF32(Out, prefix + base[index / threads]);
2829 }
2830 },
2831 else => @compileError("unsupported device scan conformance dtype"),
2832 }
2833 }
2834 };
2835 }
2836
2837 fn SpmvCsrFamilyCase(
2838 comptime case_name: []const u8,
2839 comptime case_structure: accy.kernel.library.sparse.SpmvCsrStructure,
2840 comptime case_dtype: DType,
2841 ) type {
2842 return struct {
2843 const sparse = accy.kernel.library.sparse;
2844 const rows = 48;
2845 const max_row_nnz = 6;
2846 const nnz = total_nnz;
2847 const cols_n = 32;
2848 const instance = sparse.SpmvCsr{
2849 .rows = rows,
2850 .dtype = case_dtype,
2851 .accumulation_dtype = sparse.spmvCsrAccumulationDType(case_dtype).?,
2852 .threads = 64,
2853 .structure = case_structure,
2854 };
2855
2856 const total_nnz = blk: {
2857 var count: usize = 0;
2858 for (0..rows) |row| count += rowNnz(row);
2859 break :blk count;
2860 };
2861
2862 fn rowNnz(row: usize) usize {
2863 return switch (row % 4) {
2864 0 => 0,
2865 1 => 2,
2866 2 => max_row_nnz,
2867 else => 3,
2868 };
2869 }
2870
2871 fn rowBegin(row: usize) usize {
2872 var count: usize = 0;
2873 for (0..row) |prior| count += rowNnz(prior);
2874 return count;
2875 }
2876
2877 fn colAt(element: usize) i32 {
2878 return @intCast((element * 7 + 3) % cols_n);
2879 }
2880
2881 fn valueAt(element: usize) f32 {
2882 return @floatFromInt((element % 9) + 1);
2883 }
2884
2885 fn xAt(index: usize) f32 {
2886 return @floatFromInt((index % 5) + 1);
2887 }
2888
2889 pub const name = case_name;
2890 pub const expectation: harness.Expectation = .verified;
2891 pub const tolerance: f32 = if (case_dtype == .f16) 0.001 else 0;
2892 pub const required_subgroup = spmvCsrRequiredSubgroup(case_structure);
2893 pub const buffers = [_]harness.FamilyBuffer{
2894 .{ .tensor = harness.vec(case_dtype, rows), .access = .inout },
2895 .{ .tensor = harness.vec(.i32, rows + 1) },
2896 .{ .tensor = harness.vec(.i32, nnz) },
2897 .{ .tensor = harness.vec(case_dtype, nnz) },
2898 .{ .tensor = harness.vec(case_dtype, cols_n) },
2899 };
2900 pub const observed: usize = 0;
2901 pub const geometry = choir_abi.LaunchGeometry{
2902 .grid = .{ @intCast(sparse.spmvCsrBlockCount(instance)), 1, 1 },
2903 .threadgroup = .{ instance.threads, 1, 1 },
2904 };
2905
2906 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
2907 switch (index) {
2908 0 => fillFloatBuffer(buffer, -1),
2909 1 => {
2910 const row_ptr = std.mem.bytesAsSlice(i32, buffer);
2911 for (row_ptr, 0..) |*value, row| value.* = @intCast(rowBegin(row));
2912 },
2913 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
2914 value.* = colAt(element);
2915 },
2916 3 => fillIndexedFloatBuffer(buffer, valueAt),
2917 4 => fillIndexedFloatBuffer(buffer, xAt),
2918 else => unreachable,
2919 }
2920 }
2921
2922 fn fillFloatBuffer(buffer: []u8, value: f32) void {
2923 switch (case_dtype) {
2924 .f64 => {
2925 for (std.mem.bytesAsSlice(f64, buffer)) |*item| item.* = @floatCast(value);
2926 },
2927 .f32 => {
2928 for (std.mem.bytesAsSlice(f32, buffer)) |*item| item.* = value;
2929 },
2930 .f16 => {
2931 for (std.mem.bytesAsSlice(f16, buffer)) |*item| item.* = @floatCast(value);
2932 },
2933 else => @compileError("unsupported sparse conformance dtype"),
2934 }
2935 }
2936
2937 fn fillIndexedFloatBuffer(buffer: []u8, comptime valueFn: fn (usize) f32) void {
2938 switch (case_dtype) {
2939 .f64 => for (std.mem.bytesAsSlice(f64, buffer), 0..) |*value, index| {
2940 value.* = @floatCast(valueFn(index));
2941 },
2942 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, index| {
2943 value.* = valueFn(index);
2944 },
2945 .f16 => for (std.mem.bytesAsSlice(f16, buffer), 0..) |*value, index| {
2946 value.* = @floatCast(valueFn(index));
2947 },
2948 else => @compileError("unsupported sparse conformance dtype"),
2949 }
2950 }
2951
2952 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
2953 return sparse.spmvCsrRuntimeArguments(instance, nnz, cols_n);
2954 }
2955
2956 pub fn buildArtifact(
2957 allocator: std.mem.Allocator,
2958 handle: harness.BackendHandle,
2959 ) !gpu.KernelArtifact {
2960 const entry_name = try sparse.spmvCsrFamilyEntryName(allocator, instance);
2961 defer allocator.free(entry_name);
2962 var graph = switch (case_dtype) {
2963 .f64 => try sparse.SpmvCsrRuntimeFamilyF64.buildNamed(allocator, kernel_limits, entry_name, instance),
2964 .f32 => try sparse.SpmvCsrRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
2965 .f16 => try sparse.SpmvCsrRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
2966 else => @compileError("unsupported sparse conformance dtype"),
2967 };
2968 defer graph.deinit();
2969 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
2970 .authored_kernel_diagnostic_id = "conformance/spmv-csr-family",
2971 });
2972 }
2973
2974 pub fn reference(seeded: []const []const u8, expected: []u8) void {
2975 _ = seeded;
2976 for (0..rows) |row| {
2977 var sum: f32 = 0;
2978 const begin = rowBegin(row);
2979 const end = begin + rowNnz(row);
2980 for (begin..end) |element| {
2981 sum += valueAt(element) * xAt(@intCast(colAt(element)));
2982 }
2983 setExpected(expected, row, sum);
2984 }
2985 }
2986
2987 fn setExpected(expected: []u8, row: usize, value: f32) void {
2988 switch (case_dtype) {
2989 .f64 => std.mem.bytesAsSlice(f64, expected)[row] = @floatCast(value),
2990 .f32 => std.mem.bytesAsSlice(f32, expected)[row] = value,
2991 .f16 => std.mem.bytesAsSlice(f16, expected)[row] = @floatCast(value),
2992 else => @compileError("unsupported sparse conformance dtype"),
2993 }
2994 }
2995 };
2996 }
2997
2998 fn SpmvCooFamilyCase(
2999 comptime case_name: []const u8,
3000 comptime case_dtype: DType,
3001 ) type {
3002 return struct {
3003 const sparse = accy.kernel.library.sparse;
3004 const rows = 48;
3005 const nnz = 96;
3006 const x_extent = 32;
3007 const instance = sparse.SpmvCoo{
3008 .rows = rows,
3009 .nnz = nnz,
3010 .x_extent = x_extent,
3011 .dtype = case_dtype,
3012 .accumulation_dtype = sparse.spmvCooAccumulationDType(case_dtype).?,
3013 .threads = 64,
3014 .structure = sparse.spmvCooDefaultStructure(case_dtype).?,
3015 };
3016
3017 fn rowAt(element: usize) i32 {
3018 if (element == 5) return -2;
3019 if (element == 17) return @intCast(rows + 4);
3020 if (element == 24 or element == 25) return 4;
3021 return @intCast((element * 5 + 2) % rows);
3022 }
3023
3024 fn colAt(element: usize) i32 {
3025 if (element == 11) return -3;
3026 if (element == 29) return @intCast(x_extent + 2);
3027 if (element == 24 or element == 25) return 6;
3028 return @intCast((element * 7 + 3) % x_extent);
3029 }
3030
3031 fn valueAt(element: usize) f32 {
3032 return 0.25 + @as(f32, @floatFromInt(element % 11)) * 0.5;
3033 }
3034
3035 fn xAt(index: usize) f32 {
3036 return @floatFromInt((index % 5) + 1);
3037 }
3038
3039 pub const name = case_name;
3040 pub const expectation: harness.Expectation = .verified;
3041 pub const tolerance: f32 = if (case_dtype == .f16) 0.001 else 0;
3042 pub const required_features = spmvCooRequiredFeatures(instance.structure, case_dtype);
3043 pub const buffers = [_]harness.FamilyBuffer{
3044 .{ .tensor = harness.vec(case_dtype, rows), .access = .inout },
3045 .{ .tensor = harness.vec(.i32, nnz) },
3046 .{ .tensor = harness.vec(.i32, nnz) },
3047 .{ .tensor = harness.vec(case_dtype, nnz) },
3048 .{ .tensor = harness.vec(case_dtype, x_extent) },
3049 };
3050 pub const observed: usize = 0;
3051 pub const geometry = choir_abi.LaunchGeometry{
3052 .grid = .{ @intCast(sparse.spmvCooBlockCount(instance)), 1, 1 },
3053 .threadgroup = .{ instance.threads, 1, 1 },
3054 };
3055
3056 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3057 switch (index) {
3058 0 => fillFloatBuffer(buffer, 0),
3059 1 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
3060 value.* = rowAt(element);
3061 },
3062 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
3063 value.* = colAt(element);
3064 },
3065 3 => fillIndexedFloatBuffer(buffer, valueAt),
3066 4 => fillIndexedFloatBuffer(buffer, xAt),
3067 else => unreachable,
3068 }
3069 }
3070
3071 fn fillFloatBuffer(buffer: []u8, value: f32) void {
3072 switch (case_dtype) {
3073 .f64 => for (std.mem.bytesAsSlice(f64, buffer)) |*item| {
3074 item.* = @floatCast(value);
3075 },
3076 .f32 => for (std.mem.bytesAsSlice(f32, buffer)) |*item| {
3077 item.* = value;
3078 },
3079 .f16 => for (std.mem.bytesAsSlice(f16, buffer)) |*item| {
3080 item.* = @floatCast(value);
3081 },
3082 else => @compileError("unsupported sparse conformance dtype"),
3083 }
3084 }
3085
3086 fn fillIndexedFloatBuffer(buffer: []u8, comptime valueFn: fn (usize) f32) void {
3087 switch (case_dtype) {
3088 .f64 => for (std.mem.bytesAsSlice(f64, buffer), 0..) |*value, index| {
3089 value.* = @floatCast(valueFn(index));
3090 },
3091 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, index| {
3092 value.* = valueFn(index);
3093 },
3094 .f16 => for (std.mem.bytesAsSlice(f16, buffer), 0..) |*value, index| {
3095 value.* = @floatCast(valueFn(index));
3096 },
3097 else => @compileError("unsupported sparse conformance dtype"),
3098 }
3099 }
3100
3101 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
3102 return sparse.spmvCooRuntimeArguments(instance, nnz, x_extent);
3103 }
3104
3105 pub fn buildArtifact(
3106 allocator: std.mem.Allocator,
3107 handle: harness.BackendHandle,
3108 ) !gpu.KernelArtifact {
3109 const entry_name = try sparse.spmvCooFamilyEntryName(allocator, instance);
3110 defer allocator.free(entry_name);
3111 var graph = switch (case_dtype) {
3112 .f64 => try sparse.SpmvCooRuntimeFamilyF64.buildNamed(allocator, kernel_limits, entry_name, instance),
3113 .f32 => try sparse.SpmvCooRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
3114 .f16 => try sparse.SpmvCooRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
3115 else => @compileError("unsupported sparse conformance dtype"),
3116 };
3117 defer graph.deinit();
3118 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3119 .authored_kernel_diagnostic_id = "conformance/spmv-coo-family",
3120 });
3121 }
3122
3123 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3124 var sums: [rows]f32 = undefined;
3125 for (&sums, 0..) |*value, row| value.* = seededValue(seeded[0], row);
3126 const row_indices = std.mem.bytesAsSlice(i32, seeded[1]);
3127 const cols = std.mem.bytesAsSlice(i32, seeded[2]);
3128 for (0..nnz) |element| {
3129 const row = row_indices[element];
3130 const col = cols[element];
3131 if (row < 0 or row >= @as(i32, @intCast(rows)) or col < 0 or col >= @as(i32, @intCast(x_extent))) continue;
3132 sums[@intCast(row)] += seededValue(seeded[3], element) * seededValue(seeded[4], @intCast(col));
3133 }
3134 for (sums, 0..) |value, row| {
3135 setExpected(expected, row, value);
3136 }
3137 }
3138
3139 fn seededValue(buffer: []const u8, index: usize) f32 {
3140 return switch (case_dtype) {
3141 .f64 => @floatCast(std.mem.bytesAsSlice(f64, @constCast(buffer))[index]),
3142 .f32 => std.mem.bytesAsSlice(f32, @constCast(buffer))[index],
3143 .f16 => @floatCast(std.mem.bytesAsSlice(f16, @constCast(buffer))[index]),
3144 else => @compileError("unsupported sparse conformance dtype"),
3145 };
3146 }
3147
3148 fn setExpected(expected: []u8, row: usize, value: f32) void {
3149 switch (case_dtype) {
3150 .f64 => std.mem.bytesAsSlice(f64, expected)[row] = @floatCast(value),
3151 .f32 => std.mem.bytesAsSlice(f32, expected)[row] = value,
3152 .f16 => std.mem.bytesAsSlice(f16, expected)[row] = @floatCast(value),
3153 else => @compileError("unsupported sparse conformance dtype"),
3154 }
3155 }
3156 };
3157 }
3158
3159 fn SpmvEllFamilyCase(
3160 comptime case_name: []const u8,
3161 comptime case_dtype: DType,
3162 ) type {
3163 return struct {
3164 const sparse = accy.kernel.library.sparse;
3165 const rows = 48;
3166 const slots = 6;
3167 const x_extent = 32;
3168 const instance = sparse.SpmvEll{
3169 .rows = rows,
3170 .slots = slots,
3171 .x_extent = x_extent,
3172 .dtype = case_dtype,
3173 .accumulation_dtype = sparse.spmvEllAccumulationDType(case_dtype).?,
3174 .threads = 64,
3175 };
3176
3177 fn rowSlots(row: usize) usize {
3178 return switch (row % 5) {
3179 0 => 0,
3180 1 => 1,
3181 2 => 3,
3182 3 => slots,
3183 else => 4,
3184 };
3185 }
3186
3187 fn ellIndex(slot: usize, row: usize) usize {
3188 return slot * rows + row;
3189 }
3190
3191 fn colAt(row: usize, slot: usize) i32 {
3192 return @intCast((row * 7 + slot * 11 + 3) % x_extent);
3193 }
3194
3195 fn valueAt(row: usize, slot: usize) f32 {
3196 return @floatFromInt(((row * 5 + slot * 7) % 9) + 1);
3197 }
3198
3199 fn xAt(index: usize) f32 {
3200 return @floatFromInt((index % 5) + 1);
3201 }
3202
3203 pub const name = case_name;
3204 pub const expectation: harness.Expectation = .verified;
3205 pub const tolerance: f32 = if (case_dtype == .f16) 0.001 else 0;
3206 pub const buffers = [_]harness.FamilyBuffer{
3207 .{ .tensor = harness.vec(case_dtype, rows), .access = .inout },
3208 .{ .tensor = harness.mat(.i32, slots, rows) },
3209 .{ .tensor = harness.mat(case_dtype, slots, rows) },
3210 .{ .tensor = harness.vec(case_dtype, x_extent) },
3211 };
3212 pub const observed: usize = 0;
3213 pub const geometry = choir_abi.LaunchGeometry{
3214 .grid = .{ @intCast(sparse.spmvEllBlockCount(instance)), 1, 1 },
3215 .threadgroup = .{ instance.threads, 1, 1 },
3216 };
3217
3218 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3219 switch (index) {
3220 0 => fillFloatBuffer(buffer, -1),
3221 1 => {
3222 const cols = std.mem.bytesAsSlice(i32, buffer);
3223 for (0..slots) |slot| {
3224 for (0..rows) |row| {
3225 cols[ellIndex(slot, row)] = if (slot < rowSlots(row)) colAt(row, slot) else -1;
3226 }
3227 }
3228 },
3229 2 => fillEllValues(buffer),
3230 3 => fillIndexedFloatBuffer(buffer, xAt),
3231 else => unreachable,
3232 }
3233 }
3234
3235 fn fillFloatBuffer(buffer: []u8, value: f32) void {
3236 switch (case_dtype) {
3237 .f64 => {
3238 for (std.mem.bytesAsSlice(f64, buffer)) |*item| item.* = @floatCast(value);
3239 },
3240 .f32 => {
3241 for (std.mem.bytesAsSlice(f32, buffer)) |*item| item.* = value;
3242 },
3243 .f16 => {
3244 for (std.mem.bytesAsSlice(f16, buffer)) |*item| item.* = @floatCast(value);
3245 },
3246 else => @compileError("unsupported sparse conformance dtype"),
3247 }
3248 }
3249
3250 fn fillEllValues(buffer: []u8) void {
3251 switch (case_dtype) {
3252 .f64 => {
3253 const values = std.mem.bytesAsSlice(f64, buffer);
3254 for (0..slots) |slot| {
3255 for (0..rows) |row| {
3256 values[ellIndex(slot, row)] = @floatCast(valueAt(row, slot));
3257 }
3258 }
3259 },
3260 .f32 => {
3261 const values = std.mem.bytesAsSlice(f32, buffer);
3262 for (0..slots) |slot| {
3263 for (0..rows) |row| {
3264 values[ellIndex(slot, row)] = valueAt(row, slot);
3265 }
3266 }
3267 },
3268 .f16 => {
3269 const values = std.mem.bytesAsSlice(f16, buffer);
3270 for (0..slots) |slot| {
3271 for (0..rows) |row| {
3272 values[ellIndex(slot, row)] = @floatCast(valueAt(row, slot));
3273 }
3274 }
3275 },
3276 else => @compileError("unsupported sparse conformance dtype"),
3277 }
3278 }
3279
3280 fn fillIndexedFloatBuffer(buffer: []u8, comptime valueFn: fn (usize) f32) void {
3281 switch (case_dtype) {
3282 .f64 => for (std.mem.bytesAsSlice(f64, buffer), 0..) |*value, index| {
3283 value.* = @floatCast(valueFn(index));
3284 },
3285 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, index| {
3286 value.* = valueFn(index);
3287 },
3288 .f16 => for (std.mem.bytesAsSlice(f16, buffer), 0..) |*value, index| {
3289 value.* = @floatCast(valueFn(index));
3290 },
3291 else => @compileError("unsupported sparse conformance dtype"),
3292 }
3293 }
3294
3295 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
3296 return sparse.spmvEllRuntimeArguments(instance, slots, x_extent);
3297 }
3298
3299 pub fn buildArtifact(
3300 allocator: std.mem.Allocator,
3301 handle: harness.BackendHandle,
3302 ) !gpu.KernelArtifact {
3303 const entry_name = try sparse.spmvEllFamilyEntryName(allocator, instance);
3304 defer allocator.free(entry_name);
3305 var graph = switch (case_dtype) {
3306 .f64 => try sparse.SpmvEllRuntimeFamilyF64.buildNamed(allocator, kernel_limits, entry_name, instance),
3307 .f32 => try sparse.SpmvEllRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
3308 .f16 => try sparse.SpmvEllRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
3309 else => @compileError("unsupported sparse conformance dtype"),
3310 };
3311 defer graph.deinit();
3312 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3313 .authored_kernel_diagnostic_id = "conformance/spmv-ell-family",
3314 });
3315 }
3316
3317 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3318 _ = seeded;
3319 for (0..rows) |row| {
3320 var sum: f32 = 0;
3321 for (0..rowSlots(row)) |slot| {
3322 sum += valueAt(row, slot) * xAt(@intCast(colAt(row, slot)));
3323 }
3324 setExpected(expected, row, sum);
3325 }
3326 }
3327
3328 fn setExpected(expected: []u8, row: usize, value: f32) void {
3329 switch (case_dtype) {
3330 .f64 => std.mem.bytesAsSlice(f64, expected)[row] = @floatCast(value),
3331 .f32 => std.mem.bytesAsSlice(f32, expected)[row] = value,
3332 .f16 => std.mem.bytesAsSlice(f16, expected)[row] = @floatCast(value),
3333 else => @compileError("unsupported sparse conformance dtype"),
3334 }
3335 }
3336 };
3337 }
3338
3339 fn SpmvSellFamilyCase(
3340 comptime case_name: []const u8,
3341 comptime case_dtype: DType,
3342 ) type {
3343 return struct {
3344 const sparse = accy.kernel.library.sparse;
3345 const rows = 48;
3346 const slice_size = 8;
3347 const slices = (rows + slice_size - 1) / slice_size;
3348 const x_extent = 32;
3349 const values_size = total_values_size;
3350 const instance = sparse.SpmvSell{
3351 .rows = rows,
3352 .slice_size = slice_size,
3353 .values_size = values_size,
3354 .x_extent = x_extent,
3355 .dtype = case_dtype,
3356 .accumulation_dtype = sparse.spmvSellAccumulationDType(case_dtype).?,
3357 .threads = 64,
3358 };
3359
3360 const total_values_size = blk: {
3361 var count: usize = 0;
3362 for (0..slices) |slice| count += sliceSlots(slice) * slice_size;
3363 break :blk count;
3364 };
3365
3366 fn rowSlots(row: usize) usize {
3367 return switch (row % 7) {
3368 0 => 0,
3369 1 => 1,
3370 2 => 2,
3371 3 => 7,
3372 4 => 4,
3373 5 => 6,
3374 else => 3,
3375 };
3376 }
3377
3378 fn sliceSlots(slice: usize) usize {
3379 var count: usize = 0;
3380 for (0..slice_size) |local| {
3381 const row = slice * slice_size + local;
3382 if (row < rows) count = @max(count, rowSlots(row));
3383 }
3384 return count;
3385 }
3386
3387 fn sliceBegin(slice: usize) usize {
3388 var count: usize = 0;
3389 for (0..slice) |prior| count += sliceSlots(prior) * slice_size;
3390 return count;
3391 }
3392
3393 fn sellIndex(row: usize, slot: usize) usize {
3394 const slice = row / slice_size;
3395 const local = row - slice * slice_size;
3396 return sliceBegin(slice) + slot * slice_size + local;
3397 }
3398
3399 fn colAt(row: usize, slot: usize) i32 {
3400 return @intCast((row * 7 + slot * 11 + 3) % x_extent);
3401 }
3402
3403 fn valueAt(row: usize, slot: usize) f32 {
3404 return @floatFromInt(((row * 5 + slot * 7) % 9) + 1);
3405 }
3406
3407 fn xAt(index: usize) f32 {
3408 return @floatFromInt((index % 5) + 1);
3409 }
3410
3411 pub const name = case_name;
3412 pub const expectation: harness.Expectation = .verified;
3413 pub const tolerance: f32 = if (case_dtype == .f16) 0.001 else 0;
3414 pub const buffers = [_]harness.FamilyBuffer{
3415 .{ .tensor = harness.vec(case_dtype, rows), .access = .inout },
3416 .{ .tensor = harness.vec(.i32, slices + 1) },
3417 .{ .tensor = harness.vec(.i32, values_size) },
3418 .{ .tensor = harness.vec(case_dtype, values_size) },
3419 .{ .tensor = harness.vec(case_dtype, x_extent) },
3420 };
3421 pub const observed: usize = 0;
3422 pub const geometry = choir_abi.LaunchGeometry{
3423 .grid = .{ @intCast(sparse.spmvSellBlockCount(instance)), 1, 1 },
3424 .threadgroup = .{ instance.threads, 1, 1 },
3425 };
3426
3427 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3428 switch (index) {
3429 0 => fillFloatBuffer(buffer, -1),
3430 1 => {
3431 const slice_offsets = std.mem.bytesAsSlice(i32, buffer);
3432 for (slice_offsets, 0..) |*value, slice| value.* = @intCast(sliceBegin(slice));
3433 },
3434 2 => {
3435 const cols = std.mem.bytesAsSlice(i32, buffer);
3436 for (cols) |*value| value.* = -1;
3437 for (0..rows) |row| {
3438 for (0..rowSlots(row)) |slot| {
3439 cols[sellIndex(row, slot)] = colAt(row, slot);
3440 }
3441 }
3442 },
3443 3 => fillSellValues(buffer),
3444 4 => fillIndexedFloatBuffer(buffer, xAt),
3445 else => unreachable,
3446 }
3447 }
3448
3449 fn fillFloatBuffer(buffer: []u8, value: f32) void {
3450 switch (case_dtype) {
3451 .f64 => {
3452 for (std.mem.bytesAsSlice(f64, buffer)) |*item| item.* = @floatCast(value);
3453 },
3454 .f32 => {
3455 for (std.mem.bytesAsSlice(f32, buffer)) |*item| item.* = value;
3456 },
3457 .f16 => {
3458 for (std.mem.bytesAsSlice(f16, buffer)) |*item| item.* = @floatCast(value);
3459 },
3460 else => @compileError("unsupported sparse conformance dtype"),
3461 }
3462 }
3463
3464 fn fillSellValues(buffer: []u8) void {
3465 fillFloatBuffer(buffer, 97);
3466 switch (case_dtype) {
3467 .f64 => {
3468 const values = std.mem.bytesAsSlice(f64, buffer);
3469 for (0..rows) |row| {
3470 for (0..rowSlots(row)) |slot| {
3471 values[sellIndex(row, slot)] = @floatCast(valueAt(row, slot));
3472 }
3473 }
3474 },
3475 .f32 => {
3476 const values = std.mem.bytesAsSlice(f32, buffer);
3477 for (0..rows) |row| {
3478 for (0..rowSlots(row)) |slot| {
3479 values[sellIndex(row, slot)] = valueAt(row, slot);
3480 }
3481 }
3482 },
3483 .f16 => {
3484 const values = std.mem.bytesAsSlice(f16, buffer);
3485 for (0..rows) |row| {
3486 for (0..rowSlots(row)) |slot| {
3487 values[sellIndex(row, slot)] = @floatCast(valueAt(row, slot));
3488 }
3489 }
3490 },
3491 else => @compileError("unsupported sparse conformance dtype"),
3492 }
3493 }
3494
3495 fn fillIndexedFloatBuffer(buffer: []u8, comptime valueFn: fn (usize) f32) void {
3496 switch (case_dtype) {
3497 .f64 => for (std.mem.bytesAsSlice(f64, buffer), 0..) |*value, index| {
3498 value.* = @floatCast(valueFn(index));
3499 },
3500 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, index| {
3501 value.* = valueFn(index);
3502 },
3503 .f16 => for (std.mem.bytesAsSlice(f16, buffer), 0..) |*value, index| {
3504 value.* = @floatCast(valueFn(index));
3505 },
3506 else => @compileError("unsupported sparse conformance dtype"),
3507 }
3508 }
3509
3510 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
3511 return sparse.spmvSellRuntimeArguments(instance, values_size, x_extent);
3512 }
3513
3514 pub fn buildArtifact(
3515 allocator: std.mem.Allocator,
3516 handle: harness.BackendHandle,
3517 ) !gpu.KernelArtifact {
3518 const entry_name = try sparse.spmvSellFamilyEntryName(allocator, instance);
3519 defer allocator.free(entry_name);
3520 var graph = switch (case_dtype) {
3521 .f64 => try sparse.SpmvSellRuntimeFamilyF64.buildNamed(allocator, kernel_limits, entry_name, instance),
3522 .f32 => try sparse.SpmvSellRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
3523 .f16 => try sparse.SpmvSellRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
3524 else => @compileError("unsupported sparse conformance dtype"),
3525 };
3526 defer graph.deinit();
3527 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3528 .authored_kernel_diagnostic_id = "conformance/spmv-sell-family",
3529 });
3530 }
3531
3532 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3533 _ = seeded;
3534 for (0..rows) |row| {
3535 var sum: f32 = 0;
3536 for (0..rowSlots(row)) |slot| {
3537 sum += valueAt(row, slot) * xAt(@intCast(colAt(row, slot)));
3538 }
3539 setExpected(expected, row, sum);
3540 }
3541 }
3542
3543 fn setExpected(expected: []u8, row: usize, value: f32) void {
3544 switch (case_dtype) {
3545 .f64 => std.mem.bytesAsSlice(f64, expected)[row] = @floatCast(value),
3546 .f32 => std.mem.bytesAsSlice(f32, expected)[row] = value,
3547 .f16 => std.mem.bytesAsSlice(f16, expected)[row] = @floatCast(value),
3548 else => @compileError("unsupported sparse conformance dtype"),
3549 }
3550 }
3551 };
3552 }
3553
3554 fn SpmmCsrFamilyCase(
3555 comptime case_name: []const u8,
3556 comptime case_dtype: DType,
3557 ) type {
3558 return struct {
3559 const sparse = accy.kernel.library.sparse;
3560 const rows = 32;
3561 const columns = 9;
3562 const max_row_nnz = 5;
3563 const nnz = total_nnz;
3564 const x_extent = 17;
3565 const instance = sparse.SpmmCsr{
3566 .rows = rows,
3567 .columns = columns,
3568 .x_extent = x_extent,
3569 .dtype = case_dtype,
3570 .accumulation_dtype = sparse.spmmCsrAccumulationDType(case_dtype).?,
3571 .threads = .{ .x = 8, .y = 4 },
3572 };
3573
3574 const total_nnz = blk: {
3575 var count: usize = 0;
3576 for (0..rows) |row| count += rowNnz(row);
3577 break :blk count;
3578 };
3579
3580 fn rowNnz(row: usize) usize {
3581 return switch (row % 5) {
3582 0 => 0,
3583 1 => 1,
3584 2 => max_row_nnz,
3585 3 => 3,
3586 else => 2,
3587 };
3588 }
3589
3590 fn rowBegin(row: usize) usize {
3591 var count: usize = 0;
3592 for (0..row) |prior| count += rowNnz(prior);
3593 return count;
3594 }
3595
3596 fn colAt(element: usize) i32 {
3597 return @intCast((element * 5 + 2) % x_extent);
3598 }
3599
3600 fn valueAt(element: usize) f32 {
3601 return @floatFromInt((element % 7) + 1);
3602 }
3603
3604 fn xAt(index: usize) f32 {
3605 return @floatFromInt((index % 11) + 1);
3606 }
3607
3608 pub const name = case_name;
3609 pub const expectation: harness.Expectation = .verified;
3610 pub const tolerance: f32 = if (case_dtype == .f16) 0.001 else 0;
3611 pub const buffers = [_]harness.FamilyBuffer{
3612 .{ .tensor = harness.mat(case_dtype, rows, columns), .access = .inout },
3613 .{ .tensor = harness.vec(.i32, rows + 1) },
3614 .{ .tensor = harness.vec(.i32, nnz) },
3615 .{ .tensor = harness.vec(case_dtype, nnz) },
3616 .{ .tensor = harness.mat(case_dtype, x_extent, columns) },
3617 };
3618 pub const observed: usize = 0;
3619 pub const geometry = choir_abi.LaunchGeometry{
3620 .grid = .{ @intCast(sparse.spmmCsrBlockCountX(instance)), @intCast(sparse.spmmCsrBlockCountY(instance)), 1 },
3621 .threadgroup = .{ instance.threads.x, instance.threads.y, 1 },
3622 };
3623
3624 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3625 switch (index) {
3626 0 => fillFloatBuffer(buffer, -1),
3627 1 => {
3628 const row_ptr = std.mem.bytesAsSlice(i32, buffer);
3629 for (row_ptr, 0..) |*value, row| value.* = @intCast(rowBegin(row));
3630 },
3631 2 => for (std.mem.bytesAsSlice(i32, buffer), 0..) |*value, element| {
3632 value.* = colAt(element);
3633 },
3634 3 => fillIndexedFloatBuffer(buffer, valueAt),
3635 4 => fillIndexedFloatBuffer(buffer, xAt),
3636 else => unreachable,
3637 }
3638 }
3639
3640 fn fillFloatBuffer(buffer: []u8, value: f32) void {
3641 switch (case_dtype) {
3642 .f64 => {
3643 for (std.mem.bytesAsSlice(f64, buffer)) |*item| item.* = @floatCast(value);
3644 },
3645 .f32 => {
3646 for (std.mem.bytesAsSlice(f32, buffer)) |*item| item.* = value;
3647 },
3648 .f16 => {
3649 for (std.mem.bytesAsSlice(f16, buffer)) |*item| item.* = @floatCast(value);
3650 },
3651 else => @compileError("unsupported sparse conformance dtype"),
3652 }
3653 }
3654
3655 fn fillIndexedFloatBuffer(buffer: []u8, comptime valueFn: fn (usize) f32) void {
3656 switch (case_dtype) {
3657 .f64 => for (std.mem.bytesAsSlice(f64, buffer), 0..) |*value, index| {
3658 value.* = @floatCast(valueFn(index));
3659 },
3660 .f32 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, index| {
3661 value.* = valueFn(index);
3662 },
3663 .f16 => for (std.mem.bytesAsSlice(f16, buffer), 0..) |*value, index| {
3664 value.* = @floatCast(valueFn(index));
3665 },
3666 else => @compileError("unsupported sparse conformance dtype"),
3667 }
3668 }
3669
3670 pub fn runtimeArguments() ![4]choir_abi.ScalarArgument {
3671 return sparse.spmmCsrRuntimeArguments(instance, nnz, x_extent, columns);
3672 }
3673
3674 pub fn buildArtifact(
3675 allocator: std.mem.Allocator,
3676 handle: harness.BackendHandle,
3677 ) !gpu.KernelArtifact {
3678 const entry_name = try sparse.spmmCsrFamilyEntryName(allocator, instance);
3679 defer allocator.free(entry_name);
3680 var graph = switch (case_dtype) {
3681 .f64 => try sparse.SpmmCsrRuntimeFamilyF64.buildNamed(allocator, kernel_limits, entry_name, instance),
3682 .f32 => try sparse.SpmmCsrRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance),
3683 .f16 => try sparse.SpmmCsrRuntimeFamilyF16.buildNamed(allocator, kernel_limits, entry_name, instance),
3684 else => @compileError("unsupported sparse conformance dtype"),
3685 };
3686 defer graph.deinit();
3687 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3688 .authored_kernel_diagnostic_id = "conformance/spmm-csr-family",
3689 });
3690 }
3691
3692 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3693 _ = seeded;
3694 for (0..rows) |row| {
3695 for (0..columns) |column| {
3696 var sum: f32 = 0;
3697 const begin = rowBegin(row);
3698 const end = begin + rowNnz(row);
3699 for (begin..end) |element| {
3700 const x_index: usize = @as(usize, @intCast(colAt(element))) * columns + column;
3701 sum += valueAt(element) * xAt(x_index);
3702 }
3703 setExpected(expected, row * columns + column, sum);
3704 }
3705 }
3706 }
3707
3708 fn setExpected(expected: []u8, index: usize, value: f32) void {
3709 switch (case_dtype) {
3710 .f64 => std.mem.bytesAsSlice(f64, expected)[index] = @floatCast(value),
3711 .f32 => std.mem.bytesAsSlice(f32, expected)[index] = value,
3712 .f16 => std.mem.bytesAsSlice(f16, expected)[index] = @floatCast(value),
3713 else => @compileError("unsupported sparse conformance dtype"),
3714 }
3715 }
3716 };
3717 }
3718
3719 const GridCellsFamilyCase = struct {
3720 const spatial = accy.kernel.library.spatial;
3721 const count = 96;
3722 const instance = spatial.GridCells{ .count = count, .threads = 32 };
3723 const grid = spatial.GridGeometry{
3724 .origin_x = -1.0,
3725 .origin_y = -1.0,
3726 .inv_cell_size = 4.0,
3727 .dims_x = 8,
3728 .dims_y = 8,
3729 };
3730
3731 pub const name = "grid_cells_family_f32_96points";
3732 pub const expectation: harness.Expectation = .verified;
3733 pub const tolerance: f32 = 0;
3734 pub const buffers = [_]harness.FamilyBuffer{
3735 .{ .tensor = harness.vec(.i32, count), .access = .inout },
3736 .{ .tensor = harness.vec(.f32, count) },
3737 .{ .tensor = harness.vec(.f32, count) },
3738 };
3739 pub const observed: usize = 0;
3740 pub const geometry = choir_abi.LaunchGeometry{
3741 .grid = .{ @intCast(spatial.gridCellsBlockCount(count, instance.threads)), 1, 1 },
3742 .threadgroup = .{ instance.threads, 1, 1 },
3743 };
3744
3745 fn coordAt(comptime axis: usize, point: usize) f32 {
3746 const mixed = point * 37 + axis * 11;
3747 return @as(f32, @floatFromInt(mixed % 1000)) / 250.0 - 2.0;
3748 }
3749
3750 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3751 switch (index) {
3752 0 => for (std.mem.bytesAsSlice(i32, buffer)) |*value| {
3753 value.* = -1;
3754 },
3755 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
3756 value.* = coordAt(0, point);
3757 },
3758 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
3759 value.* = coordAt(1, point);
3760 },
3761 else => unreachable,
3762 }
3763 }
3764
3765 pub fn runtimeArguments() ![6]choir_abi.ScalarArgument {
3766 return spatial.gridCellsRuntimeArguments(instance, grid);
3767 }
3768
3769 pub fn buildArtifact(
3770 allocator: std.mem.Allocator,
3771 handle: harness.BackendHandle,
3772 ) !gpu.KernelArtifact {
3773 const entry_name = try spatial.gridCellsFamilyEntryName(allocator, instance);
3774 defer allocator.free(entry_name);
3775 var graph = try spatial.GridCellsRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
3776 defer graph.deinit();
3777 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3778 .authored_kernel_diagnostic_id = "conformance/grid-cells-family",
3779 });
3780 }
3781
3782 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3783 _ = seeded;
3784 const out = std.mem.bytesAsSlice(i32, expected);
3785 for (out, 0..) |*value, point| {
3786 const fx = (coordAt(0, point) - grid.origin_x) * grid.inv_cell_size;
3787 const fy = (coordAt(1, point) - grid.origin_y) * grid.inv_cell_size;
3788 const cx = std.math.clamp(@as(i32, @intFromFloat(fx)), 0, @as(i32, @intCast(grid.dims_x - 1)));
3789 const cy = std.math.clamp(@as(i32, @intFromFloat(fy)), 0, @as(i32, @intCast(grid.dims_y - 1)));
3790 value.* = cy * @as(i32, @intCast(grid.dims_x)) + cx;
3791 }
3792 }
3793 };
3794
3795 const ImageBlurPassFamilyCase = struct {
3796 const image = accy.kernel.library.image;
3797 const width = 24;
3798 const height = 10;
3799 const pixel_count = width * height;
3800 const instance = image.BlurPass{ .radius = 2, .axis = .horizontal, .threads = .{ .x = 8, .y = 4 } };
3801 const weights = [_]f32{ 0.0625, 0.25, 0.375, 0.25, 0.0625 };
3802
3803 pub const name = "image_blur_pass_family_r2h_rgba8_24x10";
3804 pub const expectation: harness.Expectation = .verified;
3805 pub const tolerance: f32 = 0;
3806 pub const buffers = [_]harness.FamilyBuffer{
3807 .{ .tensor = harness.vec(.u32, pixel_count), .access = .inout },
3808 .{ .tensor = harness.vec(.u32, pixel_count) },
3809 .{ .tensor = harness.vec(.f32, weights.len) },
3810 };
3811 pub const observed: usize = 0;
3812 pub const geometry = image.imageLaunchGeometry(instance.threads, width, height);
3813
3814 fn pixelAt(index: usize) u32 {
3815 var value: u32 = @truncate(index *% 2654435761);
3816 value ^= value >> 13;
3817 value *%= 0x5bd1e995;
3818 value ^= value >> 15;
3819 return value;
3820 }
3821
3822 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3823 switch (index) {
3824 0 => for (std.mem.bytesAsSlice(u32, buffer)) |*value| {
3825 value.* = 0;
3826 },
3827 1 => for (std.mem.bytesAsSlice(u32, buffer), 0..) |*value, pixel| {
3828 value.* = pixelAt(pixel);
3829 },
3830 2 => for (std.mem.bytesAsSlice(f32, buffer), weights) |*value, weight| {
3831 value.* = weight;
3832 },
3833 else => unreachable,
3834 }
3835 }
3836
3837 pub fn runtimeArguments() ![2]choir_abi.ScalarArgument {
3838 return image.blurPassRuntimeArguments(width, height);
3839 }
3840
3841 pub fn buildArtifact(
3842 allocator: std.mem.Allocator,
3843 handle: harness.BackendHandle,
3844 ) !gpu.KernelArtifact {
3845 const entry_name = try image.blurPassFamilyEntryName(allocator, instance);
3846 defer allocator.free(entry_name);
3847 var graph = try image.BlurPassFamilyRgba8.buildNamed(allocator, kernel_limits, entry_name, instance);
3848 defer graph.deinit();
3849 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3850 .authored_kernel_diagnostic_id = "conformance/image-blur-pass-family",
3851 });
3852 }
3853
3854 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3855 _ = seeded;
3856 var src_pixels: [pixel_count]u32 = undefined;
3857 for (&src_pixels, 0..) |*value, pixel| value.* = pixelAt(pixel);
3858 const out = std.mem.bytesAsSlice(u32, expected);
3859 image.referenceBlurPass(@alignCast(out), src_pixels[0..], weights[0..], width, height, instance.axis);
3860 }
3861 };
3862
3863 const ImageResizeFamilyCase = struct {
3864 const image = accy.kernel.library.image;
3865 const src_width = 20;
3866 const src_height = 12;
3867 const dst_width = 11;
3868 const dst_height = 7;
3869 const instance = image.Resize{ .threads = .{ .x = 8, .y = 4 } };
3870
3871 pub const name = "image_resize_bilinear_family_rgba8_20x12_to_11x7";
3872 pub const expectation: harness.Expectation = .verified;
3873 pub const tolerance: f32 = 0;
3874 pub const buffers = [_]harness.FamilyBuffer{
3875 .{ .tensor = harness.vec(.u32, dst_width * dst_height), .access = .inout },
3876 .{ .tensor = harness.vec(.u32, src_width * src_height) },
3877 };
3878 pub const observed: usize = 0;
3879 pub const geometry = image.imageLaunchGeometry(instance.threads, dst_width, dst_height);
3880
3881 fn pixelAt(index: usize) u32 {
3882 var value: u32 = @truncate((index +% 31) *% 2246822519);
3883 value ^= value >> 15;
3884 value *%= 0x9e3779b1;
3885 value ^= value >> 13;
3886 return value;
3887 }
3888
3889 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3890 switch (index) {
3891 0 => for (std.mem.bytesAsSlice(u32, buffer)) |*value| {
3892 value.* = 0;
3893 },
3894 1 => for (std.mem.bytesAsSlice(u32, buffer), 0..) |*value, pixel| {
3895 value.* = pixelAt(pixel);
3896 },
3897 else => unreachable,
3898 }
3899 }
3900
3901 pub fn runtimeArguments() ![7]choir_abi.ScalarArgument {
3902 return image.resizeRuntimeArguments(dst_width, dst_height, src_width, src_height);
3903 }
3904
3905 pub fn buildArtifact(
3906 allocator: std.mem.Allocator,
3907 handle: harness.BackendHandle,
3908 ) !gpu.KernelArtifact {
3909 const entry_name = try image.resizeFamilyEntryName(allocator, instance);
3910 defer allocator.free(entry_name);
3911 var graph = try image.ResizeBilinearFamilyRgba8.buildNamed(allocator, kernel_limits, entry_name, instance);
3912 defer graph.deinit();
3913 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
3914 .authored_kernel_diagnostic_id = "conformance/image-resize-bilinear-family",
3915 });
3916 }
3917
3918 pub fn reference(seeded: []const []const u8, expected: []u8) void {
3919 _ = seeded;
3920 var src_pixels: [src_width * src_height]u32 = undefined;
3921 for (&src_pixels, 0..) |*value, pixel| value.* = pixelAt(pixel);
3922 const out = std.mem.bytesAsSlice(u32, expected);
3923 image.referenceResizeBilinear(@alignCast(out), src_pixels[0..], dst_width, dst_height, src_width, src_height);
3924 }
3925 };
3926
3927 fn SdfGridSampleGradient2DCase(comptime case_name: []const u8, comptime observed_index: usize) type {
3928 return struct {
3929 const sdf = accy.kernel.library.sdf;
3930 const nx = 17;
3931 const ny = 13;
3932 const count = 96;
3933 const origin: f32 = -1.0;
3934 const spacing: f32 = 0.125;
3935 const instance = sdf.GridSample{ .gradient = true, .threads = 32 };
3936
3937 pub const name = case_name;
3938 pub const expectation: harness.Expectation = .verified;
3939 pub const tolerance: f32 = 0.000001;
3940 pub const buffers = [_]harness.FamilyBuffer{
3941 .{ .tensor = harness.vec(.f32, count), .access = .inout },
3942 .{ .tensor = harness.vec(.f32, count), .access = .inout },
3943 .{ .tensor = harness.vec(.f32, count), .access = .inout },
3944 .{ .tensor = harness.vec(.f32, nx * ny) },
3945 .{ .tensor = harness.vec(.f32, count) },
3946 .{ .tensor = harness.vec(.f32, count) },
3947 };
3948 pub const observed: usize = observed_index;
3949 pub const geometry = choir_abi.LaunchGeometry{
3950 .grid = .{ @intCast(sdf.gridSampleBlockCount(count, instance.threads)), 1, 1 },
3951 .threadgroup = .{ instance.threads, 1, 1 },
3952 };
3953
3954 fn gridValueAt(index: usize) f32 {
3955 const ix = index % nx;
3956 const iy = index / nx;
3957 const x = origin + @as(f32, @floatFromInt(ix)) * spacing;
3958 const y = origin + @as(f32, @floatFromInt(iy)) * spacing;
3959 return @sqrt(x * x + y * y) - 0.75;
3960 }
3961
3962 fn queryAt(comptime axis: usize, point: usize) f32 {
3963 const mixed = point * 41 + axis * 13;
3964 return -1.4 + @as(f32, @floatFromInt(mixed % 1000)) / 340.0;
3965 }
3966
3967 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
3968 switch (index) {
3969 0, 1, 2 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
3970 value.* = 0;
3971 },
3972 3 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, cell| {
3973 value.* = gridValueAt(cell);
3974 },
3975 4 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
3976 value.* = queryAt(0, point);
3977 },
3978 5 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
3979 value.* = queryAt(1, point);
3980 },
3981 else => unreachable,
3982 }
3983 }
3984
3985 pub fn runtimeArguments() ![7]choir_abi.ScalarArgument {
3986 return .{
3987 .{ .i32 = count },
3988 .{ .f32 = origin },
3989 .{ .f32 = origin },
3990 .{ .f32 = 1.0 / spacing },
3991 .{ .f32 = 1.0 / spacing },
3992 .{ .i32 = nx },
3993 .{ .i32 = ny },
3994 };
3995 }
3996
3997 pub fn buildArtifact(
3998 allocator: std.mem.Allocator,
3999 handle: harness.BackendHandle,
4000 ) !gpu.KernelArtifact {
4001 var graph = try sdf.GridSampleGradientFamily2D.build(allocator, kernel_limits, instance);
4002 defer graph.deinit();
4003 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4004 .authored_kernel_diagnostic_id = "conformance/sdf-grid-sample-gradient-2d-family",
4005 });
4006 }
4007
4008 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4009 _ = seeded;
4010 var grid_values: [nx * ny]f32 = undefined;
4011 for (&grid_values, 0..) |*value, cell| value.* = gridValueAt(cell);
4012 const grid = sdf.ReferenceGrid2{
4013 .values = grid_values[0..],
4014 .nx = nx,
4015 .ny = ny,
4016 .origin_x = origin,
4017 .origin_y = origin,
4018 .inv_dx = 1.0 / spacing,
4019 .inv_dy = 1.0 / spacing,
4020 };
4021 const out = std.mem.bytesAsSlice(f32, expected);
4022 for (0..count) |point| {
4023 var gradient: [2]f32 = undefined;
4024 const distance = sdf.referenceGridSample2(grid, queryAt(0, point), queryAt(1, point), &gradient);
4025 out[point] = switch (observed_index) {
4026 0 => distance,
4027 1 => gradient[0],
4028 2 => gradient[1],
4029 else => unreachable,
4030 };
4031 }
4032 }
4033 };
4034 }
4035
4036 const SdfGridSample3DFamilyCase = struct {
4037 const sdf = accy.kernel.library.sdf;
4038 const nx = 9;
4039 const ny = 7;
4040 const nz = 6;
4041 const count = 64;
4042 const origin: f32 = -1.0;
4043 const spacing: f32 = 0.3;
4044 const instance = sdf.GridSample{ .threads = 32 };
4045
4046 pub const name = "sdf_grid_sample_family_f32_3d_64points";
4047 pub const expectation: harness.Expectation = .verified;
4048 pub const tolerance: f32 = 0;
4049 pub const buffers = [_]harness.FamilyBuffer{
4050 .{ .tensor = harness.vec(.f32, count), .access = .inout },
4051 .{ .tensor = harness.vec(.f32, nx * ny * nz) },
4052 .{ .tensor = harness.vec(.f32, count) },
4053 .{ .tensor = harness.vec(.f32, count) },
4054 .{ .tensor = harness.vec(.f32, count) },
4055 };
4056 pub const observed: usize = 0;
4057 pub const geometry = choir_abi.LaunchGeometry{
4058 .grid = .{ @intCast(sdf.gridSampleBlockCount(count, instance.threads)), 1, 1 },
4059 .threadgroup = .{ instance.threads, 1, 1 },
4060 };
4061
4062 fn gridValueAt(index: usize) f32 {
4063 const iz = index / (nx * ny);
4064 const rem = index % (nx * ny);
4065 const iy = rem / nx;
4066 const ix = rem % nx;
4067 const x = origin + @as(f32, @floatFromInt(ix)) * spacing;
4068 const y = origin + @as(f32, @floatFromInt(iy)) * spacing;
4069 const z = origin + @as(f32, @floatFromInt(iz)) * spacing;
4070 return @sqrt(x * x + y * y + z * z) - 0.9;
4071 }
4072
4073 fn queryAt(comptime axis: usize, point: usize) f32 {
4074 const mixed = point * 53 + axis * 19;
4075 return -1.5 + @as(f32, @floatFromInt(mixed % 1000)) / 310.0;
4076 }
4077
4078 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4079 switch (index) {
4080 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
4081 value.* = 0;
4082 },
4083 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, cell| {
4084 value.* = gridValueAt(cell);
4085 },
4086 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
4087 value.* = queryAt(0, point);
4088 },
4089 3 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
4090 value.* = queryAt(1, point);
4091 },
4092 4 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
4093 value.* = queryAt(2, point);
4094 },
4095 else => unreachable,
4096 }
4097 }
4098
4099 pub fn runtimeArguments() ![10]choir_abi.ScalarArgument {
4100 return .{
4101 .{ .i32 = count },
4102 .{ .f32 = origin },
4103 .{ .f32 = origin },
4104 .{ .f32 = origin },
4105 .{ .f32 = 1.0 / spacing },
4106 .{ .f32 = 1.0 / spacing },
4107 .{ .f32 = 1.0 / spacing },
4108 .{ .i32 = nx },
4109 .{ .i32 = ny },
4110 .{ .i32 = nz },
4111 };
4112 }
4113
4114 pub fn buildArtifact(
4115 allocator: std.mem.Allocator,
4116 handle: harness.BackendHandle,
4117 ) !gpu.KernelArtifact {
4118 var graph = try sdf.GridSampleFamily3D.build(allocator, kernel_limits, instance);
4119 defer graph.deinit();
4120 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4121 .authored_kernel_diagnostic_id = "conformance/sdf-grid-sample-3d-family",
4122 });
4123 }
4124
4125 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4126 _ = seeded;
4127 var grid_values: [nx * ny * nz]f32 = undefined;
4128 for (&grid_values, 0..) |*value, cell| value.* = gridValueAt(cell);
4129 const grid = sdf.ReferenceGrid3{
4130 .values = grid_values[0..],
4131 .nx = nx,
4132 .ny = ny,
4133 .nz = nz,
4134 .origin_x = origin,
4135 .origin_y = origin,
4136 .origin_z = origin,
4137 .inv_dx = 1.0 / spacing,
4138 .inv_dy = 1.0 / spacing,
4139 .inv_dz = 1.0 / spacing,
4140 };
4141 const out = std.mem.bytesAsSlice(f32, expected);
4142 for (0..count) |point| {
4143 out[point] = sdf.referenceGridSample3(grid, queryAt(0, point), queryAt(1, point), queryAt(2, point), null);
4144 }
4145 }
4146 };
4147
4148 const BatchedCholeskyFamilyCase = struct {
4149 const factor = accy.kernel.library.factor;
4150 const n = 3;
4151 const batch = 48;
4152 const instance = factor.BatchedCholesky{ .batch = batch, .n = n, .threads = 32 };
4153
4154 pub const name = "batched_cholesky_family_f32_48x3";
4155 pub const expectation: harness.Expectation = .verified;
4156 pub const tolerance: f32 = 0.00001;
4157 pub const buffers = [_]harness.FamilyBuffer{
4158 .{ .tensor = harness.vec(.f32, batch * n * n), .access = .inout },
4159 .{ .tensor = harness.vec(.f32, batch * n * n) },
4160 };
4161 pub const observed: usize = 0;
4162 pub const geometry = choir_abi.LaunchGeometry{
4163 .grid = .{ @intCast(factor.batchedCholeskyBlockCount(batch, instance.threads)), 1, 1 },
4164 .threadgroup = .{ instance.threads, 1, 1 },
4165 };
4166
4167 fn tileEntry(system: usize, row: usize, col: usize) f32 {
4168 var sum: f32 = 0;
4169 for (0..n) |c| {
4170 sum += mixedEntry(system, row, c) * mixedEntry(system, col, c);
4171 }
4172 if (row == col) sum += @floatFromInt(n);
4173 return sum;
4174 }
4175
4176 fn mixedEntry(system: usize, row: usize, col: usize) f32 {
4177 const mixed = system * 131 + row * 17 + col * 7;
4178 return @as(f32, @floatFromInt(mixed % 1000)) / 1000.0;
4179 }
4180
4181 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4182 switch (index) {
4183 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
4184 value.* = -1;
4185 },
4186 1 => {
4187 const values = std.mem.bytesAsSlice(f32, buffer);
4188 for (0..batch) |system| {
4189 for (0..n) |row| {
4190 for (0..n) |col| {
4191 values[system * n * n + row * n + col] = tileEntry(system, row, col);
4192 }
4193 }
4194 }
4195 },
4196 else => unreachable,
4197 }
4198 }
4199
4200 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
4201 return factor.batchedCholeskyRuntimeArguments(instance);
4202 }
4203
4204 pub fn buildArtifact(
4205 allocator: std.mem.Allocator,
4206 handle: harness.BackendHandle,
4207 ) !gpu.KernelArtifact {
4208 const entry_name = try factor.batchedCholeskyFamilyEntryName(allocator, instance);
4209 defer allocator.free(entry_name);
4210 var graph = try factor.BatchedCholeskyRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
4211 defer graph.deinit();
4212 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4213 .authored_kernel_diagnostic_id = "conformance/batched-cholesky-family",
4214 });
4215 }
4216
4217 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4218 _ = seeded;
4219 const out = std.mem.bytesAsSlice(f32, expected);
4220 for (0..batch) |system| {
4221 var a_tile: [n * n]f32 = undefined;
4222 for (0..n) |row| {
4223 for (0..n) |col| {
4224 a_tile[row * n + col] = tileEntry(system, row, col);
4225 }
4226 }
4227 var l_tile: [n * n]f32 = @as([(n * n)]f32, @splat(0));
4228 for (0..n) |j| {
4229 var diagonal = a_tile[j * n + j];
4230 for (0..j) |c| {
4231 diagonal -= l_tile[j * n + c] * l_tile[j * n + c];
4232 }
4233 l_tile[j * n + j] = @sqrt(diagonal);
4234 for (j + 1..n) |i| {
4235 var sum = a_tile[i * n + j];
4236 for (0..j) |c| {
4237 sum -= l_tile[i * n + c] * l_tile[j * n + c];
4238 }
4239 l_tile[i * n + j] = sum / l_tile[j * n + j];
4240 }
4241 }
4242 for (0..n * n) |slot| {
4243 out[system * n * n + slot] = l_tile[slot];
4244 }
4245 }
4246 }
4247 };
4248
4249 const BatchedCholeskyInterleavedFamilyCase = struct {
4250 const factor = accy.kernel.library.factor;
4251 const n = 3;
4252 const batch = 48;
4253 const instance = factor.BatchedCholesky{ .batch = batch, .n = n, .threads = 32, .layout = .interleaved };
4254
4255 pub const name = "batched_cholesky_family_f32_48x3_interleaved";
4256 pub const expectation: harness.Expectation = .verified;
4257 pub const tolerance: f32 = 0.00001;
4258 pub const buffers = [_]harness.FamilyBuffer{
4259 .{ .tensor = harness.vec(.f32, batch * n * n), .access = .inout },
4260 .{ .tensor = harness.vec(.f32, batch * n * n) },
4261 };
4262 pub const observed: usize = 0;
4263 pub const geometry = choir_abi.LaunchGeometry{
4264 .grid = .{ @intCast(factor.batchedCholeskyBlockCount(batch, instance.threads)), 1, 1 },
4265 .threadgroup = .{ instance.threads, 1, 1 },
4266 };
4267
4268 fn mixedEntry(system: usize, row: usize, col: usize) f32 {
4269 const mixed = system * 131 + row * 17 + col * 7;
4270 return @as(f32, @floatFromInt(mixed % 1000)) / 1000.0;
4271 }
4272
4273 fn tileEntry(system: usize, row: usize, col: usize) f32 {
4274 var sum: f32 = 0;
4275 for (0..n) |c| {
4276 sum += mixedEntry(system, row, c) * mixedEntry(system, col, c);
4277 }
4278 if (row == col) sum += @floatFromInt(n);
4279 return sum;
4280 }
4281
4282 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4283 switch (index) {
4284 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
4285 value.* = -1;
4286 },
4287 1 => {
4288 const values = std.mem.bytesAsSlice(f32, buffer);
4289 for (0..batch) |system| {
4290 for (0..n) |row| {
4291 for (0..n) |col| {
4292 values[(row * n + col) * batch + system] = tileEntry(system, row, col);
4293 }
4294 }
4295 }
4296 },
4297 else => unreachable,
4298 }
4299 }
4300
4301 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
4302 return factor.batchedCholeskyRuntimeArguments(instance);
4303 }
4304
4305 pub fn buildArtifact(
4306 allocator: std.mem.Allocator,
4307 handle: harness.BackendHandle,
4308 ) !gpu.KernelArtifact {
4309 const entry_name = try factor.batchedCholeskyFamilyEntryName(allocator, instance);
4310 defer allocator.free(entry_name);
4311 var graph = try factor.BatchedCholeskyRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
4312 defer graph.deinit();
4313 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4314 .authored_kernel_diagnostic_id = "conformance/batched-cholesky-interleaved-family",
4315 });
4316 }
4317
4318 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4319 _ = seeded;
4320 const out = std.mem.bytesAsSlice(f32, expected);
4321 for (0..batch) |system| {
4322 var a_tile: [n * n]f32 = undefined;
4323 for (0..n) |row| {
4324 for (0..n) |col| {
4325 a_tile[row * n + col] = tileEntry(system, row, col);
4326 }
4327 }
4328 var l_tile: [n * n]f32 = @as([(n * n)]f32, @splat(0));
4329 for (0..n) |j| {
4330 var diagonal = a_tile[j * n + j];
4331 for (0..j) |c| {
4332 diagonal -= l_tile[j * n + c] * l_tile[j * n + c];
4333 }
4334 l_tile[j * n + j] = @sqrt(diagonal);
4335 for (j + 1..n) |i| {
4336 var sum = a_tile[i * n + j];
4337 for (0..j) |c| {
4338 sum -= l_tile[i * n + c] * l_tile[j * n + c];
4339 }
4340 l_tile[i * n + j] = sum / l_tile[j * n + j];
4341 }
4342 }
4343 for (0..n * n) |slot| {
4344 out[slot * batch + system] = l_tile[slot];
4345 }
4346 }
4347 }
4348 };
4349
4350 fn BatchedCholeskySolveFamilyCase(
4351 comptime case_name: []const u8,
4352 comptime case_layout: accy.kernel.library.factor.TileLayout,
4353 ) type {
4354 return struct {
4355 const factor = accy.kernel.library.factor;
4356 const n = 3;
4357 const batch = 48;
4358 const instance = factor.BatchedCholeskySolve{ .batch = batch, .n = n, .threads = 32, .layout = case_layout };
4359
4360 pub const name = case_name;
4361 pub const expectation: harness.Expectation = .verified;
4362 pub const tolerance: f32 = 0.00001;
4363 pub const buffers = [_]harness.FamilyBuffer{
4364 .{ .tensor = harness.vec(.f32, batch * n), .access = .inout },
4365 .{ .tensor = harness.vec(.f32, batch * n * n) },
4366 .{ .tensor = harness.vec(.f32, batch * n) },
4367 };
4368 pub const observed: usize = 0;
4369 pub const geometry = choir_abi.LaunchGeometry{
4370 .grid = .{ @intCast(factor.batchedCholeskyBlockCount(batch, instance.threads)), 1, 1 },
4371 .threadgroup = .{ instance.threads, 1, 1 },
4372 };
4373
4374 fn mixedEntry(system: usize, row: usize, col: usize) f32 {
4375 const mixed = system * 131 + row * 17 + col * 7;
4376 return @as(f32, @floatFromInt(mixed % 1000)) / 1000.0;
4377 }
4378
4379 fn tileEntry(system: usize, row: usize, col: usize) f32 {
4380 var sum: f32 = 0;
4381 for (0..n) |c| {
4382 sum += mixedEntry(system, row, c) * mixedEntry(system, col, c);
4383 }
4384 if (row == col) sum += @floatFromInt(n);
4385 return sum;
4386 }
4387
4388 fn rhsEntry(system: usize, row: usize) f32 {
4389 const mixed = system * 37 + row * 11;
4390 return @as(f32, @floatFromInt(mixed % 1000)) / 500.0 - 1.0;
4391 }
4392
4393 fn tileIndex(system: usize, slot: usize) usize {
4394 return switch (case_layout) {
4395 .row_major => system * n * n + slot,
4396 .interleaved => slot * batch + system,
4397 };
4398 }
4399
4400 fn vectorIndex(system: usize, slot: usize) usize {
4401 return switch (case_layout) {
4402 .row_major => system * n + slot,
4403 .interleaved => slot * batch + system,
4404 };
4405 }
4406
4407 fn lowerTile(system: usize, l_tile: *[n * n]f32) void {
4408 var a_tile: [n * n]f32 = undefined;
4409 for (0..n) |row| {
4410 for (0..n) |col| {
4411 a_tile[row * n + col] = tileEntry(system, row, col);
4412 }
4413 }
4414 for (l_tile) |*value| value.* = 0;
4415 for (0..n) |j| {
4416 var diagonal = a_tile[j * n + j];
4417 for (0..j) |c| {
4418 diagonal -= l_tile[j * n + c] * l_tile[j * n + c];
4419 }
4420 l_tile[j * n + j] = @sqrt(diagonal);
4421 for (j + 1..n) |i| {
4422 var sum = a_tile[i * n + j];
4423 for (0..j) |c| {
4424 sum -= l_tile[i * n + c] * l_tile[j * n + c];
4425 }
4426 l_tile[i * n + j] = sum / l_tile[j * n + j];
4427 }
4428 }
4429 }
4430
4431 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4432 switch (index) {
4433 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
4434 value.* = -1;
4435 },
4436 1 => {
4437 const values = std.mem.bytesAsSlice(f32, buffer);
4438 for (0..batch) |system| {
4439 var l_tile: [n * n]f32 = undefined;
4440 lowerTile(system, &l_tile);
4441 for (0..n * n) |slot| {
4442 values[tileIndex(system, slot)] = l_tile[slot];
4443 }
4444 }
4445 },
4446 2 => {
4447 const values = std.mem.bytesAsSlice(f32, buffer);
4448 for (0..batch) |system| {
4449 for (0..n) |row| {
4450 values[vectorIndex(system, row)] = rhsEntry(system, row);
4451 }
4452 }
4453 },
4454 else => unreachable,
4455 }
4456 }
4457
4458 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
4459 return factor.batchedCholeskySolveRuntimeArguments(instance);
4460 }
4461
4462 pub fn buildArtifact(
4463 allocator: std.mem.Allocator,
4464 handle: harness.BackendHandle,
4465 ) !gpu.KernelArtifact {
4466 const entry_name = try factor.batchedCholeskySolveFamilyEntryName(allocator, instance);
4467 defer allocator.free(entry_name);
4468 var graph = try factor.BatchedCholeskySolveRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
4469 defer graph.deinit();
4470 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4471 .authored_kernel_diagnostic_id = switch (case_layout) {
4472 .row_major => "conformance/batched-cholesky-solve-family",
4473 .interleaved => "conformance/batched-cholesky-solve-family-interleaved",
4474 },
4475 });
4476 }
4477
4478 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4479 _ = seeded;
4480 const out = std.mem.bytesAsSlice(f32, expected);
4481 for (0..batch) |system| {
4482 var l_tile: [n * n]f32 = undefined;
4483 lowerTile(system, &l_tile);
4484 var solution: [n]f32 = undefined;
4485 for (0..n) |i| {
4486 var sum = rhsEntry(system, i);
4487 for (0..i) |j| {
4488 sum -= l_tile[i * n + j] * solution[j];
4489 }
4490 solution[i] = sum / l_tile[i * n + i];
4491 }
4492 var row: usize = n;
4493 while (row > 0) {
4494 row -= 1;
4495 var sum = solution[row];
4496 for (row + 1..n) |j| {
4497 sum -= l_tile[j * n + row] * solution[j];
4498 }
4499 solution[row] = sum / l_tile[row * n + row];
4500 }
4501 for (0..n) |slot| {
4502 out[vectorIndex(system, slot)] = solution[slot];
4503 }
4504 }
4505 }
4506 };
4507 }
4508
4509 fn BatchedInverseFamilyCase(
4510 comptime case_name: []const u8,
4511 comptime case_layout: accy.kernel.library.factor.TileLayout,
4512 ) type {
4513 return struct {
4514 const factor = accy.kernel.library.factor;
4515 const n = 3;
4516 const batch = 48;
4517 const instance = factor.BatchedInverse{ .batch = batch, .n = n, .threads = 32, .layout = case_layout };
4518
4519 pub const name = case_name;
4520 pub const expectation: harness.Expectation = .verified;
4521 pub const tolerance: f32 = 0.001;
4522 pub const buffers = [_]harness.FamilyBuffer{
4523 .{ .tensor = harness.vec(.f32, batch * n * n), .access = .inout },
4524 .{ .tensor = harness.vec(.f32, batch * n * n) },
4525 };
4526 pub const observed: usize = 0;
4527 pub const geometry = choir_abi.LaunchGeometry{
4528 .grid = .{ @intCast(factor.batchedCholeskyBlockCount(batch, instance.threads)), 1, 1 },
4529 .threadgroup = .{ instance.threads, 1, 1 },
4530 };
4531
4532 fn mixedEntry(system: usize, row: usize, col: usize) f32 {
4533 const mixed = system * 131 + row * 17 + col * 7;
4534 return @as(f32, @floatFromInt(mixed % 1000)) / 1000.0;
4535 }
4536
4537 fn tileEntry(system: usize, row: usize, col: usize) f32 {
4538 var sum: f32 = 0;
4539 for (0..n) |c| {
4540 sum += mixedEntry(system, row, c) * mixedEntry(system, col, c);
4541 }
4542 if (row == col) sum += @floatFromInt(n);
4543 return sum;
4544 }
4545
4546 fn tileIndex(system: usize, slot: usize) usize {
4547 return switch (case_layout) {
4548 .row_major => system * n * n + slot,
4549 .interleaved => slot * batch + system,
4550 };
4551 }
4552
4553 fn invertTile(system: usize, inv_tile: *[n * n]f32) void {
4554 var a_tile: [n * n]f32 = undefined;
4555 for (0..n) |row| {
4556 for (0..n) |col| {
4557 a_tile[row * n + col] = tileEntry(system, row, col);
4558 inv_tile[row * n + col] = if (row == col) 1 else 0;
4559 }
4560 }
4561 for (0..n) |pivot_index| {
4562 const pivot = a_tile[pivot_index * n + pivot_index];
4563 for (0..n) |col| {
4564 a_tile[pivot_index * n + col] /= pivot;
4565 inv_tile[pivot_index * n + col] /= pivot;
4566 }
4567 for (0..n) |row| {
4568 if (row == pivot_index) continue;
4569 const scale = a_tile[row * n + pivot_index];
4570 for (0..n) |col| {
4571 a_tile[row * n + col] -= scale * a_tile[pivot_index * n + col];
4572 inv_tile[row * n + col] -= scale * inv_tile[pivot_index * n + col];
4573 }
4574 }
4575 }
4576 }
4577
4578 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4579 switch (index) {
4580 0 => for (std.mem.bytesAsSlice(f32, buffer)) |*value| {
4581 value.* = -1;
4582 },
4583 1 => {
4584 const values = std.mem.bytesAsSlice(f32, buffer);
4585 for (0..batch) |system| {
4586 for (0..n) |row| {
4587 for (0..n) |col| {
4588 values[tileIndex(system, row * n + col)] = tileEntry(system, row, col);
4589 }
4590 }
4591 }
4592 },
4593 else => unreachable,
4594 }
4595 }
4596
4597 pub fn runtimeArguments() ![1]choir_abi.ScalarArgument {
4598 return factor.batchedInverseRuntimeArguments(instance);
4599 }
4600
4601 pub fn buildArtifact(
4602 allocator: std.mem.Allocator,
4603 handle: harness.BackendHandle,
4604 ) !gpu.KernelArtifact {
4605 const entry_name = try factor.batchedInverseFamilyEntryName(allocator, instance);
4606 defer allocator.free(entry_name);
4607 var graph = try factor.BatchedInverseRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
4608 defer graph.deinit();
4609 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4610 .authored_kernel_diagnostic_id = switch (case_layout) {
4611 .row_major => "conformance/batched-inverse-family",
4612 .interleaved => "conformance/batched-inverse-family-interleaved",
4613 },
4614 });
4615 }
4616
4617 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4618 _ = seeded;
4619 const out = std.mem.bytesAsSlice(f32, expected);
4620 for (0..batch) |system| {
4621 var inv_tile: [n * n]f32 = undefined;
4622 invertTile(system, &inv_tile);
4623 for (0..n * n) |slot| {
4624 out[tileIndex(system, slot)] = inv_tile[slot];
4625 }
4626 }
4627 }
4628 };
4629 }
4630
4631 const GridNeighborCountFamilyCase = struct {
4632 const spatial = accy.kernel.library.spatial;
4633 const count = 96;
4634 const cells_total = 16;
4635 const radius: f32 = 0.75;
4636 const instance = spatial.GridNeighborCount{ .count = count, .threads = 32 };
4637 const grid = spatial.GridGeometry{
4638 .origin_x = 0.0,
4639 .origin_y = 0.0,
4640 .inv_cell_size = 1.0,
4641 .dims_x = 4,
4642 .dims_y = 4,
4643 };
4644
4645 pub const name = "grid_neighbor_count_family_f32_96points";
4646 pub const expectation: harness.Expectation = .verified;
4647 pub const tolerance: f32 = 0;
4648 pub const buffers = [_]harness.FamilyBuffer{
4649 .{ .tensor = harness.vec(.i32, count), .access = .inout },
4650 .{ .tensor = harness.vec(.f32, count) },
4651 .{ .tensor = harness.vec(.f32, count) },
4652 .{ .tensor = harness.vec(.i32, count) },
4653 .{ .tensor = harness.vec(.f32, cells_total) },
4654 };
4655 pub const observed: usize = 0;
4656 pub const geometry = choir_abi.LaunchGeometry{
4657 .grid = .{ @intCast(spatial.gridCellsBlockCount(count, instance.threads)), 1, 1 },
4658 .threadgroup = .{ instance.threads, 1, 1 },
4659 };
4660
4661 fn coordAt(comptime axis: usize, point: usize) f32 {
4662 const mixed = point * 53 + axis * 17;
4663 return @as(f32, @floatFromInt(mixed % 1000)) / 250.0 - 0.5;
4664 }
4665
4666 fn cellIdAt(point: usize) i32 {
4667 const fx = (coordAt(0, point) - grid.origin_x) * grid.inv_cell_size;
4668 const fy = (coordAt(1, point) - grid.origin_y) * grid.inv_cell_size;
4669 const cx = std.math.clamp(@as(i32, @intFromFloat(fx)), 0, @as(i32, @intCast(grid.dims_x - 1)));
4670 const cy = std.math.clamp(@as(i32, @intFromFloat(fy)), 0, @as(i32, @intCast(grid.dims_y - 1)));
4671 return cy * @as(i32, @intCast(grid.dims_x)) + cx;
4672 }
4673
4674 fn sortedOrder() [count]i32 {
4675 var order: [count]i32 = undefined;
4676 for (&order, 0..) |*value, index| value.* = @intCast(index);
4677 var sort_index: usize = 1;
4678 while (sort_index < count) : (sort_index += 1) {
4679 const key = order[sort_index];
4680 const key_cell = cellIdAt(@intCast(key));
4681 var slot = sort_index;
4682 while (slot > 0 and cellIdAt(@intCast(order[slot - 1])) > key_cell) : (slot -= 1) {
4683 order[slot] = order[slot - 1];
4684 }
4685 order[slot] = key;
4686 }
4687 return order;
4688 }
4689
4690 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4691 switch (index) {
4692 0 => for (std.mem.bytesAsSlice(i32, buffer)) |*value| {
4693 value.* = -1;
4694 },
4695 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
4696 value.* = coordAt(0, point);
4697 },
4698 2 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, point| {
4699 value.* = coordAt(1, point);
4700 },
4701 3 => {
4702 const order = sortedOrder();
4703 for (std.mem.bytesAsSlice(i32, buffer), order) |*value, sorted| {
4704 value.* = sorted;
4705 }
4706 },
4707 4 => {
4708 var counts = @as([cells_total]u32, @splat(0));
4709 for (0..count) |point| counts[@intCast(cellIdAt(point))] += 1;
4710 var prefix: u32 = 0;
4711 for (std.mem.bytesAsSlice(f32, buffer), counts) |*value, cell_count| {
4712 value.* = @floatFromInt(prefix);
4713 prefix += cell_count;
4714 }
4715 },
4716 else => unreachable,
4717 }
4718 }
4719
4720 pub fn runtimeArguments() ![8]choir_abi.ScalarArgument {
4721 return spatial.gridNeighborCountRuntimeArguments(instance, grid, 1, radius);
4722 }
4723
4724 pub fn buildArtifact(
4725 allocator: std.mem.Allocator,
4726 handle: harness.BackendHandle,
4727 ) !gpu.KernelArtifact {
4728 const entry_name = try spatial.gridNeighborCountFamilyEntryName(allocator, instance);
4729 defer allocator.free(entry_name);
4730 var graph = try spatial.GridNeighborCountRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
4731 defer graph.deinit();
4732 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4733 .authored_kernel_diagnostic_id = "conformance/grid-neighbor-count-family",
4734 });
4735 }
4736
4737 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4738 _ = seeded;
4739 const out = std.mem.bytesAsSlice(i32, expected);
4740 for (out, 0..) |*value, a| {
4741 var total: i32 = 0;
4742 for (0..count) |b| {
4743 if (a == b) continue;
4744 const dx = coordAt(0, a) - coordAt(0, b);
4745 const dy = coordAt(1, a) - coordAt(1, b);
4746 if (dx * dx + dy * dy <= radius * radius) total += 1;
4747 }
4748 value.* = total;
4749 }
4750 }
4751 };
4752
4753 const PhiloxFillFamilyI32 = struct {
4754 const random = accy.kernel.library.random;
4755 const count = 64;
4756 const instance = random.Philox{
4757 .count = count,
4758 .dtype = .i32,
4759 .threads = 32,
4760 .seed = 0x00c0ffee_deadbeef,
4761 };
4762
4763 pub const name = "philox_fill_family_i32_64";
4764 pub const expectation: harness.Expectation = .verified;
4765 pub const tolerance: f32 = 0;
4766 pub const buffers = [_]harness.FamilyBuffer{
4767 .{ .tensor = harness.vec(.i32, count), .access = .inout },
4768 };
4769 pub const observed: usize = 0;
4770 pub const geometry = choir_abi.LaunchGeometry{
4771 .grid = .{ 1, 1, 1 },
4772 .threadgroup = .{ instance.threads, 1, 1 },
4773 };
4774
4775 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4776 switch (index) {
4777 0 => @memset(buffer, 0),
4778 else => unreachable,
4779 }
4780 }
4781
4782 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
4783 return random.philoxRuntimeArguments(instance);
4784 }
4785
4786 pub fn buildArtifact(
4787 allocator: std.mem.Allocator,
4788 handle: harness.BackendHandle,
4789 ) !gpu.KernelArtifact {
4790 const entry_name = try random.philoxFamilyEntryName(allocator, instance);
4791 defer allocator.free(entry_name);
4792 var graph = try random.PhiloxRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
4793 defer graph.deinit();
4794 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4795 .authored_kernel_diagnostic_id = "conformance/philox-fill-family",
4796 });
4797 }
4798
4799 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4800 _ = seeded;
4801 const out = std.mem.bytesAsSlice(i32, expected);
4802 var generator: u32 = 0;
4803 while (generator * 4 < count) : (generator += 1) {
4804 const words = random.philoxBlock(
4805 random.philox_default_rounds,
4806 .{ generator, 0, 0, 0 },
4807 .{ instance.seedLo(), instance.seedHi() },
4808 );
4809 for (words, 0..) |word, lane| {
4810 const element = generator * 4 + lane;
4811 if (element >= count) continue;
4812 out[element] = @bitCast(word);
4813 }
4814 }
4815 }
4816 };
4817
4818 const ThreefryFillFamilyI32 = struct {
4819 const random = accy.kernel.library.random;
4820 const count = 65;
4821 const observed_count = 66;
4822 const sentinel: i32 = 0x13579bdf;
4823 const instance = random.Threefry{
4824 .count = count,
4825 .dtype = .i32,
4826 .threads = 32,
4827 .seed = 0x01234567_89abcdef,
4828 };
4829
4830 pub const name = "threefry_fill_family_i32_65";
4831 pub const expectation: harness.Expectation = .verified;
4832 pub const tolerance: f32 = 0;
4833 pub const buffers = [_]harness.FamilyBuffer{
4834 .{ .tensor = harness.vec(.i32, observed_count), .access = .inout },
4835 };
4836 pub const observed: usize = 0;
4837 pub const geometry = choir_abi.LaunchGeometry{
4838 .grid = .{ blockCount(), 1, 1 },
4839 .threadgroup = .{ instance.threads, 1, 1 },
4840 };
4841
4842 fn blockCount() u32 {
4843 const generators = instance.generators();
4844 return @intCast((generators + instance.threads - 1) / instance.threads);
4845 }
4846
4847 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4848 switch (index) {
4849 0 => for (std.mem.bytesAsSlice(i32, buffer)) |*value| {
4850 value.* = sentinel;
4851 },
4852 else => unreachable,
4853 }
4854 }
4855
4856 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
4857 return random.threefryRuntimeArguments(instance);
4858 }
4859
4860 pub fn buildArtifact(
4861 allocator: std.mem.Allocator,
4862 handle: harness.BackendHandle,
4863 ) !gpu.KernelArtifact {
4864 const entry_name = try random.threefryFamilyEntryName(allocator, instance);
4865 defer allocator.free(entry_name);
4866 var graph = try random.ThreefryRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
4867 defer graph.deinit();
4868 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4869 .authored_kernel_diagnostic_id = "conformance/threefry-fill-family",
4870 });
4871 }
4872
4873 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4874 const seed = std.mem.bytesAsSlice(i32, seeded[0]);
4875 const out = std.mem.bytesAsSlice(i32, expected);
4876 for (out, seed) |*value, initial| value.* = initial;
4877 var generator: u32 = 0;
4878 while (generator * 2 < count) : (generator += 1) {
4879 const words = random.threefryBlock(
4880 random.threefry_default_rounds,
4881 .{ generator, 0 },
4882 .{ instance.seedLo(), instance.seedHi() },
4883 );
4884 for (words, 0..) |word, lane| {
4885 const element = generator * 2 + lane;
4886 if (element >= count) continue;
4887 out[element] = @bitCast(word);
4888 }
4889 }
4890 }
4891 };
4892
4893 const SquaresFillFamilyI32 = struct {
4894 const random = accy.kernel.library.random;
4895 const count = 63;
4896 const observed_count = 64;
4897 const sentinel: i32 = 0x13579bdf;
4898 const instance = random.Squares{
4899 .count = count,
4900 .dtype = .i32,
4901 .threads = 32,
4902 .key = 0x11223344_55667788,
4903 };
4904
4905 pub const name = "squares_fill_family_i32_63";
4906 pub const expectation: harness.Expectation = .verified;
4907 pub const tolerance: f32 = 0;
4908 pub const buffers = [_]harness.FamilyBuffer{
4909 .{ .tensor = harness.vec(.i32, observed_count), .access = .inout },
4910 };
4911 pub const observed: usize = 0;
4912 pub const geometry = choir_abi.LaunchGeometry{
4913 .grid = .{ blockCount(), 1, 1 },
4914 .threadgroup = .{ instance.threads, 1, 1 },
4915 };
4916
4917 fn blockCount() u32 {
4918 const generators = instance.generators();
4919 return @intCast((generators + instance.threads - 1) / instance.threads);
4920 }
4921
4922 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4923 switch (index) {
4924 0 => for (std.mem.bytesAsSlice(i32, buffer)) |*value| {
4925 value.* = sentinel;
4926 },
4927 else => unreachable,
4928 }
4929 }
4930
4931 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
4932 return random.squaresRuntimeArguments(instance);
4933 }
4934
4935 pub fn buildArtifact(
4936 allocator: std.mem.Allocator,
4937 handle: harness.BackendHandle,
4938 ) !gpu.KernelArtifact {
4939 const entry_name = try random.squaresFamilyEntryName(allocator, instance);
4940 defer allocator.free(entry_name);
4941 var graph = try random.SquaresRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
4942 defer graph.deinit();
4943 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
4944 .authored_kernel_diagnostic_id = "conformance/squares-fill-family",
4945 });
4946 }
4947
4948 pub fn reference(seeded: []const []const u8, expected: []u8) void {
4949 const seed = std.mem.bytesAsSlice(i32, seeded[0]);
4950 const out = std.mem.bytesAsSlice(i32, expected);
4951 for (out, seed) |*value, initial| value.* = initial;
4952 for (0..count) |element| {
4953 out[element] = @bitCast(random.squaresBlock(@intCast(element), instance.key));
4954 }
4955 }
4956 };
4957
4958 const WhileTriangularI32 = struct {
4959 const kernel = accy.kernel;
4960 const count = 64;
4961 const threads = 32;
4962 const sentinel: i32 = -1;
4963
4964 pub const name = "scf_while_triangular_i32_64";
4965 pub const expectation: harness.Expectation = .verified;
4966 pub const tolerance: f32 = 0;
4967 pub const buffers = [_]harness.FamilyBuffer{
4968 .{ .tensor = harness.vec(.i32, count), .access = .inout },
4969 };
4970 pub const observed: usize = 0;
4971 pub const geometry = choir_abi.LaunchGeometry{
4972 .grid = .{ count / threads, 1, 1 },
4973 .threadgroup = .{ threads, 1, 1 },
4974 };
4975
4976 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
4977 switch (index) {
4978 0 => for (std.mem.bytesAsSlice(i32, buffer)) |*value| {
4979 value.* = sentinel;
4980 },
4981 else => unreachable,
4982 }
4983 }
4984
4985 pub fn runtimeArguments() ![0]choir_abi.ScalarArgument {
4986 return .{};
4987 }
4988
4989 pub fn buildArtifact(
4990 allocator: std.mem.Allocator,
4991 handle: harness.BackendHandle,
4992 ) !gpu.KernelArtifact {
4993 var builder = try kernel.Builder.init(allocator, kernel.Builder.Limits.standard, "accy_conformance_while_triangular", &.{
4994 kernel.dynamicBuffer(.i32),
4995 });
4996 errdefer builder.deinit();
4997
4998 const dst = builder.argument(0);
4999 const index = try builder.globalId(.x);
5000 const start = try builder.cast(index, .i32);
5001 const zero = try builder.constantInt(.i32, 0);
5002 const one = try builder.constantInt(.i32, 1);
5003
5004 var scope = try builder.whileScope(&.{ start, zero }, &.{ start.valueType(), zero.valueType() });
5005 errdefer scope.abort();
5006 const remaining = scope.beforeArg(0).?;
5007 const total = scope.beforeArg(1).?;
5008 const proceed = try builder.compare(.gt, remaining, zero);
5009 try scope.condition(proceed, &.{ remaining, total });
5010 const after_remaining = scope.afterArg(0).?;
5011 const after_total = scope.afterArg(1).?;
5012 const next_total = try builder.add(after_total, after_remaining);
5013 const next_remaining = try builder.sub(after_remaining, one);
5014 try scope.leave(&.{ next_remaining, next_total });
5015
5016 try builder.store(scope.result(1).?, dst, index);
5017 try builder.return_();
5018
5019 var graph = try builder.finish();
5020 defer graph.deinit();
5021 return kernel.createKernelArtifact(allocator, handle, &graph, .{
5022 .authored_kernel_diagnostic_id = "conformance/scf-while-triangular",
5023 });
5024 }
5025
5026 pub fn reference(seeded: []const []const u8, expected: []u8) void {
5027 _ = seeded;
5028 const out = std.mem.bytesAsSlice(i32, expected);
5029 for (out, 0..) |*value, lane| {
5030 var total: i64 = 0;
5031 var remaining: i64 = @intCast(lane);
5032 while (remaining > 0) : (remaining -= 1) total += remaining;
5033 value.* = @intCast(total);
5034 }
5035 }
5036 };
5037
5038 const FeistelPermutationFamilyI32 = struct {
5039 const random = accy.kernel.library.random;
5040 const count = 64;
5041 const observed_count = 65;
5042 const sentinel: i32 = -1;
5043 const instance = random.Feistel{
5044 .count = count,
5045 .rounds = 6,
5046 .dtype = .i32,
5047 .threads = 32,
5048 .seed = 0x10203040_55667788,
5049 };
5050
5051 pub const name = "feistel_permutation_family_i32_64";
5052 pub const expectation: harness.Expectation = .verified;
5053 pub const tolerance: f32 = 0;
5054 pub const buffers = [_]harness.FamilyBuffer{
5055 .{ .tensor = harness.vec(.i32, observed_count), .access = .inout },
5056 };
5057 pub const observed: usize = 0;
5058 pub const geometry = choir_abi.LaunchGeometry{
5059 .grid = .{ blockCount(), 1, 1 },
5060 .threadgroup = .{ instance.threads, 1, 1 },
5061 };
5062
5063 fn blockCount() u32 {
5064 return @intCast((instance.count + instance.threads - 1) / instance.threads);
5065 }
5066
5067 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
5068 switch (index) {
5069 0 => for (std.mem.bytesAsSlice(i32, buffer)) |*value| {
5070 value.* = sentinel;
5071 },
5072 else => unreachable,
5073 }
5074 }
5075
5076 pub fn runtimeArguments() ![3]choir_abi.ScalarArgument {
5077 return random.feistelRuntimeArguments(instance);
5078 }
5079
5080 pub fn buildArtifact(
5081 allocator: std.mem.Allocator,
5082 handle: harness.BackendHandle,
5083 ) !gpu.KernelArtifact {
5084 const entry_name = try random.feistelFamilyEntryName(allocator, instance);
5085 defer allocator.free(entry_name);
5086 var graph = try random.FeistelRuntimeFamilyI32.buildNamed(allocator, kernel_limits, entry_name, instance);
5087 defer graph.deinit();
5088 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
5089 .authored_kernel_diagnostic_id = "conformance/feistel-permutation-family",
5090 });
5091 }
5092
5093 pub fn reference(seeded: []const []const u8, expected: []u8) void {
5094 const seed = std.mem.bytesAsSlice(i32, seeded[0]);
5095 const out = std.mem.bytesAsSlice(i32, expected);
5096 for (out, seed) |*value, initial| value.* = initial;
5097 const bits = instance.domainBits().?;
5098 for (0..count) |element| {
5099 out[element] = @intCast(random.feistelPermuteReference(@intCast(element), instance.rounds, instance.seed, bits));
5100 }
5101 }
5102 };
5103
5104 fn HistogramFamilyCase(
5105 comptime case_name: []const u8,
5106 comptime case_variant: accy.kernel.library.histogram.HistogramVariant,
5107 ) type {
5108 return struct {
5109 const histogram = accy.kernel.library.histogram;
5110 const bin_count = 16;
5111 const element_count_local = 96;
5112 const instance = histogram.Histogram{
5113 .bins = bin_count,
5114 .count = element_count_local,
5115 .lo = -1.0,
5116 .width = 0.25,
5117 .variant = case_variant,
5118 .threads = 32,
5119 };
5120
5121 pub const name = case_name;
5122 pub const expectation: harness.Expectation = .verified;
5123 pub const tolerance: f32 = 0;
5124 pub const buffers = [_]harness.FamilyBuffer{
5125 .{ .tensor = harness.vec(.i32, bin_count), .access = .inout },
5126 .{ .tensor = harness.vec(.f32, element_count_local) },
5127 };
5128 pub const observed: usize = 0;
5129 pub const geometry = choir_abi.LaunchGeometry{
5130 .grid = .{ element_count_local / 32, 1, 1 },
5131 .threadgroup = .{ 32, 1, 1 },
5132 };
5133
5134 fn valueAt(element: usize) f32 {
5135 return -1.6 + @as(f32, @floatFromInt((element * 13) % 41)) * 0.125;
5136 }
5137
5138 pub fn fillBuffer(comptime index: usize, buffer: []u8) void {
5139 switch (index) {
5140 0 => @memset(buffer, 0),
5141 1 => for (std.mem.bytesAsSlice(f32, buffer), 0..) |*value, element| {
5142 value.* = valueAt(element);
5143 },
5144 else => unreachable,
5145 }
5146 }
5147
5148 pub fn runtimeArguments() ![4]choir_abi.ScalarArgument {
5149 return histogram.histogramRuntimeArguments(instance);
5150 }
5151
5152 pub fn buildArtifact(
5153 allocator: std.mem.Allocator,
5154 handle: harness.BackendHandle,
5155 ) !gpu.KernelArtifact {
5156 const entry_name = try histogram.histogramFamilyEntryName(allocator, instance);
5157 defer allocator.free(entry_name);
5158 var graph = try histogram.HistogramRuntimeFamilyF32.buildNamed(allocator, kernel_limits, entry_name, instance);
5159 defer graph.deinit();
5160 return accy.kernel.createKernelArtifact(allocator, handle, &graph, .{
5161 .authored_kernel_diagnostic_id = "conformance/histogram-family",
5162 });
5163 }
5164
5165 pub fn reference(seeded: []const []const u8, expected: []u8) void {
5166 _ = seeded;
5167 const out = std.mem.bytesAsSlice(i32, expected);
5168 for (out) |*value| value.* = 0;
5169 for (0..element_count_local) |element| {
5170 const value = valueAt(element);
5171 if (histogram.histogramBinForValue(instance, value)) |bin| out[bin] += 1;
5172 }
5173 }
5174 };
5175 }
5176
5177 pub const all = productRowsFor(.f32) ++
5178 productRowsFor(.i8) ++
5179 productRowsFor(.i16) ++
5180 productRowsFor(.i32) ++
5181 productRowsFor(.u8) ++
5182 productRowsFor(.u16) ++
5183 productRowsFor(.u32) ++
5184 productRowsFor(.i64) ++
5185 productRowsFor(.u64) ++
5186 productRowsFor(.f16) ++
5187 productRowsFor(.bf16) ++
5188 productRowsFor(.f64) ++
5189 boolMovementRows() ++
5190 [_]type{
5191 TanhAddF32,
5192 BiasBroadcastF32,
5193 DotRectF32,
5194 EinsumMatmulF32,
5195 EinsumPermuteF32,
5196 ReduceSumAxisZeroF32,
5197 ReduceSumAllF32,
5198 AddRankThreeF32,
5199 ReduceSumRankThreeTrailingF32,
5200 ReduceSumRankThreeF32,
5201 ReduceMaxRankThreeF32,
5202 ReduceSumSplitAxesF32,
5203 GatherClampF32,
5204 ScatterCase("scatter_duplicate_f32_6x8_axis0", .f32, .verified, .{ 2, 0, 2, 5 }),
5205 ScatterCase("scatter_oob_f32_6x8_axis0", .f32, .verified, .{ -1, 9, 3, 0 }),
5206 MatrixProductFamilyCase,
5207 BatchedMatrixProductFamilyCase,
5208 MatrixVectorProductFamilyCase,
5209 OuterProductFamilyCase,
5210 GatherFamilyCase("gather_family_f32_2x5x6x3", .f32),
5211 GatherFamilyCase("gather_family_f16_2x5x6x3", .f16),
5212 ScatterFamilyCase("scatter_family_f32_2x5x6x3", .f32),
5213 ScatterFamilyCase("scatter_family_f16_2x5x6x3", .f16),
5214 ScatterAddFamilyCase("scatter_add_family_direct_i32_16x64", .i32, .direct, 0),
5215 ScatterAddFamilyCase("scatter_add_family_shared_i32_16x64", .i32, .shared_bins, 0),
5216 ScatterAddFamilyCase("scatter_add_family_direct_f32_16x64", .f32, .direct, 0.001),
5217 PrefixSumFamilyCase("prefix_sum_family_inclusive_f32_64", .f32, .inclusive),
5218 PrefixSumFamilyCase("prefix_sum_family_exclusive_f32_64", .f32, .exclusive),
5219 PrefixSumFamilyCase("prefix_sum_family_inclusive_u32_64", .u32, .inclusive),
5220 PrefixSumFamilyCase("prefix_sum_family_exclusive_u32_64", .u32, .exclusive),
5221 SegmentSumFamilyCase("segment_sum_family_thread_f32_8x80", .thread),
5222 SegmentSumFamilyCase("segment_sum_family_warp_f32_8x80", .warp),
5223 FilterFamilyCase("filter_family_nonzero_f32_70x32", .f32, .nonzero),
5224 FilterFamilyCase("filter_family_greater_i32_40x32", .i32, .greater_than),
5225 BitonicBlockFamilyCase,
5226 TopKBlockFamilyCase,
5227 TopKBlockPairsFamilyCase("top_k_block_pairs_family_keys_i32_8of45x64", 0),
5228 TopKBlockPairsFamilyCase("top_k_block_pairs_family_values_i32_8of45x64", 1),
5229 DeviceScanBlockScanFamilyCase("device_scan_block_scan_family_dst_f32_3x32", .f32, 0),
5230 DeviceScanBlockScanFamilyCase("device_scan_block_scan_family_sums_f32_3x32", .f32, 2),
5231 DeviceScanBlockScanFamilyCase("device_scan_block_scan_family_dst_f16_3x32", .f16, 0),
5232 DeviceScanBlockScanFamilyCase("device_scan_block_scan_family_sums_f16_3x32", .f16, 2),
5233 DeviceScanBlockScanFamilyCase("device_scan_block_scan_family_dst_u32_3x32", .u32, 0),
5234 DeviceScanBlockScanFamilyCase("device_scan_block_scan_family_sums_u32_3x32", .u32, 2),
5235 DeviceScanAddBaseFamilyCase("device_scan_add_base_family_f32_3x32", .f32),
5236 DeviceScanAddBaseFamilyCase("device_scan_add_base_family_f16_3x32", .f16),
5237 DeviceScanAddBaseFamilyCase("device_scan_add_base_family_u32_3x32", .u32),
5238 BatchedCholeskyFamilyCase,
5239 BatchedCholeskyInterleavedFamilyCase,
5240 BatchedCholeskySolveFamilyCase("batched_cholesky_solve_family_f32_48x3", .row_major),
5241 BatchedCholeskySolveFamilyCase("batched_cholesky_solve_family_f32_48x3_interleaved", .interleaved),
5242 BatchedInverseFamilyCase("batched_inverse_family_f32_48x3", .row_major),
5243 BatchedInverseFamilyCase("batched_inverse_family_f32_48x3_interleaved", .interleaved),
5244 GridCellsFamilyCase,
5245 ImageBlurPassFamilyCase,
5246 ImageResizeFamilyCase,
5247 SdfGridSampleGradient2DCase("sdf_grid_sample_gradient_family_f32_2d_96points_dist", 0),
5248 SdfGridSampleGradient2DCase("sdf_grid_sample_gradient_family_f32_2d_96points_gx", 1),
5249 SdfGridSampleGradient2DCase("sdf_grid_sample_gradient_family_f32_2d_96points_gy", 2),
5250 SdfGridSample3DFamilyCase,
5251 GridNeighborCountFamilyCase,
5252 SpmvCsrFamilyCase("spmv_csr_row_warp_family_f32_48rows", .row_warp, .f32),
5253 SpmvCsrFamilyCase("spmv_csr_row_thread_family_f32_48rows", .row_thread, .f32),
5254 SpmvCsrFamilyCase("spmv_csr_row_warp_family_f16_48rows", .row_warp, .f16),
5255 SpmvCsrFamilyCase("spmv_csr_row_thread_family_f16_48rows", .row_thread, .f16),
5256 SpmvCsrFamilyCase("spmv_csr_row_warp_family_f64_48rows", .row_warp, .f64),
5257 SpmvCsrFamilyCase("spmv_csr_row_thread_family_f64_48rows", .row_thread, .f64),
5258 SpmvCooFamilyCase("spmv_coo_element_thread_family_f32_48x96", .f32),
5259 SpmvCooFamilyCase("spmv_coo_row_thread_family_f16_48x96", .f16),
5260 SpmvCooFamilyCase("spmv_coo_row_thread_family_f64_48x96", .f64),
5261 SpmvEllFamilyCase("spmv_ell_row_thread_family_f32_48x6", .f32),
5262 SpmvEllFamilyCase("spmv_ell_row_thread_family_f16_48x6", .f16),
5263 SpmvEllFamilyCase("spmv_ell_row_thread_family_f64_48x6", .f64),
5264 SpmvSellFamilyCase("spmv_sell_row_thread_slice8_family_f32_48x8", .f32),
5265 SpmvSellFamilyCase("spmv_sell_row_thread_slice8_family_f16_48x8", .f16),
5266 SpmvSellFamilyCase("spmv_sell_row_thread_slice8_family_f64_48x8", .f64),
5267 SpmmCsrFamilyCase("spmm_csr_row_column_thread_family_f32_32x9", .f32),
5268 SpmmCsrFamilyCase("spmm_csr_row_column_thread_family_f16_32x9", .f16),
5269 SpmmCsrFamilyCase("spmm_csr_row_column_thread_family_f64_32x9", .f64),
5270 PhiloxFillFamilyI32,
5271 ThreefryFillFamilyI32,
5272 SquaresFillFamilyI32,
5273 FeistelPermutationFamilyI32,
5274 WhileTriangularI32,
5275 HistogramFamilyCase("histogram_family_direct_f32_16x96", .direct),
5276 HistogramFamilyCase("histogram_family_shared_f32_16x96", .shared_bins),
5277 };
5278
5279 test "conformance family rows declare backend feature and subgroup requirements" {
5280 const ScatterF32 = ScatterAddFamilyCase("require_scatter_add_f32", .f32, .direct, 0.001);
5281 try std.testing.expect(harness.requiredFeatures(ScatterF32).atomic_f32_add_device);
5282
5283 const Prefix = PrefixSumFamilyCase("require_prefix_sum", .f32, .inclusive);
5284 try std.testing.expect(harness.requiredSubgroup(Prefix).scan);
5285
5286 const SegmentThread = SegmentSumFamilyCase("require_segment_thread", .thread);
5287 try std.testing.expect(std.meta.eql(choir_abi.SubgroupRequirements{}, harness.requiredSubgroup(SegmentThread)));
5288 const SegmentWarp = SegmentSumFamilyCase("require_segment_warp", .warp);
5289 try std.testing.expect(harness.requiredSubgroup(SegmentWarp).arithmetic);
5290
5291 const Filter = FilterFamilyCase("require_filter", .f32, .nonzero);
5292 try std.testing.expect(harness.requiredSubgroup(Filter).scan);
5293
5294 const DeviceScan = DeviceScanBlockScanFamilyCase("require_device_scan", .f32, 0);
5295 try std.testing.expect(harness.requiredSubgroup(DeviceScan).scan);
5296
5297 const CsrThread = SpmvCsrFamilyCase("require_csr_thread", .row_thread, .f32);
5298 try std.testing.expect(std.meta.eql(choir_abi.SubgroupRequirements{}, harness.requiredSubgroup(CsrThread)));
5299 const CsrWarp = SpmvCsrFamilyCase("require_csr_warp", .row_warp, .f32);
5300 try std.testing.expect(harness.requiredSubgroup(CsrWarp).arithmetic);
5301
5302 const CooF32 = SpmvCooFamilyCase("require_coo_f32", .f32);
5303 try std.testing.expect(harness.requiredFeatures(CooF32).atomic_f32_add_device);
5304 const CooF16 = SpmvCooFamilyCase("require_coo_f16", .f16);
5305 try std.testing.expect(std.meta.eql(choir_abi.Features{}, harness.requiredFeatures(CooF16)));
5306 }
5307
5308 test "conformance case names are unique" {
5309 comptime {
5310 @setEvalBranchQuota(4_000_000);
5311 for (all, 0..) |Spec, index| {
5312 for (all, 0..) |Other, other_index| {
5313 if (index != other_index and std.mem.eql(u8, Spec.name, Other.name)) {
5314 @compileError("duplicate conformance case name: " ++ Spec.name);
5315 }
5316 }
5317 }
5318 }
5319 }
5320
5321 test "conformance case modules build and verify" {
5322 @setEvalBranchQuota(200_000);
5323 inline for (all) |Spec| {
5324 if (comptime @hasDecl(Spec, "buildArtifact")) continue;
5325 if (comptime Spec.expectation == .invalid) {
5326 if (harness.buildModule(Spec, std.testing.allocator)) |module| {
5327 module.deinit();
5328 return error.TestUnexpectedResult;
5329 } else |_| {}
5330 } else {
5331 const module = try harness.buildModule(Spec, std.testing.allocator);
5332 module.deinit();
5333 }
5334 }
5335 }
5336
5337 test "conformance float references stay finite on their fills" {
5338 @setEvalBranchQuota(200_000);
5339 inline for (all) |Spec| {
5340 if (comptime @hasDecl(Spec, "buildArtifact")) continue;
5341 if (comptime (Spec.output.dtype.isFloat() and Spec.expectation != .invalid)) {
5342 var views: [Spec.inputs.len][]const u8 = undefined;
5343 var buffers: [Spec.inputs.len][]align(16) u8 = undefined;
5344 inline for (Spec.inputs, 0..) |input, index| {
5345 const buffer = try std.testing.allocator.alignedAlloc(u8, .@"16", input.byteCount());
5346 if (@hasDecl(Spec, "fill")) {
5347 Spec.fill(index, buffer);
5348 } else {
5349 harness.defaultFill(input.dtype, index, buffer);
5350 }
5351 buffers[index] = buffer;
5352 views[index] = buffer;
5353 }
5354 defer {
5355 inline for (0..Spec.inputs.len) |index| std.testing.allocator.free(buffers[index]);
5356 }
5357
5358 const output_bytes = try std.testing.allocator.alignedAlloc(u8, .@"16", Spec.output.byteCount());
5359 defer std.testing.allocator.free(output_bytes);
5360 Spec.reference(&views, output_bytes);
5361
5362 const T = comptime Spec.output.dtype.ZigType();
5363 const values = std.mem.bytesAsSlice(T, output_bytes);
5364 for (values) |value| {
5365 try std.testing.expect(std.math.isFinite(harness.numericToF32(T, value)));
5366 }
5367 }
5368 }
5369 }
5370
5371 test "conformance references are deterministic" {
5372 var lhs: [element_count * 4]u8 align(16) = undefined;
5373 var rhs: [element_count * 4]u8 align(16) = undefined;
5374 harness.defaultFill(.f32, 0, lhs[0..]);
5375 harness.defaultFill(.f32, 1, rhs[0..]);
5376 var first: [element_count * 4]u8 align(16) = undefined;
5377 var second: [element_count * 4]u8 align(16) = undefined;
5378 TanhAddF32.reference(&.{ lhs[0..], rhs[0..] }, first[0..]);
5379 TanhAddF32.reference(&.{ lhs[0..], rhs[0..] }, second[0..]);
5380 try std.testing.expectEqualSlices(u8, first[0..], second[0..]);
5381 try std.testing.expect(harness.checksum(.f32, first[0..]) != 0);
5382 }