tiny.accy.kernel.library.scan
Defined in kernel.library.
API (71)
Actions
Public operations.
DeviceScanPipelineArtifacts.deinitDeviceScanPipelineArtifacts.entriesDeviceScanThreadCandidates.slicePrefixSumMode.fromOperationPrefixSumMode.operationPrefixSumThreadCandidates.slicecreateDeviceScanAddBaseFamilyArtifactcreateDeviceScanBlockScanFamilyArtifactcreateDeviceScanPipelineArtifactscreatePrefixSumFamilyArtifactdeviceScanAddBaseFamilyEntryNamedeviceScanAddBaseFamilyFingerprintdeviceScanAddBaseFamilyTargetdeviceScanAddBaseShapeFamilydeviceScanBlockCountdeviceScanBlockScanFamilyEntryNamedeviceScanBlockScanFamilyFingerprintdeviceScanBlockScanFamilyTargetdeviceScanBlockScanShapeFamilydeviceScanDTypeSupporteddeviceScanFamilySpecializationdeviceScanFamilyTargetdeviceScanInstanceFromSpecializationdeviceScanInstanceValiddeviceScanMaxExtentdeviceScanPipelinedeviceScanRuntimeArgumentsdeviceScanShapeProfileDimensionsdeviceScanStagesdeviceScanThreadCandidatesForExtentdeviceScanThreadsForExtentprefixSumDTypeSupportedprefixSumF32prefixSumFamilyEntryNameprefixSumFamilyFingerprintprefixSumFamilySpecializationprefixSumFamilyTargetprefixSumFamilyTuningKeyprefixSumInstanceEntryNameprefixSumInstanceFromSpecializationprefixSumInstanceTargetprefixSumInstanceValidprefixSumRuntimeArgumentsprefixSumShapeFamilyprefixSumShapeProfileDimensionsprefixSumThreadCandidatesForExtentprefixSumThreadsForExtentprefixSumTuningExtentsprefixSumTuningOperation
Types and contracts
Public types and contracts.
DeviceScanDeviceScanAddBaseRuntimeFamilyF16DeviceScanAddBaseRuntimeFamilyF32DeviceScanAddBaseRuntimeFamilyU32DeviceScanBlockScanRuntimeFamilyF16DeviceScanBlockScanRuntimeFamilyF32DeviceScanBlockScanRuntimeFamilyU32DeviceScanPipelineArtifactsDeviceScanStagesDeviceScanThreadCandidatesPrefixSumPrefixSum8F32PrefixSumModePrefixSumRuntimeFamilyF16PrefixSumRuntimeFamilyF32PrefixSumRuntimeFamilyU32PrefixSumThreadCandidates
Values and defaults
Public values and defaults.
device_scan_family_versiondevice_scan_max_blocksprefix_sum_family_versionprefix_sum_max_threadsprefix_sum_warp_size
Source
Source: lib/accy/src/kernel/library/root.zig:22
zig
pub const scan = @import("scan.zig");Source: lib/accy/src/kernel/library/scan.zig
zig
const std = @import("std");const gpu = @import("gpu");const choir_abi = @import("choir_abi");const artifact_product = @import("../../artifact/model/root.zig");const shape = @import("../../choir/shape/root.zig");const entry = @import("entry.zig");const extent_mod = @import("extent.zig");const kernel = @import("../root.zig");const tuning = @import("tuning.zig");const DType = choir_abi.DType;const indexExtent = extent_mod.indexExtent;const runtimeExtentArgument = extent_mod.runtimeExtentArgument;pub const PrefixSumMode = enum { inclusive, exclusive, pub fn operation(self: PrefixSumMode) entry.ScanOperator { return switch (self) { .inclusive => .prefix_sum, .exclusive => .prefix_sum_exclusive, }; } pub fn fromOperation(operator: entry.ScanOperator) PrefixSumMode { return switch (operator) { .prefix_sum => .inclusive, .prefix_sum_exclusive => .exclusive, }; }};pub const PrefixSum = struct { extent: u64, dtype: DType = .f32, mode: PrefixSumMode = .inclusive, threads: u32 = 256, element_axis: []const u8 = "e",};pub const prefix_sum_family_version: u32 = 1;pub const prefix_sum_warp_size: u32 = 32;pub const prefix_sum_max_threads: u32 = 1024;pub fn prefixSumDTypeSupported(dtype: DType) bool { return switch (dtype) { .f32, .f16, .u32 => true, else => false, };}pub fn prefixSumInstanceValid(instance: PrefixSum) bool { if (!prefixSumDTypeSupported(instance.dtype)) return false; if (instance.extent == 0) return false; if (instance.threads == 0 or instance.threads > prefix_sum_max_threads) return false; if (instance.threads % prefix_sum_warp_size != 0) return false; return instance.extent <= instance.threads;}pub fn prefixSumThreadsForExtent(extent: u64) ?u32 { if (extent == 0 or extent > prefix_sum_max_threads) return null; const wide: u64 = extent + prefix_sum_warp_size - 1; const rounded: u32 = @intCast((wide / prefix_sum_warp_size) * prefix_sum_warp_size); return @max(rounded, prefix_sum_warp_size);}pub const PrefixSumThreadCandidates = struct { count: usize = 0, items: [6]u32 = @as([6]u32, @splat(0)), pub fn slice(self: *const PrefixSumThreadCandidates) []const u32 { return self.items[0..self.count]; }};pub fn prefixSumThreadCandidatesForExtent(extent: u64) PrefixSumThreadCandidates { var result = PrefixSumThreadCandidates{}; const base = prefixSumThreadsForExtent(extent) orelse return result; result.items[result.count] = base; result.count += 1; var threads: u32 = prefix_sum_warp_size; while (threads <= prefix_sum_max_threads) : (threads *= 2) { if (threads == base) continue; if (@as(u64, threads) < extent) continue; if (result.count >= result.items.len) break; result.items[result.count] = threads; result.count += 1; } return result;}fn prefix_sum_scan_core_seeds_zero(inner: anytype, ctx: anytype) !void { try inner.storeIndex(ctx.zero_value, ctx.warp_sums, ctx.local);}fn prefix_sum_scan_core_is_last_lane(inner: anytype, ctx: anytype) !void { try inner.storeIndex(ctx.scanned, ctx.warp_sums, ctx.warp);}fn prefix_sum_scan_core_is_first_warp(inner: anytype, ctx: anytype) !void { const warp_sum = try inner.loadIndex(ctx.warp_sums, ctx.lane); const warp_scan = try inner.warpScan(.add, .inclusive, warp_sum); try inner.storeIndex(warp_scan, ctx.warp_sums, ctx.lane);}fn prefix_sum_scan_core_in_range(inner: anytype, ctx: anytype) !void { try ctx.args.param(.dst).store(inner, ctx.result, ctx.tid);}fn prefixSumScanCore( k: anytype, spec: PrefixSum, args: anytype, extent: kernel.Value, local: kernel.Value,) !kernel.Value { const tid = try k.globalId(.x); const lane = try k.laneId(); const warp = try k.warpId(); const zero = try k.constantIndex(0); const one = try k.constantIndex(1); const in_range = try k.compare(.lt, tid, extent); const extent_minus_one = try k.sub(extent, one); const clamped_tid = try k.min(tid, extent_minus_one); const loaded = try args.param(.data).load(k, clamped_tid); const accumulator_dtype = comptime prefixSumAccumulatorDType(@TypeOf(loaded).scalar_dtype); const loaded_accumulated = if (comptime @TypeOf(loaded).scalar_dtype == accumulator_dtype) loaded.raw() else (try loaded.cast(k, accumulator_dtype)).raw(); const zero_value = try zeroForDType(k, accumulator_dtype); const element = try k.select(in_range, loaded_accumulated, zero_value); const scanned = try k.warpScan(.add, .inclusive, element); const warp_sums = try k.sharedBuffer(accumulator_dtype, prefix_sum_warp_size); const lane_limit = try k.constantIndex(prefix_sum_warp_size - 1); const warp_count_value = try k.constantIndex(prefix_sum_warp_size); const seeds_zero = try k.compare(.lt, local, warp_count_value); try k.guardDo(seeds_zero, .{ .warp_sums = warp_sums, .local = local, .zero_value = zero_value }, prefix_sum_scan_core_seeds_zero); try k.barrier(.block); const is_last_lane = try k.compare(.eq, lane, lane_limit); try k.guardDo(is_last_lane, .{ .warp_sums = warp_sums, .warp = warp, .scanned = scanned }, prefix_sum_scan_core_is_last_lane); try k.barrier(.block); const is_first_warp = try k.compare(.eq, warp, zero); try k.guardDo(is_first_warp, .{ .warp_sums = warp_sums, .lane = lane }, prefix_sum_scan_core_is_first_warp); try k.barrier(.block); const has_base = try k.compare(.gt, warp, zero); const warp_minus_one = try k.sub(warp, one); const base_index = try k.select(has_base, warp_minus_one, zero); const base_loaded = try k.loadIndex(warp_sums, base_index); const base = try k.select(has_base, base_loaded, zero_value); const inclusive = try k.add(scanned, base); const accumulated = switch (spec.mode) { .inclusive => inclusive, .exclusive => try k.sub(inclusive, element), }; const dst_dtype = comptime @TypeOf(args.param(.dst)).element_dtype; const result = switch (dst_dtype) { .f32 => accumulated, .u32 => accumulated, .f16 => try k.cast(accumulated, .f16), else => return error.UnsupportedDType, }; try k.guardDo(in_range, .{ .args = args, .tid = tid, .result = result }, prefix_sum_scan_core_in_range); return warp_sums;}fn prefixSumAccumulatorDType(dtype: DType) DType { return switch (dtype) { .f16 => .f32, else => dtype, };}fn zeroForDType(k: anytype, comptime dtype: DType) !kernel.Value { return switch (dtype) { .f32 => k.constantFloat(.f32, 0.0), .u32 => k.constantInt(.u32, 0), else => error.UnsupportedDType, };}fn prefixSumBody(k: anytype, spec: PrefixSum, args: anytype) !void { if (!prefixSumInstanceValid(spec)) return error.UnsupportedPrefixSumInstance; const extent = try k.constantIndex(try indexExtent(spec.extent)); const local = try k.threadId(.x); _ = try prefixSumScanCore(k, spec, args, extent, local);}fn prefixSumRuntimeBody(k: anytype, spec: PrefixSum, args: anytype) !void { if (!prefixSumInstanceValid(spec)) return error.UnsupportedPrefixSumInstance; const extent = try k.castIndex(args.param(.extent).raw()); const local = try k.threadId(.x); _ = try prefixSumScanCore(k, spec, args, extent, local);}fn prefixSumFamilySchedule(instance: PrefixSum) kernel.logical.schedule.ThreadBlocks { return kernel.logical.schedule.threadBlocks(.{ .x = instance.threads });}fn prefixSumRuntimeFamily(comptime dtype: DType) type { return kernel.logical.Family(.{ .name = std.fmt.comptimePrint("accy_kernel_scan_prefix_sum_runtime_{s}", .{dtype.name()}), .parameters = .{ .dst = kernel.dynamicBuffer(dtype), .data = kernel.dynamicBuffer(dtype), .extent = kernel.scalar(.i32), }, .Instance = PrefixSum, .schedule = prefixSumFamilySchedule, .body = prefixSumRuntimeBody, });}pub const PrefixSumRuntimeFamilyF32 = prefixSumRuntimeFamily(.f32);pub const PrefixSumRuntimeFamilyF16 = prefixSumRuntimeFamily(.f16);pub const PrefixSumRuntimeFamilyU32 = prefixSumRuntimeFamily(.u32);pub const DeviceScan = struct { extent: u64, dtype: DType = .f32, mode: PrefixSumMode = .inclusive, threads: u32 = 256, element_axis: []const u8 = "e",};pub const device_scan_family_version: u32 = 1;pub const device_scan_max_blocks: u32 = prefix_sum_max_threads;pub fn deviceScanBlockCount(extent: u64, threads: u32) u64 { return (extent + threads - 1) / threads;}pub fn deviceScanDTypeSupported(dtype: DType) bool { return switch (dtype) { .f32, .f16, .u32 => true, else => false, };}pub fn deviceScanInstanceValid(instance: DeviceScan) bool { if (!deviceScanDTypeSupported(instance.dtype)) return false; if (instance.extent == 0) return false; if (instance.threads == 0 or instance.threads > prefix_sum_max_threads) return false; if (instance.threads % prefix_sum_warp_size != 0) return false; return extent_mod.blockCountWithinLimit(instance.extent, instance.threads, device_scan_max_blocks);}fn deviceScanPrefixSum(spec: DeviceScan) PrefixSum { return .{ .extent = spec.extent, .dtype = spec.dtype, .mode = spec.mode, .threads = spec.threads, .element_axis = spec.element_axis, };}fn device_scan_block_scan_body_writes_total(inner: anytype, ctx: anytype) !void { try ctx.args.param(.sums).store(inner, ctx.total, ctx.block);}fn deviceScanBlockScanBody(k: anytype, spec: DeviceScan, args: anytype) !void { if (!deviceScanInstanceValid(spec)) return error.UnsupportedDeviceScanInstance; const extent = try k.castIndex(args.param(.extent).raw()); const local = try k.threadId(.x); const warp_sums = try prefixSumScanCore(k, deviceScanPrefixSum(spec), args, extent, local); const block = try k.blockId(.x); const zero = try k.constantIndex(0); const last_slot = try k.constantIndex(prefix_sum_warp_size - 1); const total = try k.loadIndex(warp_sums, last_slot); const writes_total = try k.compare(.eq, local, zero); try k.guardDo(writes_total, .{ .args = args, .total = total, .block = block }, device_scan_block_scan_body_writes_total);}fn device_scan_add_base_body_in_range(inner: anytype, ctx: anytype) !void { const current = try ctx.args.param(.dst).load(inner, ctx.tid); const updated = try inner.add(current.raw(), ctx.base.raw()); try ctx.args.param(.dst).store(inner, updated, ctx.tid);}fn deviceScanAddBaseBody(k: anytype, spec: DeviceScan, args: anytype) !void { if (!deviceScanInstanceValid(spec)) return error.UnsupportedDeviceScanInstance; const extent = try k.castIndex(args.param(.extent).raw()); const tid = try k.globalId(.x); const block = try k.blockId(.x); const base = try args.param(.base).load(k, block); const in_range = try k.compare(.lt, tid, extent); try k.guardDo(in_range, .{ .args = args, .tid = tid, .base = base }, device_scan_add_base_body_in_range);}fn deviceScanFamilySchedule(instance: DeviceScan) kernel.logical.schedule.ThreadBlocks { return kernel.logical.schedule.threadBlocks(.{ .x = instance.threads });}fn deviceScanBlockScanRuntimeFamily(comptime dtype: DType) type { return kernel.logical.Family(.{ .name = std.fmt.comptimePrint("accy_kernel_scan_device_block_scan_runtime_{s}", .{dtype.name()}), .parameters = .{ .dst = kernel.dynamicBuffer(deviceScanBlockScanOutputDType(dtype)), .data = kernel.dynamicBuffer(dtype), .sums = kernel.dynamicBuffer(prefixSumAccumulatorDType(dtype)), .extent = kernel.scalar(.i32), }, .Instance = DeviceScan, .schedule = deviceScanFamilySchedule, .body = deviceScanBlockScanBody, });}fn deviceScanBlockScanOutputDType(comptime dtype: DType) DType { return switch (dtype) { .f16 => .f32, else => dtype, };}fn device_scan_add_base_body_f16_in_range(inner: anytype, ctx: anytype) !void { const local = try ctx.args.param(.local).load(inner, ctx.tid); const updated = try inner.add(local.raw(), ctx.base.raw()); const result = try inner.cast(updated, .f16); try ctx.args.param(.dst).store(inner, result, ctx.tid);}fn deviceScanAddBaseBodyF16(k: anytype, spec: DeviceScan, args: anytype) !void { if (!deviceScanInstanceValid(spec)) return error.UnsupportedDeviceScanInstance; const extent = try k.castIndex(args.param(.extent).raw()); const tid = try k.globalId(.x); const block = try k.blockId(.x); const base = try args.param(.base).load(k, block); const in_range = try k.compare(.lt, tid, extent); try k.guardDo(in_range, .{ .args = args, .tid = tid, .base = base }, device_scan_add_base_body_f16_in_range);}fn deviceScanAddBaseRuntimeFamily(comptime dtype: DType) type { return kernel.logical.Family(.{ .name = std.fmt.comptimePrint("accy_kernel_scan_device_add_base_runtime_{s}", .{dtype.name()}), .parameters = switch (dtype) { .f16 => .{ .dst = kernel.dynamicBuffer(dtype), .local = kernel.dynamicBuffer(.f32), .base = kernel.dynamicBuffer(.f32), .extent = kernel.scalar(.i32), }, else => .{ .dst = kernel.dynamicBuffer(dtype), .base = kernel.dynamicBuffer(prefixSumAccumulatorDType(dtype)), .extent = kernel.scalar(.i32), }, }, .Instance = DeviceScan, .schedule = deviceScanFamilySchedule, .body = switch (dtype) { .f16 => deviceScanAddBaseBodyF16, else => deviceScanAddBaseBody, }, });}pub const DeviceScanBlockScanRuntimeFamilyF32 = deviceScanBlockScanRuntimeFamily(.f32);pub const DeviceScanBlockScanRuntimeFamilyF16 = deviceScanBlockScanRuntimeFamily(.f16);pub const DeviceScanBlockScanRuntimeFamilyU32 = deviceScanBlockScanRuntimeFamily(.u32);pub const DeviceScanAddBaseRuntimeFamilyF32 = deviceScanAddBaseRuntimeFamily(.f32);pub const DeviceScanAddBaseRuntimeFamilyF16 = deviceScanAddBaseRuntimeFamily(.f16);pub const DeviceScanAddBaseRuntimeFamilyU32 = deviceScanAddBaseRuntimeFamily(.u32);pub const DeviceScanStages = struct { block_count: u32, block_scan: DeviceScan, sums_scan: PrefixSum, add_base: DeviceScan,};pub fn deviceScanStages(instance: DeviceScan) !DeviceScanStages { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; const blocks: u32 = @intCast(deviceScanBlockCount(instance.extent, instance.threads)); const sums_threads = prefixSumThreadsForExtent(blocks) orelse return error.UnsupportedDeviceScanInstance; return .{ .block_count = blocks, .block_scan = instance, .sums_scan = .{ .extent = blocks, .dtype = prefixSumAccumulatorDType(instance.dtype), .mode = .exclusive, .threads = sums_threads, .element_axis = "b", }, .add_base = instance, };}pub fn deviceScanBlockScanFamilyTarget(allocator: std.mem.Allocator, instance: DeviceScan) ![]u8 { return std.fmt.allocPrint( allocator, "accy.kernel.scan.device_{s}_block_scan_family_{d}_{s}", .{ prefixSumModePrefix(instance.mode), instance.threads, instance.dtype.name() }, );}pub fn deviceScanBlockScanFamilyEntryName(allocator: std.mem.Allocator, instance: DeviceScan) ![]u8 { return std.fmt.allocPrint( allocator, "accy_kernel_scan_device_{s}_block_scan_family_{d}_{s}", .{ prefixSumModePrefix(instance.mode), instance.threads, instance.dtype.name() }, );}pub fn deviceScanAddBaseFamilyTarget(allocator: std.mem.Allocator, instance: DeviceScan) ![]u8 { return std.fmt.allocPrint( allocator, "accy.kernel.scan.device_add_base_family_{d}_{s}", .{ instance.threads, instance.dtype.name() }, );}pub fn deviceScanAddBaseFamilyEntryName(allocator: std.mem.Allocator, instance: DeviceScan) ![]u8 { return std.fmt.allocPrint( allocator, "accy_kernel_scan_device_add_base_family_{d}_{s}", .{ instance.threads, instance.dtype.name() }, );}pub fn deviceScanFamilyTarget(allocator: std.mem.Allocator, instance: DeviceScan) ![]u8 { return std.fmt.allocPrint( allocator, "accy.kernel.scan.device_{s}_family_{d}_{s}", .{ prefixSumModePrefix(instance.mode), instance.threads, instance.dtype.name() }, );}pub fn deviceScanThreadsForExtent(extent: u64) ?u32 { if (extent == 0) return null; const max_extent = @as(u64, prefix_sum_max_threads) * device_scan_max_blocks; if (extent > max_extent) return null; const needed = (extent + device_scan_max_blocks - 1) / device_scan_max_blocks; const wide = needed + prefix_sum_warp_size - 1; const rounded: u32 = @intCast((wide / prefix_sum_warp_size) * prefix_sum_warp_size); return @max(rounded, prefix_sum_warp_size);}pub const DeviceScanThreadCandidates = struct { count: usize = 0, items: [6]u32 = @as([6]u32, @splat(0)), pub fn slice(self: *const DeviceScanThreadCandidates) []const u32 { return self.items[0..self.count]; }};pub fn deviceScanThreadCandidatesForExtent(extent: u64) DeviceScanThreadCandidates { var result = DeviceScanThreadCandidates{}; const base = deviceScanThreadsForExtent(extent) orelse return result; result.items[result.count] = base; result.count += 1; var threads: u32 = prefix_sum_warp_size; while (threads <= prefix_sum_max_threads) : (threads *= 2) { if (threads == base) continue; if (threads < base) continue; if (result.count >= result.items.len) break; result.items[result.count] = threads; result.count += 1; } return result;}pub fn deviceScanFamilySpecialization(backing_allocator: std.mem.Allocator, instance: DeviceScan) !entry.OwnedSpecialization { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; var owned = entry.OwnedSpecialization.init(backing_allocator); errdefer owned.deinit(); const lifetime_allocator = owned.allocator(); const inputs = try lifetime_allocator.alloc(entry.Shape, 1); inputs[0] = try entry.runtimeShape1D(lifetime_allocator, instance.element_axis, instance.extent); const outputs = try lifetime_allocator.alloc(entry.Shape, 1); outputs[0] = try entry.runtimeShape1D(lifetime_allocator, instance.element_axis, instance.extent); owned.value = .{ .dtype = instance.dtype, .operation = .{ .scan = instance.mode.operation() }, .inputs = inputs, .outputs = outputs, .schedule = try entry.runtimeThreadBlocks1D(lifetime_allocator, instance.element_axis, instance.extent, instance.threads), }; owned.value.launch = owned.value.schedule.?.launch(); var family = try deviceScanBlockScanShapeFamily(backing_allocator, instance); errdefer family.deinit(); try owned.takeShapeFamily(&family); return owned;}pub fn deviceScanInstanceFromSpecialization(specialization: entry.Specialization) ?DeviceScan { if (!specialization.scheduleMatchesLaunch()) return null; const mode: PrefixSumMode = if (specialization.operationIs(.{ .scan = .prefix_sum })) .inclusive else if (specialization.operationIs(.{ .scan = .prefix_sum_exclusive })) .exclusive else return null; const dtype = specialization.dtype orelse return null; if (!deviceScanDTypeSupported(dtype)) return null; if (specialization.inputs.len != 1 or specialization.outputs.len != 1) return null; if (specialization.reductions.len != 0) return null; const data = specialization.inputs[0]; const output = specialization.outputs[0]; if (data.axes.len != 1 or output.axes.len != 1) return null; const extent = data.axes[0].extent; if (output.axes[0].extent != extent) return null; const launch = specialization.launch orelse return null; if (launch.grid[0] <= 1) return null; const instance = DeviceScan{ .extent = extent, .dtype = dtype, .mode = mode, .threads = launch.threadgroup[0], .element_axis = data.axes[0].name, }; if (!deviceScanInstanceValid(instance)) return null; if (launch.grid[0] != deviceScanBlockCount(extent, instance.threads)) return null; return instance;}pub fn deviceScanPipeline( backing_allocator: std.mem.Allocator, instance: DeviceScan,) !artifact_product.OwnedKernelCallPipeline { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; const stages = try deviceScanStages(instance); var owned = artifact_product.OwnedKernelCallPipeline.init(backing_allocator); errdefer owned.deinit(); const arena = owned.allocator(); const block_count_extent = artifact_product.PipelineScalarDerivation{ .ceil_div = .{ .argument_index = 0, .divisor = instance.threads }, }; const accumulator_dtype = prefixSumAccumulatorDType(instance.dtype); const local_prefix_index: ?u32 = if (instance.dtype == .f16) 2 else null; const intermediate_count: usize = if (local_prefix_index != null) 3 else 2; const intermediates = try arena.alloc(artifact_product.PipelineIntermediate, intermediate_count); intermediates[0] = .{ .dtype = accumulator_dtype, .extent = block_count_extent }; intermediates[1] = .{ .dtype = accumulator_dtype, .extent = block_count_extent }; if (local_prefix_index) |index| { intermediates[index] = .{ .dtype = accumulator_dtype, .extent = .{ .forward = 0 } }; } const runtime_scalar_bounds = try arena.alloc(artifact_product.PipelineRuntimeScalarBound, 1); runtime_scalar_bounds[0] = .{ .argument_index = 0, .max_u32 = try deviceScanPipelineRuntimeExtentCapacity(instance, stages), }; const pipeline_stages = try arena.alloc(artifact_product.PipelineStage, 3); pipeline_stages[0] = .{ .target = try deviceScanBlockScanFamilyTarget(arena, instance), .version = device_scan_family_version, .buffers = try arena.dupe(artifact_product.PipelineValueRef, if (local_prefix_index) |index| &.{ .{ .intermediate = index }, .{ .operand = 0 }, .{ .intermediate = 0 }, } else &.{ .{ .result = 0 }, .{ .operand = 0 }, .{ .intermediate = 0 }, }), .scalars = try arena.dupe(artifact_product.PipelineScalarDerivation, &.{ .{ .forward = 0 }, }), }; pipeline_stages[1] = .{ .target = try prefixSumFamilyTarget(arena, stages.sums_scan), .version = prefix_sum_family_version, .buffers = try arena.dupe(artifact_product.PipelineValueRef, &.{ .{ .intermediate = 1 }, .{ .intermediate = 0 }, }), .scalars = try arena.dupe(artifact_product.PipelineScalarDerivation, &.{ block_count_extent, }), }; pipeline_stages[2] = .{ .target = try deviceScanAddBaseFamilyTarget(arena, instance), .version = device_scan_family_version, .buffers = try arena.dupe(artifact_product.PipelineValueRef, if (local_prefix_index) |index| &.{ .{ .result = 0 }, .{ .intermediate = index }, .{ .intermediate = 1 }, } else &.{ .{ .result = 0 }, .{ .intermediate = 1 }, }), .scalars = try arena.dupe(artifact_product.PipelineScalarDerivation, &.{ .{ .forward = 0 }, }), }; owned.value = .{ .target = try deviceScanFamilyTarget(arena, instance), .version = device_scan_family_version, .operand_count = 1, .result_count = 1, .runtime_scalar_argument_count = 1, .runtime_scalar_bounds = runtime_scalar_bounds, .intermediates = intermediates, .stages = pipeline_stages, }; return owned;}pub const DeviceScanPipelineArtifacts = struct { block_scan: kernel.OwnedKernelCallArtifact, sums_scan: kernel.OwnedKernelCallArtifact, add_base: kernel.OwnedKernelCallArtifact, pub fn entries(self: *const DeviceScanPipelineArtifacts) [3]artifact_product.KernelCallArtifact { return .{ self.block_scan.entry(), self.sums_scan.entry(), self.add_base.entry() }; } pub fn deinit(self: *DeviceScanPipelineArtifacts) void { self.block_scan.deinit(); self.sums_scan.deinit(); self.add_base.deinit(); self.* = undefined; }};pub fn createDeviceScanPipelineArtifacts( allocator: std.mem.Allocator, handle: kernel.BackendHandle, instance: DeviceScan, options: entry.ArtifactOptions,) !DeviceScanPipelineArtifacts { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; const stages = try deviceScanStages(instance); var block_scan = try createDeviceScanBlockScanFamilyArtifact(allocator, handle, instance, options); errdefer block_scan.deinit(); var sums_scan = try createPrefixSumFamilyArtifact(allocator, handle, stages.sums_scan, options); errdefer sums_scan.deinit(); const add_base = try createDeviceScanAddBaseFamilyArtifact(allocator, handle, instance, options); return .{ .block_scan = block_scan, .sums_scan = sums_scan, .add_base = add_base };}pub fn deviceScanRuntimeArguments(instance: DeviceScan) ![1]choir_abi.ScalarArgument { return .{ .{ .u32 = try runtimeExtentArgument(instance.extent) }, };}fn deviceScanPipelineRuntimeExtentCapacity(instance: DeviceScan, stages: DeviceScanStages) !u32 { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; return std.math.mul(u32, instance.threads, stages.sums_scan.threads) catch return error.UnsupportedDeviceScanInstance;}pub fn deviceScanMaxExtent(instance: DeviceScan) u64 { return @as(u64, instance.threads) * device_scan_max_blocks;}pub fn deviceScanShapeProfileDimensions(instance: DeviceScan) [1]artifact_product.KernelCallShapeProfileDimension { return .{ .{ .name = instance.element_axis, .runtime_scalar_argument_index = 0, .bounds = .{ .min = 1, .max = deviceScanMaxExtent(instance) }, }, };}fn deviceScanLaunch(instance: DeviceScan) !artifact_product.KernelCallLaunch { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; return .{ .derived = .{ .grid = .{ .{ .runtime_u32_ceil_div = .{ .argument_index = 0, .divisor = instance.threads } }, .{ .fixed = 1 }, .{ .fixed = 1 }, }, .threadgroup = .{ instance.threads, 1, 1 }, } };}pub fn deviceScanBlockScanShapeFamily(backing_allocator: std.mem.Allocator, instance: DeviceScan) !shape.Family { var builder = try shape.Builder.init(backing_allocator, "device_scan_block_scan"); errdefer builder.deinit(); const elements = try builder.symbol(instance.element_axis); const elements_expr = try builder.symbolExpression(elements); const blocks = try builder.symbol("blocks"); const blocks_expr = try builder.symbolExpression(blocks); _ = try builder.tensor("data", &.{elements_expr}); _ = try builder.tensor("out", &.{elements_expr}); _ = try builder.tensor("sums", &.{blocks_expr}); try builder.assumeBounds(elements_expr, .{ .min = 1, .max = deviceScanMaxExtent(instance) }); try builder.assumeBounds(blocks_expr, .{ .min = 1, .max = device_scan_max_blocks }); return builder.finish();}pub fn deviceScanBlockScanFamilyFingerprint(backing_allocator: std.mem.Allocator, instance: DeviceScan) !u64 { var family = try deviceScanBlockScanShapeFamily(backing_allocator, instance); defer family.deinit(); return shape.fingerprint(family);}pub fn deviceScanAddBaseShapeFamily(backing_allocator: std.mem.Allocator, instance: DeviceScan) !shape.Family { var builder = try shape.Builder.init(backing_allocator, "device_scan_add_base"); errdefer builder.deinit(); const elements = try builder.symbol(instance.element_axis); const elements_expr = try builder.symbolExpression(elements); const blocks = try builder.symbol("blocks"); const blocks_expr = try builder.symbolExpression(blocks); _ = try builder.tensor("out", &.{elements_expr}); if (instance.dtype == .f16) { _ = try builder.tensor("local", &.{elements_expr}); } _ = try builder.tensor("base", &.{blocks_expr}); try builder.assumeBounds(elements_expr, .{ .min = 1, .max = deviceScanMaxExtent(instance) }); try builder.assumeBounds(blocks_expr, .{ .min = 1, .max = device_scan_max_blocks }); return builder.finish();}pub fn deviceScanAddBaseFamilyFingerprint(backing_allocator: std.mem.Allocator, instance: DeviceScan) !u64 { var family = try deviceScanAddBaseShapeFamily(backing_allocator, instance); defer family.deinit(); return shape.fingerprint(family);}pub fn createDeviceScanBlockScanFamilyArtifact( allocator: std.mem.Allocator, handle: kernel.BackendHandle, instance: DeviceScan, options: entry.ArtifactOptions,) !kernel.OwnedKernelCallArtifact { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; const target = try deviceScanBlockScanFamilyTarget(allocator, instance); defer allocator.free(target); const entry_name = try deviceScanBlockScanFamilyEntryName(allocator, instance); defer allocator.free(entry_name); const family_fingerprint = options.shape_family_fingerprint orelse try deviceScanBlockScanFamilyFingerprint(allocator, instance); const shape_profile_dimensions = deviceScanShapeProfileDimensions(instance); const shape_profile = options.shape_profile orelse artifact_product.KernelCallShapeProfile{ .name = "device_scan_block_scan", .fingerprint = family_fingerprint, .dimensions = shape_profile_dimensions[0..], }; var graph = switch (instance.dtype) { .f32 => try DeviceScanBlockScanRuntimeFamilyF32.buildNamed(allocator, options.limits, entry_name, instance), .f16 => try DeviceScanBlockScanRuntimeFamilyF16.buildNamed(allocator, options.limits, entry_name, instance), .u32 => try DeviceScanBlockScanRuntimeFamilyU32.buildNamed(allocator, options.limits, entry_name, instance), else => return error.UnsupportedDType, }; defer graph.deinit(); return kernel.createKernelCallArtifact(allocator, handle, &graph, .{ .target = target, .version = device_scan_family_version, .format = options.format, .kernel_plan = options.kernel_plan, .element_count_argument = options.element_count_argument, .shape_family_fingerprint = family_fingerprint, .shape_profile = shape_profile, .launch = options.launch orelse try deviceScanLaunch(instance), .runtime_scalar_argument_count = if (options.runtime_scalar_argument_count == 0) 1 else options.runtime_scalar_argument_count, .static_arguments = options.static_arguments, });}pub fn createDeviceScanAddBaseFamilyArtifact( allocator: std.mem.Allocator, handle: kernel.BackendHandle, instance: DeviceScan, options: entry.ArtifactOptions,) !kernel.OwnedKernelCallArtifact { if (!deviceScanInstanceValid(instance)) return error.UnsupportedDeviceScanInstance; const target = try deviceScanAddBaseFamilyTarget(allocator, instance); defer allocator.free(target); const entry_name = try deviceScanAddBaseFamilyEntryName(allocator, instance); defer allocator.free(entry_name); const family_fingerprint = options.shape_family_fingerprint orelse try deviceScanAddBaseFamilyFingerprint(allocator, instance); const shape_profile_dimensions = deviceScanShapeProfileDimensions(instance); const shape_profile = options.shape_profile orelse artifact_product.KernelCallShapeProfile{ .name = "device_scan_add_base", .fingerprint = family_fingerprint, .dimensions = shape_profile_dimensions[0..], }; var graph = switch (instance.dtype) { .f32 => try DeviceScanAddBaseRuntimeFamilyF32.buildNamed(allocator, options.limits, entry_name, instance), .f16 => try DeviceScanAddBaseRuntimeFamilyF16.buildNamed(allocator, options.limits, entry_name, instance), .u32 => try DeviceScanAddBaseRuntimeFamilyU32.buildNamed(allocator, options.limits, entry_name, instance), else => return error.UnsupportedDType, }; defer graph.deinit(); return kernel.createKernelCallArtifact(allocator, handle, &graph, .{ .target = target, .version = device_scan_family_version, .format = options.format, .kernel_plan = options.kernel_plan, .element_count_argument = options.element_count_argument, .shape_family_fingerprint = family_fingerprint, .shape_profile = shape_profile, .launch = options.launch orelse try deviceScanLaunch(instance), .runtime_scalar_argument_count = if (options.runtime_scalar_argument_count == 0) 1 else options.runtime_scalar_argument_count, .static_arguments = options.static_arguments, });}pub fn prefixSumInstanceTarget(allocator: std.mem.Allocator, instance: PrefixSum) ![]u8 { return std.fmt.allocPrint( allocator, "accy.kernel.scan.prefix_sum{d}_{d}_{s}", .{ instance.extent, instance.threads, instance.dtype.name() }, );}pub fn prefixSumInstanceEntryName(allocator: std.mem.Allocator, instance: PrefixSum) ![]u8 { return std.fmt.allocPrint( allocator, "accy_kernel_scan_prefix_sum{d}_{d}_{s}", .{ instance.extent, instance.threads, instance.dtype.name() }, );}fn prefixSumModePrefix(mode: PrefixSumMode) []const u8 { return switch (mode) { .inclusive => "prefix_sum", .exclusive => "prefix_sum_exclusive", };}pub fn prefixSumFamilyTarget(allocator: std.mem.Allocator, instance: PrefixSum) ![]u8 { return std.fmt.allocPrint( allocator, "accy.kernel.scan.{s}_family_{d}_{s}", .{ prefixSumModePrefix(instance.mode), instance.threads, instance.dtype.name() }, );}pub fn prefixSumFamilyEntryName(allocator: std.mem.Allocator, instance: PrefixSum) ![]u8 { return std.fmt.allocPrint( allocator, "accy_kernel_scan_{s}_family_{d}_{s}", .{ prefixSumModePrefix(instance.mode), instance.threads, instance.dtype.name() }, );}pub fn prefixSumTuningExtents(instance: PrefixSum) [1]u64 { return .{instance.extent};}pub fn prefixSumTuningOperation(instance: PrefixSum) entry.Operation { return .{ .scan = instance.mode.operation() };}pub fn prefixSumFamilyTuningKey( backing_allocator: std.mem.Allocator, device_fingerprint: u64, instance: PrefixSum,) !tuning.FamilyTuningKey { const family_fingerprint = try prefixSumFamilyFingerprint(backing_allocator, instance); const extents = prefixSumTuningExtents(instance); return tuning.FamilyTuningKey.init( device_fingerprint, family_fingerprint, entry.operationFingerprint(prefixSumTuningOperation(instance)), instance.dtype, prefix_sum_family_version, extents[0..], ) orelse unreachable;}pub fn prefixSumRuntimeArguments(instance: PrefixSum) ![1]choir_abi.ScalarArgument { return .{ .{ .u32 = try runtimeExtentArgument(instance.extent) }, };}pub fn prefixSumShapeProfileDimensions(instance: PrefixSum) [1]artifact_product.KernelCallShapeProfileDimension { return .{ .{ .name = instance.element_axis, .runtime_scalar_argument_index = 0, .bounds = .{ .min = 1, .max = instance.threads }, }, };}fn prefixSumLaunch(instance: PrefixSum) !artifact_product.KernelCallLaunch { if (!prefixSumInstanceValid(instance)) return error.UnsupportedPrefixSumInstance; return .{ .derived = .{ .grid = .{ .{ .fixed = 1 }, .{ .fixed = 1 }, .{ .fixed = 1 }, }, .threadgroup = .{ instance.threads, 1, 1 }, } };}pub fn createPrefixSumFamilyArtifact( allocator: std.mem.Allocator, handle: kernel.BackendHandle, instance: PrefixSum, options: entry.ArtifactOptions,) !kernel.OwnedKernelCallArtifact { if (!prefixSumInstanceValid(instance)) return error.UnsupportedPrefixSumInstance; const target = try prefixSumFamilyTarget(allocator, instance); defer allocator.free(target); const entry_name = try prefixSumFamilyEntryName(allocator, instance); defer allocator.free(entry_name); const family_fingerprint = options.shape_family_fingerprint orelse try prefixSumFamilyFingerprint(allocator, instance); const shape_profile_dimensions = prefixSumShapeProfileDimensions(instance); const shape_profile = options.shape_profile orelse artifact_product.KernelCallShapeProfile{ .name = "prefix_sum", .fingerprint = family_fingerprint, .dimensions = shape_profile_dimensions[0..], }; var graph = switch (instance.dtype) { .f32 => try PrefixSumRuntimeFamilyF32.buildNamed(allocator, options.limits, entry_name, instance), .f16 => try PrefixSumRuntimeFamilyF16.buildNamed(allocator, options.limits, entry_name, instance), .u32 => try PrefixSumRuntimeFamilyU32.buildNamed(allocator, options.limits, entry_name, instance), else => return error.UnsupportedDType, }; defer graph.deinit(); return kernel.createKernelCallArtifact(allocator, handle, &graph, .{ .target = target, .version = prefix_sum_family_version, .format = options.format, .kernel_plan = options.kernel_plan, .element_count_argument = options.element_count_argument, .shape_family_fingerprint = family_fingerprint, .shape_profile = shape_profile, .launch = options.launch orelse try prefixSumLaunch(instance), .runtime_scalar_argument_count = if (options.runtime_scalar_argument_count == 0) 1 else options.runtime_scalar_argument_count, .static_arguments = options.static_arguments, });}pub fn prefixSumFamilyFingerprint(backing_allocator: std.mem.Allocator, instance: PrefixSum) !u64 { var family = try prefixSumShapeFamily(backing_allocator, instance); defer family.deinit(); return shape.fingerprint(family);}pub fn prefixSumShapeFamily(backing_allocator: std.mem.Allocator, instance: PrefixSum) !shape.Family { var builder = try shape.Builder.init(backing_allocator, "prefix_sum"); errdefer builder.deinit(); const elements = try builder.symbol(instance.element_axis); const elements_expr = try builder.symbolExpression(elements); _ = try builder.tensor("data", &.{elements_expr}); _ = try builder.tensor("out", &.{elements_expr}); try builder.assumeBounds(elements_expr, .{ .min = 1, .max = instance.threads }); return builder.finish();}pub fn prefixSumFamilySpecialization(backing_allocator: std.mem.Allocator, instance: PrefixSum) !entry.OwnedSpecialization { if (!prefixSumInstanceValid(instance)) return error.UnsupportedPrefixSumInstance; var owned = entry.OwnedSpecialization.init(backing_allocator); errdefer owned.deinit(); const lifetime_allocator = owned.allocator(); const inputs = try lifetime_allocator.alloc(entry.Shape, 1); inputs[0] = try entry.runtimeShape1D(lifetime_allocator, instance.element_axis, instance.extent); const outputs = try lifetime_allocator.alloc(entry.Shape, 1); outputs[0] = try entry.runtimeShape1D(lifetime_allocator, instance.element_axis, instance.extent); owned.value = .{ .dtype = instance.dtype, .operation = .{ .scan = instance.mode.operation() }, .inputs = inputs, .outputs = outputs, .schedule = try entry.runtimeThreadBlocks1D(lifetime_allocator, instance.element_axis, instance.threads, instance.threads), }; owned.value.launch = owned.value.schedule.?.launch(); var family = try prefixSumShapeFamily(backing_allocator, instance); errdefer family.deinit(); try owned.takeShapeFamily(&family); return owned;}pub fn prefixSumInstanceFromSpecialization(specialization: entry.Specialization) ?PrefixSum { if (!specialization.scheduleMatchesLaunch()) return null; const mode: PrefixSumMode = if (specialization.operationIs(.{ .scan = .prefix_sum })) .inclusive else if (specialization.operationIs(.{ .scan = .prefix_sum_exclusive })) .exclusive else return null; const dtype = specialization.dtype orelse return null; if (!prefixSumDTypeSupported(dtype)) return null; if (specialization.inputs.len != 1 or specialization.outputs.len != 1) return null; if (specialization.reductions.len != 0) return null; const data = specialization.inputs[0]; const output = specialization.outputs[0]; if (data.axes.len != 1 or output.axes.len != 1) return null; const extent = data.axes[0].extent; if (output.axes[0].extent != extent) return null; const launch = specialization.launch orelse return null; if (launch.grid[0] != 1) return null; const instance = PrefixSum{ .extent = extent, .dtype = dtype, .mode = mode, .threads = launch.threadgroup[0], .element_axis = data.axes[0].name, }; if (!prefixSumInstanceValid(instance)) return null; return instance;}fn prefixSumSpecialization(comptime spec: PrefixSum) entry.Specialization { return .{ .dtype = spec.dtype, .operation = .{ .scan = spec.mode.operation() }, .inputs = &.{entry.shape1D(spec.element_axis, spec.extent)}, .outputs = &.{entry.shape1D(spec.element_axis, spec.extent)}, .launch = entry.launch1D(1, spec.threads), .schedule = entry.threadBlocks1D(spec.element_axis, spec.threads, spec.threads), };}fn prefixSumProgram(comptime spec: PrefixSum) type { const Body = struct { fn run(k: anytype, args: anytype) !void { try prefixSumBody(k, spec, args); } }; return kernel.logical.Program(.{ .name = std.fmt.comptimePrint( "accy_kernel_scan_prefix_sum{}_{}_{s}", .{ spec.extent, spec.threads, spec.dtype.name() }, ), .parameters = .{ .dst = kernel.dynamicBuffer(spec.dtype), .data = kernel.dynamicBuffer(spec.dtype), }, .body = Body.run, }).withSchedule(kernel.logical.schedule.threadBlocks(.{ .x = spec.threads }));}pub fn prefixSumF32(comptime spec: PrefixSum) type { return entry.Entry(prefixSumProgram(spec), .{ .target = std.fmt.comptimePrint( "accy.kernel.scan.prefix_sum{}_{}_{s}", .{ spec.extent, spec.threads, spec.dtype.name() }, ), .layer = .logical, .category = .scan, .specialization = prefixSumSpecialization(spec), });}pub const PrefixSum8F32 = prefixSumF32(.{ .extent = 8, .threads = 32 });const testing = std.testing;test "scan prefix sum entry runs on CPU" { const allocator = std.testing.allocator; var data = [_]f32{ 1, 2, 3, 4, 5, 6, 7, 8 }; var dst = @as([8]f32, @splat(0)); const ProgramType = prefixSumProgram(.{ .extent = 8, .threads = 32 }); var graph = try ProgramType.build(allocator, ProgramType.Limits.testing); defer graph.deinit(); try graph.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, dst[0..]), kernel.argumentBuffer(f32, data[0..]), }, .{ .grid = .{ 1, 1, 1 }, .block = .{ 32, 1, 1 }, }); try std.testing.expectEqualSlices(f32, &.{ 1, 3, 6, 10, 15, 21, 28, 36 }, dst[0..]);}fn expectPrefixSumMatchesOracle(extent: usize, threads: u32) !void { try expectPrefixSumModeMatchesOracle(extent, threads, .inclusive);}fn expectPrefixSumModeMatchesOracle(extent: usize, threads: u32, mode: PrefixSumMode) !void { const allocator = testing.allocator; const compiled = PrefixSum{ .extent = 1, .threads = threads, .mode = mode }; const runtime = PrefixSum{ .extent = extent, .threads = threads, .mode = mode }; var graph = try PrefixSumRuntimeFamilyF32.build(allocator, PrefixSumRuntimeFamilyF32.Limits.testing, compiled); defer graph.deinit(); const data = try allocator.alloc(f32, extent); defer allocator.free(data); for (data, 0..) |*value, index| value.* = @floatFromInt((index % 7) + 1); const dst = try allocator.alloc(f32, extent); defer allocator.free(dst); @memset(dst, 0); const expected = try allocator.alloc(f32, extent); defer allocator.free(expected); var running: f32 = 0; for (data, 0..) |value, index| { switch (mode) { .inclusive => { running += value; expected[index] = running; }, .exclusive => { expected[index] = running; running += value; }, } } try graph.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, dst), kernel.argumentBuffer(f32, data), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ 1, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); for (expected, dst) |want, got| { try testing.expectApproxEqAbs(want, got, 0.001); }}test "scan prefix sum runtime family matches oracle across warp boundaries" { try expectPrefixSumMatchesOracle(40, 64); try expectPrefixSumMatchesOracle(5, 32); try expectPrefixSumMatchesOracle(128, 128); try expectPrefixSumMatchesOracle(100, 256);}test "scan exclusive prefix sum runtime family matches oracle across warp boundaries" { try expectPrefixSumModeMatchesOracle(40, 64, .exclusive); try expectPrefixSumModeMatchesOracle(5, 32, .exclusive); try expectPrefixSumModeMatchesOracle(128, 128, .exclusive); try expectPrefixSumModeMatchesOracle(100, 256, .exclusive);}test "scan prefix sum f16 runtime family matches oracle through f32 accumulation" { const allocator = testing.allocator; const compiled = PrefixSum{ .extent = 1, .dtype = .f16, .threads = 64 }; const runtime = PrefixSum{ .extent = 40, .dtype = .f16, .threads = 64 }; var graph = try PrefixSumRuntimeFamilyF16.build(allocator, PrefixSumRuntimeFamilyF16.Limits.testing, compiled); defer graph.deinit(); var data: [40]f16 = undefined; for (&data, 0..) |*value, index| value.* = @floatFromInt((index % 7) + 1); var dst = @as([40]f16, @splat(0)); var expected: [40]f32 = undefined; var running: f32 = 0; for (data, 0..) |value, index| { running += @floatCast(value); expected[index] = running; } const launch_value = try entry.runtimeLaunch1D(runtime.threads, runtime.threads); try graph.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f16, dst[0..]), kernel.argumentBuffer(f16, data[0..]), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = launch_value.grid, .block = launch_value.threadgroup, }); for (expected, dst) |want, got| { try testing.expectApproxEqAbs(want, @as(f32, @floatCast(got)), 0.5); }}fn expectPrefixSumU32ModeMatchesOracle(extent: usize, threads: u32, mode: PrefixSumMode) !void { const allocator = testing.allocator; const compiled = PrefixSum{ .extent = 1, .dtype = .u32, .threads = threads, .mode = mode }; const runtime = PrefixSum{ .extent = extent, .dtype = .u32, .threads = threads, .mode = mode }; var graph = try PrefixSumRuntimeFamilyU32.build(allocator, PrefixSumRuntimeFamilyU32.Limits.testing, compiled); defer graph.deinit(); const data = try allocator.alloc(u32, extent); defer allocator.free(data); for (data, 0..) |*value, index| value.* = @intCast((index % 7) + 1); const dst = try allocator.alloc(u32, extent); defer allocator.free(dst); @memset(dst, 0); const expected = try allocator.alloc(u32, extent); defer allocator.free(expected); var running: u32 = 0; for (data, 0..) |value, index| { switch (mode) { .inclusive => { running += value; expected[index] = running; }, .exclusive => { expected[index] = running; running += value; }, } } const launch_value = try entry.runtimeLaunch1D(runtime.threads, runtime.threads); try graph.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(u32, dst), kernel.argumentBuffer(u32, data), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = launch_value.grid, .block = launch_value.threadgroup, }); try testing.expectEqualSlices(u32, expected, dst);}test "scan prefix sum u32 runtime family matches oracle across warp boundaries" { try expectPrefixSumU32ModeMatchesOracle(40, 64, .inclusive); try expectPrefixSumU32ModeMatchesOracle(5, 32, .inclusive); try expectPrefixSumU32ModeMatchesOracle(128, 128, .inclusive); try expectPrefixSumU32ModeMatchesOracle(100, 256, .inclusive); try expectPrefixSumU32ModeMatchesOracle(40, 64, .exclusive); try expectPrefixSumU32ModeMatchesOracle(100, 256, .exclusive);}test "scan prefix sum f16 identity carries the dtype" { const instance = PrefixSum{ .extent = 100, .dtype = .f16, .threads = 128 }; const family_target = try prefixSumFamilyTarget(testing.allocator, instance); defer testing.allocator.free(family_target); try testing.expectEqualStrings("accy.kernel.scan.prefix_sum_family_128_f16", family_target);}test "scan prefix sum u32 identity carries the dtype" { const instance = PrefixSum{ .extent = 100, .dtype = .u32, .threads = 128 }; const family_target = try prefixSumFamilyTarget(testing.allocator, instance); defer testing.allocator.free(family_target); try testing.expectEqualStrings("accy.kernel.scan.prefix_sum_family_128_u32", family_target);}test "scan exclusive prefix sum identity carries the mode" { const instance = PrefixSum{ .extent = 100, .mode = .exclusive, .threads = 128 }; const family_target = try prefixSumFamilyTarget(testing.allocator, instance); defer testing.allocator.free(family_target); try testing.expectEqualStrings("accy.kernel.scan.prefix_sum_exclusive_family_128_f32", family_target);}test "scan prefix sum family tuning keys discriminate modes" { const allocator = testing.allocator; const device = tuning.deviceFingerprint(.{ .identity = .{ .backend = .cuda, .family = .nvidia_cuda, .name = "scan-family-tuning-test-device", .vendor_id = 0x10de, .device_id = 0x2684, } }); const inclusive = PrefixSum{ .extent = 64 }; const exclusive = PrefixSum{ .extent = 64, .mode = .exclusive }; const inclusive_family = try prefixSumFamilyFingerprint(allocator, inclusive); const exclusive_family = try prefixSumFamilyFingerprint(allocator, exclusive); try testing.expectEqual(inclusive_family, exclusive_family); const inclusive_key = try prefixSumFamilyTuningKey(allocator, device, inclusive); const exclusive_key = try prefixSumFamilyTuningKey(allocator, device, exclusive); try testing.expect(!inclusive_key.eql(exclusive_key)); try testing.expectEqual(inclusive_key.family_fingerprint, exclusive_key.family_fingerprint); try testing.expect(inclusive_key.operation_fingerprint != exclusive_key.operation_fingerprint); const repeat_key = try prefixSumFamilyTuningKey(allocator, device, inclusive); try testing.expect(inclusive_key.eql(repeat_key));}test "scan exclusive prefix sum instance round-trips through specialization" { const instance = PrefixSum{ .extent = 100, .mode = .exclusive, .threads = 128 }; var owned = try prefixSumFamilySpecialization(testing.allocator, instance); defer owned.deinit(); const recovered = prefixSumInstanceFromSpecialization(owned.value) orelse return error.TestExpectedPrefixSumInstance; try testing.expectEqual(PrefixSumMode.exclusive, recovered.mode); try testing.expectEqual(instance.extent, recovered.extent); try testing.expectEqual(instance.threads, recovered.threads);}test "scan prefix sum instance validity bounds extent by threads" { try testing.expect(prefixSumInstanceValid(.{ .extent = 256, .threads = 256 })); try testing.expect(!prefixSumInstanceValid(.{ .extent = 257, .threads = 256 })); try testing.expect(!prefixSumInstanceValid(.{ .extent = 8, .threads = 24 })); try testing.expect(!prefixSumInstanceValid(.{ .extent = 0, .threads = 32 })); try testing.expect(!prefixSumInstanceValid(.{ .extent = 8, .threads = 2048 }));}test "scan prefix sum thread selection rounds to warps" { try testing.expectEqual(@as(?u32, 32), prefixSumThreadsForExtent(5)); try testing.expectEqual(@as(?u32, 64), prefixSumThreadsForExtent(40)); try testing.expectEqual(@as(?u32, 1024), prefixSumThreadsForExtent(1024)); try testing.expectEqual(@as(?u32, null), prefixSumThreadsForExtent(1025)); const candidates = prefixSumThreadCandidatesForExtent(40); try testing.expect(candidates.count >= 2); try testing.expectEqual(@as(u32, 64), candidates.items[0]); for (candidates.slice()) |threads| { try testing.expect(@as(u64, threads) >= 40); try testing.expect(threads % prefix_sum_warp_size == 0); }}test "scan prefix sum family identity and artifact contract" { const allocator = testing.allocator; var state = gpu.recording.BackendState{ .allocator = allocator, .kind = .cuda, .format = .cuda_ptx, }; const instance = PrefixSum{ .extent = 40, .threads = 64 }; const family_target = try prefixSumFamilyTarget(allocator, instance); defer allocator.free(family_target); try testing.expectEqualStrings("accy.kernel.scan.prefix_sum_family_64_f32", family_target); var family_artifact = try createPrefixSumFamilyArtifact(allocator, state.handle(), instance, .{ .limits = .testing }); defer family_artifact.deinit(); const family_entry = family_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_prefix_sum_family_64_f32", family_entry.entry_name); try testing.expectEqual(@as(u32, 3), family_entry.argument_count); try testing.expectEqual(@as(u32, 1), family_entry.runtime_scalar_argument_count); switch (family_entry.launch) { .derived => |launch| { try testing.expectEqual(@as(u32, 64), launch.threadgroup[0]); switch (launch.grid[0]) { .fixed => |value| try testing.expectEqual(@as(u32, 1), value), else => return error.TestExpectedFixedGrid, } }, else => return error.TestExpectedDerivedLaunch, }}test "scan prefix sum u32 recording artifacts cover native targets" { const allocator = testing.allocator; const instance = PrefixSum{ .extent = 40, .dtype = .u32, .threads = 64 }; inline for (.{ gpu.ArtifactFormat.cuda_ptx, .vulkan_spirv, .metal_msl }) |format| { var state = gpu.recording.BackendState{ .allocator = allocator, .kind = switch (format) { .cuda_ptx => .cuda, .vulkan_spirv => .vulkan, .metal_msl => .metal, else => .external, }, .format = format, }; var artifact = try createPrefixSumFamilyArtifact(allocator, state.handle(), instance, .{ .limits = .testing, .format = format }); defer artifact.deinit(); const entry_value = artifact.entry(); try testing.expectEqual(format, entry_value.format); try testing.expectEqualStrings("accy_kernel_scan_prefix_sum_family_64_u32", entry_value.entry_name); }}test "scan prefix sum instance round-trips through specialization" { const instance = PrefixSum{ .extent = 100, .threads = 128 }; var owned = try prefixSumFamilySpecialization(testing.allocator, instance); defer owned.deinit(); const recovered = prefixSumInstanceFromSpecialization(owned.value) orelse return error.TestExpectedPrefixSumInstance; try testing.expectEqual(instance.extent, recovered.extent); try testing.expectEqual(instance.threads, recovered.threads); try testing.expectEqual(instance.dtype, recovered.dtype); const u32_instance = PrefixSum{ .extent = 100, .dtype = .u32, .threads = 128, .mode = .exclusive }; var u32_owned = try prefixSumFamilySpecialization(testing.allocator, u32_instance); defer u32_owned.deinit(); const u32_recovered = prefixSumInstanceFromSpecialization(u32_owned.value) orelse return error.TestExpectedPrefixSumInstance; try testing.expectEqual(u32_instance.mode, u32_recovered.mode); try testing.expectEqual(u32_instance.extent, u32_recovered.extent); try testing.expectEqual(u32_instance.threads, u32_recovered.threads); try testing.expectEqual(u32_instance.dtype, u32_recovered.dtype); try testing.expectEqual(@as(?PrefixSum, null), prefixSumInstanceFromSpecialization(.{}));}fn expectDeviceScanMatchesOracle(extent: usize, threads: u32, mode: PrefixSumMode) !void { const allocator = testing.allocator; const compiled = DeviceScan{ .extent = 1, .threads = threads, .mode = mode }; const runtime = DeviceScan{ .extent = extent, .threads = threads, .mode = mode }; const stages = try deviceScanStages(runtime); const data = try allocator.alloc(f32, extent); defer allocator.free(data); for (data, 0..) |*value, index| value.* = @floatFromInt((index % 7) + 1); const dst = try allocator.alloc(f32, extent); defer allocator.free(dst); @memset(dst, 0); const sums = try allocator.alloc(f32, stages.block_count); defer allocator.free(sums); @memset(sums, 0); const bases = try allocator.alloc(f32, stages.block_count); defer allocator.free(bases); @memset(bases, 0); var block_scan = try DeviceScanBlockScanRuntimeFamilyF32.build(allocator, DeviceScanBlockScanRuntimeFamilyF32.Limits.testing, compiled); defer block_scan.deinit(); try block_scan.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, dst), kernel.argumentBuffer(f32, data), kernel.argumentBuffer(f32, sums), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ stages.block_count, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); var sums_scan = try PrefixSumRuntimeFamilyF32.build(allocator, PrefixSumRuntimeFamilyF32.Limits.testing, .{ .extent = 1, .mode = .exclusive, .threads = stages.sums_scan.threads, }); defer sums_scan.deinit(); try sums_scan.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, bases), kernel.argumentBuffer(f32, sums), kernel.argumentI32(@intCast(stages.block_count)), }, .{ .grid = .{ 1, 1, 1 }, .block = .{ stages.sums_scan.threads, 1, 1 }, }); var add_base = try DeviceScanAddBaseRuntimeFamilyF32.build(allocator, DeviceScanAddBaseRuntimeFamilyF32.Limits.testing, compiled); defer add_base.deinit(); try add_base.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, dst), kernel.argumentBuffer(f32, bases), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ stages.block_count, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); var running: f32 = 0; for (data, dst) |value, got| { switch (mode) { .inclusive => { running += value; try testing.expectEqual(running, got); }, .exclusive => { try testing.expectEqual(running, got); running += value; }, } }}fn expectDeviceScanF16MatchesOracle(extent: usize, threads: u32, mode: PrefixSumMode) !void { const allocator = testing.allocator; const compiled = DeviceScan{ .extent = 1, .dtype = .f16, .threads = threads, .mode = mode }; const runtime = DeviceScan{ .extent = extent, .dtype = .f16, .threads = threads, .mode = mode }; const stages = try deviceScanStages(runtime); const data = try allocator.alloc(f16, extent); defer allocator.free(data); for (data, 0..) |*value, index| value.* = @floatFromInt((index % 5) + 1); const local = try allocator.alloc(f32, extent); defer allocator.free(local); @memset(local, 0); const dst = try allocator.alloc(f16, extent); defer allocator.free(dst); @memset(dst, 0); const sums = try allocator.alloc(f32, stages.block_count); defer allocator.free(sums); @memset(sums, 0); const bases = try allocator.alloc(f32, stages.block_count); defer allocator.free(bases); @memset(bases, 0); var block_scan = try DeviceScanBlockScanRuntimeFamilyF16.build(allocator, DeviceScanBlockScanRuntimeFamilyF16.Limits.testing, compiled); defer block_scan.deinit(); try block_scan.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, local), kernel.argumentBuffer(f16, data), kernel.argumentBuffer(f32, sums), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ stages.block_count, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); var sums_scan = try PrefixSumRuntimeFamilyF32.build(allocator, PrefixSumRuntimeFamilyF32.Limits.testing, .{ .extent = 1, .mode = .exclusive, .threads = stages.sums_scan.threads, }); defer sums_scan.deinit(); try sums_scan.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f32, bases), kernel.argumentBuffer(f32, sums), kernel.argumentI32(@intCast(stages.block_count)), }, .{ .grid = .{ 1, 1, 1 }, .block = .{ stages.sums_scan.threads, 1, 1 }, }); var add_base = try DeviceScanAddBaseRuntimeFamilyF16.build(allocator, DeviceScanAddBaseRuntimeFamilyF16.Limits.testing, compiled); defer add_base.deinit(); try add_base.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(f16, dst), kernel.argumentBuffer(f32, local), kernel.argumentBuffer(f32, bases), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ stages.block_count, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); var running: f32 = 0; for (data, dst) |value, got| { const value_f32: f32 = @floatCast(value); switch (mode) { .inclusive => { running += value_f32; const expected: f16 = @floatCast(running); try testing.expectEqual(expected, got); }, .exclusive => { const expected: f16 = @floatCast(running); try testing.expectEqual(expected, got); running += value_f32; }, } }}fn expectDeviceScanU32MatchesOracle(extent: usize, threads: u32, mode: PrefixSumMode) !void { const allocator = testing.allocator; const compiled = DeviceScan{ .extent = 1, .dtype = .u32, .threads = threads, .mode = mode }; const runtime = DeviceScan{ .extent = extent, .dtype = .u32, .threads = threads, .mode = mode }; const stages = try deviceScanStages(runtime); const data = try allocator.alloc(u32, extent); defer allocator.free(data); for (data, 0..) |*value, index| value.* = @intCast((index % 7) + 1); const dst = try allocator.alloc(u32, extent); defer allocator.free(dst); @memset(dst, 0); const sums = try allocator.alloc(u32, stages.block_count); defer allocator.free(sums); @memset(sums, 0); const bases = try allocator.alloc(u32, stages.block_count); defer allocator.free(bases); @memset(bases, 0); var block_scan = try DeviceScanBlockScanRuntimeFamilyU32.build(allocator, DeviceScanBlockScanRuntimeFamilyU32.Limits.testing, compiled); defer block_scan.deinit(); try block_scan.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(u32, dst), kernel.argumentBuffer(u32, data), kernel.argumentBuffer(u32, sums), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ stages.block_count, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); var sums_scan = try PrefixSumRuntimeFamilyU32.build(allocator, PrefixSumRuntimeFamilyU32.Limits.testing, .{ .extent = 1, .dtype = .u32, .mode = .exclusive, .threads = stages.sums_scan.threads, }); defer sums_scan.deinit(); try sums_scan.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(u32, bases), kernel.argumentBuffer(u32, sums), kernel.argumentI32(@intCast(stages.block_count)), }, .{ .grid = .{ 1, 1, 1 }, .block = .{ stages.sums_scan.threads, 1, 1 }, }); var add_base = try DeviceScanAddBaseRuntimeFamilyU32.build(allocator, DeviceScanAddBaseRuntimeFamilyU32.Limits.testing, compiled); defer add_base.deinit(); try add_base.runCpuWithLaunch(allocator, &.{ kernel.argumentBuffer(u32, dst), kernel.argumentBuffer(u32, bases), kernel.argumentI32(@intCast(runtime.extent)), }, .{ .grid = .{ stages.block_count, 1, 1 }, .block = .{ runtime.threads, 1, 1 }, }); var running: u32 = 0; for (data, dst) |value, got| { switch (mode) { .inclusive => { running += value; try testing.expectEqual(running, got); }, .exclusive => { try testing.expectEqual(running, got); running += value; }, } }}test "scan device-wide composition matches running-sum oracle across blocks" { try expectDeviceScanMatchesOracle(100, 32, .inclusive); try expectDeviceScanMatchesOracle(33, 32, .inclusive); try expectDeviceScanMatchesOracle(2048, 256, .inclusive); try expectDeviceScanMatchesOracle(1000, 64, .inclusive);}test "scan device-wide exclusive composition matches running-sum oracle" { try expectDeviceScanMatchesOracle(100, 32, .exclusive); try expectDeviceScanMatchesOracle(1000, 64, .exclusive);}test "scan device-wide f16 composition uses f32 intermediates across blocks" { try expectDeviceScanF16MatchesOracle(100, 32, .inclusive); try expectDeviceScanF16MatchesOracle(1000, 64, .inclusive); try expectDeviceScanF16MatchesOracle(100, 32, .exclusive);}test "scan device-wide u32 composition matches running-sum oracle across blocks" { try expectDeviceScanU32MatchesOracle(100, 32, .inclusive); try expectDeviceScanU32MatchesOracle(33, 32, .inclusive); try expectDeviceScanU32MatchesOracle(2048, 256, .inclusive); try expectDeviceScanU32MatchesOracle(1000, 64, .inclusive); try expectDeviceScanU32MatchesOracle(100, 32, .exclusive); try expectDeviceScanU32MatchesOracle(1000, 64, .exclusive);}test "scan device-wide instance validity caps blocks and accepts supported dtypes" { try testing.expect(deviceScanInstanceValid(.{ .extent = 1024 * 1024, .threads = 1024 })); try testing.expect(!deviceScanInstanceValid(.{ .extent = 1024 * 1024 + 1, .threads = 1024 })); try testing.expect(deviceScanInstanceValid(.{ .extent = 100, .dtype = .f16, .threads = 32 })); try testing.expect(deviceScanInstanceValid(.{ .extent = 100, .dtype = .u32, .threads = 32 })); try testing.expect(!deviceScanInstanceValid(.{ .extent = 0, .threads = 32 })); try testing.expect(!deviceScanInstanceValid(.{ .extent = 100, .threads = 48 })); try testing.expect(!deviceScanInstanceValid(.{ .extent = std.math.maxInt(u64), .threads = 32 }));}test "scan device-wide stages derive the sums scan from the block count" { const stages = try deviceScanStages(.{ .extent = 1000, .threads = 64 }); try testing.expectEqual(@as(u32, 16), stages.block_count); try testing.expectEqual(@as(u64, 16), stages.sums_scan.extent); try testing.expectEqual(PrefixSumMode.exclusive, stages.sums_scan.mode); try testing.expectEqual(@as(u32, 32), stages.sums_scan.threads); try testing.expectEqual(@as(u64, 1000), stages.block_scan.extent); try testing.expectEqual(@as(u64, 1000), stages.add_base.extent); const u32_stages = try deviceScanStages(.{ .extent = 1000, .dtype = .u32, .threads = 64 }); try testing.expectEqual(DType.u32, u32_stages.sums_scan.dtype);}test "scan device-wide identity carries mode threads and dtype" { const block_target = try deviceScanBlockScanFamilyTarget(testing.allocator, .{ .extent = 1000, .threads = 64 }); defer testing.allocator.free(block_target); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_block_scan_family_64_f32", block_target); const exclusive_target = try deviceScanBlockScanFamilyTarget(testing.allocator, .{ .extent = 1000, .mode = .exclusive, .threads = 64 }); defer testing.allocator.free(exclusive_target); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_exclusive_block_scan_family_64_f32", exclusive_target); const add_target = try deviceScanAddBaseFamilyTarget(testing.allocator, .{ .extent = 1000, .threads = 64 }); defer testing.allocator.free(add_target); try testing.expectEqualStrings("accy.kernel.scan.device_add_base_family_64_f32", add_target); const f16_target = try deviceScanFamilyTarget(testing.allocator, .{ .extent = 1000, .dtype = .f16, .threads = 64 }); defer testing.allocator.free(f16_target); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_family_64_f16", f16_target); const u32_target = try deviceScanFamilyTarget(testing.allocator, .{ .extent = 1000, .dtype = .u32, .threads = 64 }); defer testing.allocator.free(u32_target); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_family_64_u32", u32_target);}test "scan device-wide u32 instance round-trips through specialization" { const instance = DeviceScan{ .extent = 2000, .dtype = .u32, .threads = 64, .mode = .exclusive }; var owned = try deviceScanFamilySpecialization(testing.allocator, instance); defer owned.deinit(); const recovered = deviceScanInstanceFromSpecialization(owned.value) orelse return error.TestExpectedDeviceScanInstance; try testing.expectEqual(instance.mode, recovered.mode); try testing.expectEqual(instance.extent, recovered.extent); try testing.expectEqual(instance.threads, recovered.threads); try testing.expectEqual(instance.dtype, recovered.dtype); try testing.expectEqual(@as(?DeviceScan, null), deviceScanInstanceFromSpecialization(.{}));}test "scan device-wide family artifact contract derives the grid from the extent" { const allocator = testing.allocator; var state = gpu.recording.BackendState{ .allocator = allocator, .kind = .cuda, .format = .cuda_ptx, }; const instance = DeviceScan{ .extent = 1000, .threads = 64 }; var block_artifact = try createDeviceScanBlockScanFamilyArtifact(allocator, state.handle(), instance, .{ .limits = .testing }); defer block_artifact.deinit(); const block_entry = block_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_device_prefix_sum_block_scan_family_64_f32", block_entry.entry_name); try testing.expectEqual(@as(u32, 4), block_entry.argument_count); try testing.expectEqual(@as(u32, 1), block_entry.runtime_scalar_argument_count); switch (block_entry.launch) { .derived => |launch| { try testing.expectEqual(@as(u32, 64), launch.threadgroup[0]); switch (launch.grid[0]) { .runtime_u32_ceil_div => |axis| { try testing.expectEqual(@as(u32, 0), axis.argument_index); try testing.expectEqual(@as(u32, 64), axis.divisor); }, else => return error.TestExpectedDerivedGrid, } }, else => return error.TestExpectedDerivedLaunch, } var add_artifact = try createDeviceScanAddBaseFamilyArtifact(allocator, state.handle(), instance, .{ .limits = .testing }); defer add_artifact.deinit(); const add_entry = add_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_device_add_base_family_64_f32", add_entry.entry_name); try testing.expectEqual(@as(u32, 3), add_entry.argument_count); try testing.expectEqual(@as(u32, 1), add_entry.runtime_scalar_argument_count); switch (add_entry.launch) { .derived => |launch| { try testing.expectEqual(@as(u32, 64), launch.threadgroup[0]); switch (launch.grid[0]) { .runtime_u32_ceil_div => |axis| try testing.expectEqual(@as(u32, 64), axis.divisor), else => return error.TestExpectedDerivedGrid, } }, else => return error.TestExpectedDerivedLaunch, } const f16_instance = DeviceScan{ .extent = 1000, .dtype = .f16, .threads = 64 }; var f16_block_artifact = try createDeviceScanBlockScanFamilyArtifact(allocator, state.handle(), f16_instance, .{ .limits = .testing }); defer f16_block_artifact.deinit(); const f16_block_entry = f16_block_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_device_prefix_sum_block_scan_family_64_f16", f16_block_entry.entry_name); try testing.expectEqual(@as(u32, 4), f16_block_entry.argument_count); var f16_add_artifact = try createDeviceScanAddBaseFamilyArtifact(allocator, state.handle(), f16_instance, .{ .limits = .testing }); defer f16_add_artifact.deinit(); const f16_add_entry = f16_add_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_device_add_base_family_64_f16", f16_add_entry.entry_name); try testing.expectEqual(@as(u32, 4), f16_add_entry.argument_count); const u32_instance = DeviceScan{ .extent = 1000, .dtype = .u32, .threads = 64 }; var u32_block_artifact = try createDeviceScanBlockScanFamilyArtifact(allocator, state.handle(), u32_instance, .{ .limits = .testing }); defer u32_block_artifact.deinit(); const u32_block_entry = u32_block_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_device_prefix_sum_block_scan_family_64_u32", u32_block_entry.entry_name); try testing.expectEqual(@as(u32, 4), u32_block_entry.argument_count); var u32_add_artifact = try createDeviceScanAddBaseFamilyArtifact(allocator, state.handle(), u32_instance, .{ .limits = .testing }); defer u32_add_artifact.deinit(); const u32_add_entry = u32_add_artifact.entry(); try testing.expectEqualStrings("accy_kernel_scan_device_add_base_family_64_u32", u32_add_entry.entry_name); try testing.expectEqual(@as(u32, 3), u32_add_entry.argument_count);}test "scan device-wide u32 recording artifacts cover native targets" { const allocator = testing.allocator; const instance = DeviceScan{ .extent = 1000, .dtype = .u32, .threads = 64 }; inline for (.{ gpu.ArtifactFormat.cuda_ptx, .vulkan_spirv, .metal_msl }) |format| { var state = gpu.recording.BackendState{ .allocator = allocator, .kind = switch (format) { .cuda_ptx => .cuda, .vulkan_spirv => .vulkan, .metal_msl => .metal, else => .external, }, .format = format, }; var block_artifact = try createDeviceScanBlockScanFamilyArtifact(allocator, state.handle(), instance, .{ .limits = .testing, .format = format }); defer block_artifact.deinit(); try testing.expectEqual(format, block_artifact.entry().format); try testing.expectEqualStrings("accy_kernel_scan_device_prefix_sum_block_scan_family_64_u32", block_artifact.entry().entry_name); var add_artifact = try createDeviceScanAddBaseFamilyArtifact(allocator, state.handle(), instance, .{ .limits = .testing, .format = format }); defer add_artifact.deinit(); try testing.expectEqual(format, add_artifact.entry().format); try testing.expectEqualStrings("accy_kernel_scan_device_add_base_family_64_u32", add_artifact.entry().entry_name); }}test "scan device-wide pipeline descriptor binds the family artifacts" { const allocator = testing.allocator; var state = gpu.recording.BackendState{ .allocator = allocator, .kind = .cuda, .format = .cuda_ptx, }; const instance = DeviceScan{ .extent = 5000, .threads = 64 }; var artifacts = try createDeviceScanPipelineArtifacts(allocator, state.handle(), instance, .{ .limits = kernel.Limits.testing, }); defer artifacts.deinit(); const entries = artifacts.entries(); const registry = artifact_product.KernelCallRegistry{ .entries = entries[0..] }; var owned = try deviceScanPipeline(allocator, instance); defer owned.deinit(); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_family_64_f32", owned.value.target); try testing.expectEqual(@as(u32, 1), owned.value.operand_count); try testing.expectEqual(@as(usize, 1), owned.value.runtime_scalar_bounds.len); try testing.expectEqual(@as(u32, 0), owned.value.runtime_scalar_bounds[0].argument_index); try testing.expectEqual(@as(u32, 6144), owned.value.runtime_scalar_bounds[0].max_u32); try testing.expectEqual(@as(usize, 2), owned.value.intermediates.len); try testing.expectEqual(@as(usize, 3), owned.value.stages.len); try owned.value.validate(registry, .cuda_ptx); const capacity_args = [_]choir_abi.ScalarArgument{.{ .u32 = 6144 }}; try owned.value.validateRuntimeScalarArguments(capacity_args[0..]); const too_large_args = [_]choir_abi.ScalarArgument{.{ .u32 = 10000 }}; try testing.expectError(error.LaunchArgumentMismatch, owned.value.validateRuntimeScalarArguments(too_large_args[0..])); const exclusive = DeviceScan{ .extent = 5000, .mode = .exclusive, .threads = 64 }; var owned_exclusive = try deviceScanPipeline(allocator, exclusive); defer owned_exclusive.deinit(); try testing.expectEqualStrings( "accy.kernel.scan.device_prefix_sum_exclusive_family_64_f32", owned_exclusive.value.target, ); try testing.expectError(error.InvalidArtifact, owned_exclusive.value.validate(registry, .cuda_ptx)); try testing.expectError( error.UnsupportedDeviceScanInstance, deviceScanPipeline(allocator, .{ .extent = 0, .threads = 64 }), ); const f16_instance = DeviceScan{ .extent = 5000, .dtype = .f16, .threads = 64 }; var f16_artifacts = try createDeviceScanPipelineArtifacts(allocator, state.handle(), f16_instance, .{ .limits = kernel.Limits.testing, }); defer f16_artifacts.deinit(); const f16_entries = f16_artifacts.entries(); const f16_registry = artifact_product.KernelCallRegistry{ .entries = f16_entries[0..] }; var f16_pipeline = try deviceScanPipeline(allocator, f16_instance); defer f16_pipeline.deinit(); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_family_64_f16", f16_pipeline.value.target); try testing.expectEqual(@as(usize, 3), f16_pipeline.value.intermediates.len); try testing.expectEqual(DType.f32, f16_pipeline.value.intermediates[2].dtype); try testing.expectEqual(@as(usize, 3), f16_pipeline.value.stages[2].buffers.len); try f16_pipeline.value.validate(f16_registry, .cuda_ptx); const u32_instance = DeviceScan{ .extent = 5000, .dtype = .u32, .threads = 64 }; var u32_artifacts = try createDeviceScanPipelineArtifacts(allocator, state.handle(), u32_instance, .{ .limits = kernel.Limits.testing, }); defer u32_artifacts.deinit(); const u32_entries = u32_artifacts.entries(); const u32_registry = artifact_product.KernelCallRegistry{ .entries = u32_entries[0..] }; var u32_pipeline = try deviceScanPipeline(allocator, u32_instance); defer u32_pipeline.deinit(); try testing.expectEqualStrings("accy.kernel.scan.device_prefix_sum_family_64_u32", u32_pipeline.value.target); try testing.expectEqual(@as(usize, 2), u32_pipeline.value.intermediates.len); try testing.expectEqual(DType.u32, u32_pipeline.value.intermediates[0].dtype); try testing.expectEqual(DType.u32, u32_pipeline.value.intermediates[1].dtype); try u32_pipeline.value.validate(u32_registry, .cuda_ptx);}Also reachable as
kernel.library.spatial.scan_mod.
Complete call list for kernel.library.scan.createDeviceScanAddBaseFamilyArtifact
7 direct calls.
tiny.accy.kernel.library.scan.deviceScanAddBaseFamilyEntryName[function] atlib/accy/src/kernel/library/scan.zig:426tiny.accy.kernel.library.scan.deviceScanAddBaseFamilyFingerprint[function] atlib/accy/src/kernel/library/scan.zig:725tiny.accy.kernel.library.scan.deviceScanAddBaseFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:418tiny.accy.kernel.library.scan.deviceScanInstanceValid[function] atlib/accy/src/kernel/library/scan.zig:251lib.accy.src.kernel.library.scan.deviceScanLaunch[function] — private source atlib/accy/src/kernel/library/scan.zig:669in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.deviceScanShapeProfileDimensions[function] atlib/accy/src/kernel/library/scan.zig:659tiny.smg.graph.deinit[function] attools/smg/src/graph.zig:243
Complete call list for kernel.library.scan.createDeviceScanBlockScanFamilyArtifact
7 direct calls.
tiny.accy.kernel.library.scan.deviceScanBlockScanFamilyEntryName[function] atlib/accy/src/kernel/library/scan.zig:410tiny.accy.kernel.library.scan.deviceScanBlockScanFamilyFingerprint[function] atlib/accy/src/kernel/library/scan.zig:699tiny.accy.kernel.library.scan.deviceScanBlockScanFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:402tiny.accy.kernel.library.scan.deviceScanInstanceValid[function] atlib/accy/src/kernel/library/scan.zig:251lib.accy.src.kernel.library.scan.deviceScanLaunch[function] — private source atlib/accy/src/kernel/library/scan.zig:669in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.deviceScanShapeProfileDimensions[function] atlib/accy/src/kernel/library/scan.zig:659tiny.smg.graph.deinit[function] attools/smg/src/graph.zig:243
Complete caller list for kernel.library.scan.createPrefixSumFamilyArtifact
8 direct callers.
lib.accy.src.integration.test.runPrefixSumFamilyOnLiveCuda[function] — private source atlib/accy/src/integration/test.zig:1834in nearest public ownerlib.accy.src.integration.testlib.accy.src.kernel.library.catalog.artifact.specialized.scan.create[function] — private source atlib/accy/src/kernel/library/catalog/artifact/specialized/scan.zig:7in nearest public ownerlib.accy.src.kernel.library.catalog.artifact.specialized.scantiny.accy.kernel.library.scan.createDeviceScanPipelineArtifacts[function] atlib/accy/src/kernel/library/scan.zig:628lib.accy.src.kernel.library.scan.test_scan_prefix_sum_family_identity_and_artifact_contract[function] — test source atlib/accy/src/kernel/library/scan.zig:1313in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.test_scan_prefix_sum_u32_recording_artifacts_cover_native_targets[function] — test source atlib/accy/src/kernel/library/scan.zig:1345in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.sort.createRadixDigitPairsPipelineArtifacts[function] atlib/accy/src/kernel/library/sort.zig:1906tiny.accy.kernel.library.sort.createRadixDigitPipelineArtifacts[function] atlib/accy/src/kernel/library/sort.zig:1720tiny.accy.kernel.library.sort.createRadixSplitPipelineArtifacts[function] atlib/accy/src/kernel/library/sort.zig:2067
Complete call list for kernel.library.scan.createPrefixSumFamilyArtifact
7 direct calls.
tiny.accy.kernel.library.scan.prefixSumFamilyEntryName[function] atlib/accy/src/kernel/library/scan.zig:842tiny.accy.kernel.library.scan.prefixSumFamilyFingerprint[function] atlib/accy/src/kernel/library/scan.zig:943tiny.accy.kernel.library.scan.prefixSumFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:834tiny.accy.kernel.library.scan.prefixSumInstanceValid[function] atlib/accy/src/kernel/library/scan.zig:54lib.accy.src.kernel.library.scan.prefixSumLaunch[function] — private source atlib/accy/src/kernel/library/scan.zig:891in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.prefixSumShapeProfileDimensions[function] atlib/accy/src/kernel/library/scan.zig:881tiny.smg.graph.deinit[function] attools/smg/src/graph.zig:243
Complete call list for kernel.library.scan.deviceScanFamilySpecialization
8 direct calls.
tiny.accy.kernel.library.OwnedSpecialization.allocator[method] atlib/accy/src/kernel/library/entry.zig:616tiny.accy.kernel.library.OwnedSpecialization.deinit[method] atlib/accy/src/kernel/library/entry.zig:620tiny.accy.kernel.library.OwnedSpecialization.init[function] atlib/accy/src/kernel/library/entry.zig:609tiny.accy.kernel.library.OwnedSpecialization.takeShapeFamily[method] atlib/accy/src/kernel/library/entry.zig:626tiny.accy.kernel.library.entry.runtimeShape1D[function] atlib/accy/src/kernel/library/entry.zig:651tiny.accy.kernel.library.entry.runtimeThreadBlocks1D[function] atlib/accy/src/kernel/library/entry.zig:955tiny.accy.kernel.library.scan.deviceScanBlockScanShapeFamily[function] atlib/accy/src/kernel/library/scan.zig:681tiny.accy.kernel.library.scan.deviceScanInstanceValid[function] atlib/accy/src/kernel/library/scan.zig:251
Complete caller list for kernel.library.scan.deviceScanInstanceValid
14 direct callers.
lib.accy.src.kernel.library.catalog.family.scan.prefix.deviceScanFamilyInstance[function] — private source atlib/accy/src/kernel/library/catalog/family/scan/prefix.zig:99in nearest public ownerlib.accy.src.kernel.library.catalog.family.scan.prefixtiny.accy.kernel.library.scan.createDeviceScanAddBaseFamilyArtifact[function] atlib/accy/src/kernel/library/scan.zig:771tiny.accy.kernel.library.scan.createDeviceScanBlockScanFamilyArtifact[function] atlib/accy/src/kernel/library/scan.zig:731tiny.accy.kernel.library.scan.createDeviceScanPipelineArtifacts[function] atlib/accy/src/kernel/library/scan.zig:628lib.accy.src.kernel.library.scan.deviceScanAddBaseBody[function] — private source atlib/accy/src/kernel/library/scan.zig:292in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.deviceScanAddBaseBodyF16[function] — private source atlib/accy/src/kernel/library/scan.zig:335in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.deviceScanBlockScanBody[function] — private source atlib/accy/src/kernel/library/scan.zig:273in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.deviceScanFamilySpecialization[function] atlib/accy/src/kernel/library/scan.zig:477tiny.accy.kernel.library.scan.deviceScanInstanceFromSpecialization[function] atlib/accy/src/kernel/library/scan.zig:503lib.accy.src.kernel.library.scan.deviceScanLaunch[function] — private source atlib/accy/src/kernel/library/scan.zig:669in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.deviceScanPipeline[function] atlib/accy/src/kernel/library/scan.zig:534lib.accy.src.kernel.library.scan.deviceScanPipelineRuntimeExtentCapacity[function] — private source atlib/accy/src/kernel/library/scan.zig:650in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.deviceScanStages[function] atlib/accy/src/kernel/library/scan.zig:384lib.accy.src.kernel.library.scan.test_scan_device-wide_instance_validity_caps_blocks_and_accepts_supported_dtypes[function] — test source atlib/accy/src/kernel/library/scan.zig:1641in nearest public ownertiny.accy.kernel.library.scan
Complete call list for kernel.library.scan.deviceScanPipeline
8 direct calls.
tiny.accy.kernel.library.scan.deviceScanAddBaseFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:418tiny.accy.kernel.library.scan.deviceScanBlockScanFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:402tiny.accy.kernel.library.scan.deviceScanFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:434tiny.accy.kernel.library.scan.deviceScanInstanceValid[function] atlib/accy/src/kernel/library/scan.zig:251lib.accy.src.kernel.library.scan.deviceScanPipelineRuntimeExtentCapacity[function] — private source atlib/accy/src/kernel/library/scan.zig:650in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.deviceScanStages[function] atlib/accy/src/kernel/library/scan.zig:384lib.accy.src.kernel.library.scan.prefixSumAccumulatorDType[function] — private source atlib/accy/src/kernel/library/scan.zig:178in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.prefixSumFamilyTarget[function] atlib/accy/src/kernel/library/scan.zig:834
Complete caller list for kernel.library.scan.deviceScanStages
12 direct callers.
tiny.accy.kernel.library.scan.createDeviceScanPipelineArtifacts[function] atlib/accy/src/kernel/library/scan.zig:628tiny.accy.kernel.library.scan.deviceScanPipeline[function] atlib/accy/src/kernel/library/scan.zig:534lib.accy.src.kernel.library.scan.expectDeviceScanF16MatchesOracle[function] — private source atlib/accy/src/kernel/library/scan.zig:1462in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.expectDeviceScanMatchesOracle[function] — private source atlib/accy/src/kernel/library/scan.zig:1390in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.expectDeviceScanU32MatchesOracle[function] — private source atlib/accy/src/kernel/library/scan.zig:1541in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.test_scan_device-wide_stages_derive_the_sums_scan_from_the_block_count[function] — test source atlib/accy/src/kernel/library/scan.zig:1651in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.sort.createRadixDigitPairsPipelineArtifacts[function] atlib/accy/src/kernel/library/sort.zig:1906tiny.accy.kernel.library.sort.createRadixDigitPipelineArtifacts[function] atlib/accy/src/kernel/library/sort.zig:1720tiny.accy.kernel.library.sort.createRadixSplitPipelineArtifacts[function] atlib/accy/src/kernel/library/sort.zig:2067tiny.accy.kernel.library.sort.radixDigitPairsPipeline[function] atlib/accy/src/kernel/library/sort.zig:1942tiny.accy.kernel.library.sort.radixDigitPipeline[function] atlib/accy/src/kernel/library/sort.zig:1756tiny.accy.kernel.library.sort.radixSplitPipeline[function] atlib/accy/src/kernel/library/sort.zig:2103
Complete call list for kernel.library.scan.prefixSumFamilySpecialization
8 direct calls.
tiny.accy.kernel.library.OwnedSpecialization.allocator[method] atlib/accy/src/kernel/library/entry.zig:616tiny.accy.kernel.library.OwnedSpecialization.deinit[method] atlib/accy/src/kernel/library/entry.zig:620tiny.accy.kernel.library.OwnedSpecialization.init[function] atlib/accy/src/kernel/library/entry.zig:609tiny.accy.kernel.library.OwnedSpecialization.takeShapeFamily[method] atlib/accy/src/kernel/library/entry.zig:626tiny.accy.kernel.library.entry.runtimeShape1D[function] atlib/accy/src/kernel/library/entry.zig:651tiny.accy.kernel.library.entry.runtimeThreadBlocks1D[function] atlib/accy/src/kernel/library/entry.zig:955tiny.accy.kernel.library.scan.prefixSumInstanceValid[function] atlib/accy/src/kernel/library/scan.zig:54tiny.accy.kernel.library.scan.prefixSumShapeFamily[function] atlib/accy/src/kernel/library/scan.zig:949
Complete caller list for kernel.library.scan.prefixSumFamilyTarget
12 direct callers.
lib.accy.src.integration.test.runPrefixSumFamilyOnLiveCuda[function] — private source atlib/accy/src/integration/test.zig:1834in nearest public ownerlib.accy.src.integration.testlib.accy.src.kernel.library.catalog.artifact.specialized.scan.create[function] — private source atlib/accy/src/kernel/library/catalog/artifact/specialized/scan.zig:7in nearest public ownerlib.accy.src.kernel.library.catalog.artifact.specialized.scanlib.accy.src.kernel.library.catalog.family.scan.prefix.prefixSumDescriptorForInstance[function] — private source atlib/accy/src/kernel/library/catalog/family/scan/prefix.zig:142in nearest public ownerlib.accy.src.kernel.library.catalog.family.scan.prefixtiny.accy.kernel.library.scan.createPrefixSumFamilyArtifact[function] atlib/accy/src/kernel/library/scan.zig:903tiny.accy.kernel.library.scan.deviceScanPipeline[function] atlib/accy/src/kernel/library/scan.zig:534lib.accy.src.kernel.library.scan.test_scan_exclusive_prefix_sum_identity_carries_the_mode[function] — test source atlib/accy/src/kernel/library/scan.zig:1245in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.test_scan_prefix_sum_f16_identity_carries_the_dtype[function] — test source atlib/accy/src/kernel/library/scan.zig:1231in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.test_scan_prefix_sum_family_identity_and_artifact_contract[function] — test source atlib/accy/src/kernel/library/scan.zig:1313in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.test_scan_prefix_sum_u32_identity_carries_the_dtype[function] — test source atlib/accy/src/kernel/library/scan.zig:1238in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.sort.radixDigitPairsPipeline[function] atlib/accy/src/kernel/library/sort.zig:1942tiny.accy.kernel.library.sort.radixDigitPipeline[function] atlib/accy/src/kernel/library/sort.zig:1756tiny.accy.kernel.library.sort.radixSplitPipeline[function] atlib/accy/src/kernel/library/sort.zig:2103
Complete caller list for kernel.library.scan.prefixSumInstanceValid
8 direct callers.
lib.accy.src.kernel.library.catalog.family.scan.prefix.prefixSumFamilyInstance[function] — private source atlib/accy/src/kernel/library/catalog/family/scan/prefix.zig:167in nearest public ownerlib.accy.src.kernel.library.catalog.family.scan.prefixtiny.accy.kernel.library.scan.createPrefixSumFamilyArtifact[function] atlib/accy/src/kernel/library/scan.zig:903lib.accy.src.kernel.library.scan.prefixSumBody[function] — private source atlib/accy/src/kernel/library/scan.zig:193in nearest public ownertiny.accy.kernel.library.scantiny.accy.kernel.library.scan.prefixSumFamilySpecialization[function] atlib/accy/src/kernel/library/scan.zig:963tiny.accy.kernel.library.scan.prefixSumInstanceFromSpecialization[function] atlib/accy/src/kernel/library/scan.zig:989lib.accy.src.kernel.library.scan.prefixSumLaunch[function] — private source atlib/accy/src/kernel/library/scan.zig:891in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.prefixSumRuntimeBody[function] — private source atlib/accy/src/kernel/library/scan.zig:200in nearest public ownertiny.accy.kernel.library.scanlib.accy.src.kernel.library.scan.test_scan_prefix_sum_instance_validity_bounds_extent_by_threads[function] — test source atlib/accy/src/kernel/library/scan.zig:1290in nearest public ownertiny.accy.kernel.library.scan
Audit
| Definitions | 71 |
|---|---|
| Public names | 144 |
| Members | 23 |
| Version | 26.7.0 |
| Revision | daab053ee433 |