tiny.accy.kernel.library.catalog

1pub const Kind = enum {2    gelu,3    relu,4    silu,56    pub fn fromName(name: []const u8) ?Kind {7        inline for (8            @typeInfo(Kind).@"enum".field_names,9            @typeInfo(Kind).@"enum".field_values,10        ) |field_name, field_name_value| {11            const field = .{ .name = field_name, .value = field_name_value };12            if (std.mem.eql(u8, name, field.name)) return @fromBackingInt(@intCast(field.value));13        }14        return null;15    }16};

1pub const OwnedKernelCallArtifactRegistry = struct {2    allocator: std.mem.Allocator,3    artifacts: []kernel.OwnedKernelCallArtifact = &.{},4    entries: []artifact_product.KernelCallArtifact = &.{},5    index: artifact_product.KernelCallRegistryIndex = .{},67    pub fn registry(self: *const OwnedKernelCallArtifactRegistry) artifact_product.KernelCallRegistry {8        return .{9            .entries = self.entries,10            .index = self.index,11        };12    }1314    pub fn deinit(self: *OwnedKernelCallArtifactRegistry) void {15        artifact_product.deinitKernelCallRegistryIndex(self.allocator, self.index);16        for (self.artifacts) |*artifact| artifact.deinit();17        if (self.artifacts.len != 0) self.allocator.free(self.artifacts);18        if (self.entries.len != 0) self.allocator.free(self.entries);19        self.* = undefined;20    }21};

1pub const OwnedKernelCallPipelinePackage = struct {2    allocator: std.mem.Allocator,3    artifacts: []kernel.OwnedKernelCallArtifact = &.{},4    entries: []artifact_product.KernelCallArtifact = &.{},5    index: artifact_product.KernelCallRegistryIndex = .{},6    pipeline: artifact_product.OwnedKernelCallPipeline,78    pub fn registry(self: *const OwnedKernelCallPipelinePackage) artifact_product.KernelCallRegistry {9        return .{10            .entries = self.entries,11            .index = self.index,12        };13    }1415    pub fn deinit(self: *OwnedKernelCallPipelinePackage) void {16        artifact_product.deinitKernelCallRegistryIndex(self.allocator, self.index);17        for (self.artifacts) |*artifact| artifact.deinit();18        if (self.artifacts.len != 0) self.allocator.free(self.artifacts);19        if (self.entries.len != 0) self.allocator.free(self.entries);20        self.pipeline.deinit();21        self.* = undefined;22    }23};

1pub const Descriptor = struct {2    name: []const u8,3    metadata: entry.Metadata,4};

1pub const OwnedDescriptor = struct {2    descriptor: Descriptor,3    specialization: ?entry.OwnedSpecialization = null,45    pub fn deinit(self: *OwnedDescriptor) void {6        if (self.specialization) |*specialization| specialization.deinit();7        self.* = undefined;8    }9};

1pub const FilterCandidateDescriptors = struct {2    count: usize = 0,3    items: [8]OwnedDescriptor = undefined,45    pub fn slice(self: *const FilterCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *FilterCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const GatherCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const GatherCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *GatherCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const HistogramCandidateDescriptors = struct {2    count: usize = 0,3    items: [2 * geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const HistogramCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *HistogramCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const ImageCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const ImageCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *ImageCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const MatrixProductCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const MatrixProductCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *MatrixProductCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const RandomCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const RandomCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *RandomCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const PrefixSumCandidateDescriptors = struct {2    count: usize = 0,3    items: [8]OwnedDescriptor = undefined,45    pub fn slice(self: *const PrefixSumCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *PrefixSumCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const ScatterAddCandidateDescriptors = struct {2    count: usize = 0,3    items: [2 * geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const ScatterAddCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *ScatterAddCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const ScatterCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const ScatterCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *ScatterCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const SegmentSumCandidateDescriptors = struct {2    count: usize = 0,3    items: [2 * geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const SegmentSumCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *SegmentSumCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const SparseCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const SparseCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *SparseCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const SpatialCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const SpatialCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *SpatialCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const StencilWindowCandidateDescriptors = struct {2    count: usize = 0,3    items: [geometry.max_thread_candidates]OwnedDescriptor = undefined,45    pub fn slice(self: *const StencilWindowCandidateDescriptors) []const OwnedDescriptor {6        return self.items[0..self.count];7    }89    pub fn deinit(self: *StencilWindowCandidateDescriptors) void {10        for (self.items[0..self.count]) |*descriptor| descriptor.deinit();11        self.* = undefined;12    }13};

1pub const ActivationQuery = struct {2    dtype: choir_abi.DType,3    kind: ActivationKind,4    extent: u64,5};

1pub const ArtifactRequest = struct {2    target: []const u8,3    version: u32 = 1,4    options: entry.ArtifactOptions,5};

1pub const AttentionQuery = struct {2    dtype: choir_abi.DType,3    kind: AttentionKind,4    query_indices: []const u8,5    key_indices: []const u8,6    value_indices: []const u8,7    output_indices: []const u8,8    query_dims: []const i64,9    key_dims: []const i64,10    value_dims: []const i64,11    output_dims: []const i64,12    schedule: ?AttentionSchedule = null,13};

1pub const AttentionSchedule = union(enum) {2    thread_blocks: entry.Threads3D,3};

1pub const BatchedMatrixProductQuery = struct {2    dtype: choir_abi.DType,3    lhs_indices: []const u8,4    rhs_indices: []const u8,5    output_indices: []const u8,6    lhs_dims: []const i64,7    rhs_dims: []const i64,8    output_dims: []const i64,9    schedule: ?BatchedMatrixProductSchedule = null,10};

1pub const BatchedMatrixProductSchedule = union(enum) {2    thread_blocks: entry.Threads3D,3};

1pub const CompactionPredicateKind = enum {2    nonzero,3    greater_than,4};

1pub const FactorKind = union(enum) {2    batched_cholesky: struct { n: u32, batch: u64 },3    batched_cholesky_solve: struct { n: u32, batch: u64 },4    batched_inverse: struct { n: u32, batch: u64 },5};

1/// A caller fills this request to get a batched Cholesky factorization, Cholesky solve or matrix2/// inverse from the kernel library. The request names the element type, the operation with its3/// matrix size `n` and batch count, the buffer layout, and an optional thread-block count. `layout`4/// states how the matrices sit in the device buffer, row by row or interleaved across the batch.5/// Because the layout changes where each element is read, it is part of which kernel the request6/// names, and it enters the kernel's name.7pub const FactorQuery = struct {8    dtype: choir_abi.DType,9    kind: FactorKind,10    layout: FactorTileLayout = .row_major,11    schedule: ?FactorSchedule = null,12};

1pub const FactorSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const FactorTileLayout = enum {2    row_major,3    interleaved,4};

1pub const FilterQuery = struct {2    dtype: choir_abi.DType,3    predicate: CompactionPredicateKind = .nonzero,4    extent: u64,5    schedule: ?FilterSchedule = null,6};

1pub const FilterSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const FusedLinalgEpilogue = union(enum) {2    bias_activation: ActivationKind,3};

1pub const FusedMatrixProductQuery = struct {2    dtype: choir_abi.DType,3    epilogue: FusedLinalgEpilogue,4    lhs_indices: []const u8,5    rhs_indices: []const u8,6    bias_indices: []const u8,7    output_indices: []const u8,8    lhs_dims: []const i64,9    rhs_dims: []const i64,10    bias_dims: []const i64,11    output_dims: []const i64,12    schedule: ?MatrixProductSchedule = null,13};

1pub const FusedMatrixVectorProductQuery = struct {2    dtype: choir_abi.DType,3    epilogue: FusedLinalgEpilogue,4    matrix_indices: []const u8,5    vector_indices: []const u8,6    bias_indices: []const u8,7    output_indices: []const u8,8    matrix_dims: []const i64,9    vector_dims: []const i64,10    bias_dims: []const i64,11    output_dims: []const i64,12    schedule: ?MatrixVectorProductSchedule = null,13};

1pub const FusedQuery = union(enum) {2    vector: FusedVectorQuery,3    matrix_product: FusedMatrixProductQuery,4    matrix_vector_product: FusedMatrixVectorProductQuery,5    row_normalization: FusedRowNormalizationQuery,6};

1pub const FusedRowNormalizationKind = union(enum) {2    residual: RowNormalizationKind,3};

1pub const FusedRowNormalizationQuery = struct {2    dtype: choir_abi.DType,3    kind: FusedRowNormalizationKind,4    rows: u64,5    cols: u64,6    schedule: ?RowNormalizationSchedule = null,7};

1pub const FusedVectorKind = union(enum) {2    bias_activation: ActivationKind,3    gated_activation: ActivationKind,4};

1pub const FusedVectorQuery = struct {2    dtype: choir_abi.DType,3    kind: FusedVectorKind,4    extent: u64,5};

1pub const GatherQuery = struct {2    dtype: choir_abi.DType,3    outer: u64 = 1,4    axis_size: u64,5    gathered: u64,6    inner: u64 = 1,7    schedule: ?GatherSchedule = null,8};

1pub const GatherSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const HistogramQuery = struct {2    dtype: choir_abi.DType,3    bins: u64,4    count: u64,5    binning: HistogramBinningPolicy = .lower_inclusive_upper_exclusive,6    schedule: ?HistogramSchedule = null,7};

1pub const HistogramSchedule = union(enum) {2    thread_blocks: u32,3    shared_bins: u32,4};

1pub const ImageKind = union(enum) {2    blur_pass: struct {3        radius: u32 = 1,4        axis: library.image.Axis,5    },6    resize_bilinear: struct {7        src_width: u64,8        src_height: u64,9    },10};

1pub const ImageQuery = struct {2    dtype: choir_abi.DType,3    kind: ImageKind,4    width: u64,5    height: u64,6    schedule: ?ImageSchedule = null,7};

1pub const ImageSchedule = union(enum) {2    thread_blocks: entry.Threads2D,3};

1pub const LayoutQuery = struct {2    dtype: choir_abi.DType,3    kind: LayoutKind,4    input_indices: []const u8,5    output_indices: []const u8,6    input_dims: []const i64,7    output_dims: []const i64,8};

1pub const MatrixProductQuery = struct {2    dtype: choir_abi.DType,3    lhs_indices: []const u8,4    rhs_indices: []const u8,5    output_indices: []const u8,6    lhs_dims: []const i64,7    rhs_dims: []const i64,8    output_dims: []const i64,9    schedule: ?MatrixProductSchedule = null,10};

1pub const MatrixProductSchedule = union(enum) {2    thread_blocks: entry.Threads2D,3};

1pub const MatrixVectorProductQuery = struct {2    dtype: choir_abi.DType,3    matrix_indices: []const u8,4    vector_indices: []const u8,5    output_indices: []const u8,6    matrix_dims: []const i64,7    vector_dims: []const i64,8    output_dims: []const i64,9    schedule: ?MatrixVectorProductSchedule = null,10};

1pub const MatrixVectorProductSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const OuterProductQuery = struct {2    dtype: choir_abi.DType,3    lhs_indices: []const u8,4    rhs_indices: []const u8,5    output_indices: []const u8,6    lhs_dims: []const i64,7    rhs_dims: []const i64,8    output_dims: []const i64,9    schedule: ?OuterProductSchedule = null,10};

1pub const OuterProductSchedule = union(enum) {2    thread_blocks: entry.Threads2D,3};

1pub const Query = union(enum) {2    activation: ActivationQuery,3    attention: AttentionQuery,4    batched_matrix_product: BatchedMatrixProductQuery,5    fused: FusedQuery,6    gather: GatherQuery,7    layout: LayoutQuery,8    matrix_product: MatrixProductQuery,9    matrix_vector_product: MatrixVectorProductQuery,10    filter: FilterQuery,11    outer_product: OuterProductQuery,12    random: RandomQuery,13    reduction: ReductionQuery,14    row_normalization: RowNormalizationQuery,15    row_sparse_cross_entropy: RowSparseCrossEntropyQuery,16    scan: ScanQuery,17    sort: SortQuery,18    sparse: SparseQuery,19    spatial: SpatialQuery,20    image: ImageQuery,21    factor: FactorQuery,22    scatter: ScatterQuery,23    scatter_add: ScatterAddQuery,24    histogram: HistogramQuery,25    segmented: SegmentedQuery,26    stencil: StencilQuery,27};

1pub const RandomAlgorithm = enum {2    philox,3    threefry,4    squares,5};

1pub const RandomQuery = struct {2    dtype: choir_abi.DType,3    algorithm: RandomAlgorithm = .philox,4    count: u64,5    rounds: u32 = 0,6    schedule: ?RandomSchedule = null,7};

1pub const RandomSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const ReductionOperand = struct {2    indices: []const u8,3    dims: []const i64,4};

1pub const ReductionQuery = struct {2    dtype: choir_abi.DType,3    kind: ReductionKind,4    inputs: []const ReductionOperand,5    output_indices: []const u8,6    output_dims: []const i64,7};

1pub const RowNormalizationQuery = struct {2    dtype: choir_abi.DType,3    kind: RowNormalizationKind,4    rows: u64,5    cols: u64,6    schedule: ?RowNormalizationSchedule = null,7};

1pub const RowNormalizationSchedule = union(enum) {2    thread_blocks: entry.Threads2D,3};

1pub const RowSparseCrossEntropyQuery = struct {2    dtype: choir_abi.DType,3    rows: u64,4    classes: u64,5    schedule: ?RowSparseCrossEntropySchedule = null,6};

1pub const RowSparseCrossEntropySchedule = union(enum) {2    thread_blocks: u32,3};

1pub const ScanQuery = struct {2    dtype: choir_abi.DType,3    kind: ScanKind,4    extent: u64,5    schedule: ?ScanSchedule = null,6};

1pub const ScanSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const ScatterAddQuery = struct {2    dtype: choir_abi.DType,3    outer: u64 = 1,4    axis_size: u64,5    updates: u64,6    inner: u64 = 1,7    schedule: ?ScatterAddSchedule = null,8};

1pub const ScatterAddSchedule = union(enum) {2    thread_blocks: u32,3    shared_bins: u32,4};

1pub const ScatterQuery = struct {2    dtype: choir_abi.DType,3    outer: u64 = 1,4    axis_size: u64,5    updates: u64,6    inner: u64 = 1,7    schedule: ?ScatterSchedule = null,8};

1pub const ScatterSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const SegmentedQuery = struct {2    dtype: choir_abi.DType,3    kind: SegmentedKind,4    segments: u64,5    total: u64,6    schedule: ?SegmentedSchedule = null,7};

1pub const SegmentedSchedule = union(enum) {2    thread_blocks: u32,3    warp_blocks: u32,4};

1/// A caller fills this request to get a sort kernel from the kernel library, the package's2/// hand-written kernel families, chosen by a typed request. The request names the element type, the3/// sort operation, the number of elements, and `k` for the operations that keep only the first `k`4/// results. `structure` picks one way to carry out the sort, such as a radix pass or a bitonic5/// block, and `schedule` picks a thread-block size. Setting `structure` or `schedule` changes how6/// the sort is built and leaves the sort that is computed the same.7pub const SortQuery = struct {8    dtype: choir_abi.DType,9    kind: SortKind,10    extent: u64,11    k: u64 = 0,12    structure: ?SortStructure = null,13    schedule: ?SortSchedule = null,14};

1pub const SortSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const SortStructure = enum {2    radix_split,3    radix_digit,4    bitonic_block,5    top_k_block,6    top_k_block_pairs,7};

1pub const SparseKind = union(enum) {2    coo_spmv: struct { rows: u64, nnz: u64, x_extent: u64 },3    csr_spmv: struct { rows: u64, nnz: u64, x_extent: u64 },4    csr_spmm: struct { rows: u64, columns: u64, nnz: u64, x_extent: u64 },5    ell_spmv: struct { rows: u64, slots: u64, x_extent: u64 },6    sell_spmv: struct { rows: u64, slice_size: u64, values_size: u64, x_extent: u64 },7};

1pub const SparseQuery = struct {2    dtype: choir_abi.DType,3    kind: SparseKind,4    structure: ?SparseStructure = null,5    schedule: ?SparseSchedule = null,6};

1pub const SparseSchedule = union(enum) {2    thread_blocks: u32,3    thread_blocks_2d: entry.Threads2D,4};

1pub const SparseStructure = enum {2    element_thread,3    row_thread,4    row_warp,5    row_column_thread,6};

1pub const SpatialKind = union(enum) {2    grid_cells,3    grid_count: struct { cells: u32 },4    grid_neighbor_count: struct { cells: u32, stride: u32 },5};

1/// A caller fills this request to get a grid kernel for points in the plane, such as assigning2/// points to cells or counting neighbors. The request names the element type, the grid operation,3/// the number of points, and an optional thread-block count. Each operation also carries the sizes4/// fixed when the kernel is compiled, such as the cell count and, for neighbor counting, the5/// stride, beside the point count, which is given at run time.6pub const SpatialQuery = struct {7    dtype: choir_abi.DType,8    kind: SpatialKind,9    count: u64,10    schedule: ?SpatialSchedule = null,11};

1pub const SpatialSchedule = union(enum) {2    thread_blocks: u32,3};

1pub const StencilQuery = struct {2    dtype: choir_abi.DType,3    kind: StencilKind,4    rows: u64,5    cols: u64,6    radius: u32 = 1,7    schedule: ?StencilSchedule = null,8};

1pub const StencilSchedule = union(enum) {2    thread_blocks: entry.Threads2D,3};

1pub const Policy = enum(u64) {2    lower_inclusive_upper_exclusive = 0,3};

1pub fn createOwnedKernelCallArtifact(2    allocator: std.mem.Allocator,3    handle: kernel.BackendHandle,4    descriptor: OwnedDescriptor,5    options: entry.ArtifactOptions,6) !kernel.OwnedKernelCallArtifact {7    return request_mod.createKernelCallArtifact(allocator, handle, .{8        .target = descriptor.descriptor.metadata.target,9        .version = descriptor.descriptor.metadata.version,10        .options = options,11    }) catch |err| switch (err) {12        error.UnknownKernelLibraryEntry => try specialized.createSpecializedKernelCallArtifact(allocator, handle, descriptor, options),13        else => err,14    };15}

1pub fn createOwnedKernelCallArtifactRegistry(2    allocator: std.mem.Allocator,3    handle: kernel.BackendHandle,4    descriptors: []const OwnedDescriptor,5    options: entry.ArtifactOptions,6) !OwnedKernelCallArtifactRegistry {7    if (descriptors.len == 0) return .{ .allocator = allocator };89    const artifacts = try allocator.alloc(kernel.OwnedKernelCallArtifact, descriptors.len);10    var artifact_count: usize = 0;11    errdefer {12        for (artifacts[0..artifact_count]) |*artifact| artifact.deinit();13        allocator.free(artifacts);14    }1516    const entries = try allocator.alloc(artifact_product.KernelCallArtifact, descriptors.len);17    errdefer allocator.free(entries);1819    for (descriptors, 0..) |descriptor, index| {20        artifacts[index] = try createOwnedKernelCallArtifact(allocator, handle, descriptor, options);21        artifact_count += 1;22        entries[index] = artifacts[index].entry();23    }2425    const index = try artifact_product.buildKernelCallRegistryIndex(allocator, entries);26    errdefer artifact_product.deinitKernelCallRegistryIndex(allocator, index);2728    return .{29        .allocator = allocator,30        .artifacts = artifacts,31        .entries = entries,32        .index = index,33    };34}

1pub fn createOwnedKernelCallPipelinePackage(2    allocator: std.mem.Allocator,3    handle: kernel.BackendHandle,4    descriptor: OwnedDescriptor,5    options: entry.ArtifactOptions,6) !?OwnedKernelCallPipelinePackage {7    const metadata = descriptor.descriptor.metadata;8    return switch (metadata.category) {9        .scan => try createDeviceScanPipelinePackage(allocator, handle, metadata, options),10        .sort => try createRadixSortPipelinePackage(allocator, handle, metadata, options),11        else => null,12    };13}

1pub fn createKernelCallArtifact(2    allocator: std.mem.Allocator,3    handle: kernel.BackendHandle,4    request: ArtifactRequest,5) !kernel.OwnedKernelCallArtifact {6    inline for (registry.entries) |EntryType| {7        if (EntryType.version == request.version and std.mem.eql(u8, EntryType.target, request.target)) {8            return try EntryType.createKernelCallArtifact(allocator, handle, request.options);9        }10    }11    return error.UnknownKernelLibraryEntry;12}

1pub fn selectFactor(backing_allocator: std.mem.Allocator, query: FactorQuery) !?OwnedDescriptor {2    if (query.dtype != .f32) return null;3    const threads = factorQueryThreads(query) orelse return null;4    switch (query.kind) {5        .batched_cholesky => |facts| {6            const instance = factor_mod.BatchedCholesky{7                .batch = facts.batch,8                .n = facts.n,9                .threads = threads,10                .layout = factorInstanceLayout(query.layout),11            };12            if (!factor_mod.batchedCholeskyInstanceValid(instance)) return null;13            var specialization = try factor_mod.batchedCholeskyFamilySpecialization(backing_allocator, instance);14            errdefer specialization.deinit();15            const lifetime_allocator = specialization.allocator();16            const descriptor = Descriptor{17                .name = try factor_mod.batchedCholeskyFamilyEntryName(lifetime_allocator, instance),18                .metadata = .{19                    .target = try factor_mod.batchedCholeskyFamilyTarget(lifetime_allocator, instance),20                    .version = factor_mod.batched_cholesky_family_version,21                    .layer = .logical,22                    .category = .linalg,23                    .specialization = specialization.value,24                },25            };26            if (!match_mod.factorDescriptorMatches(descriptor, query)) {27                specialization.deinit();28                return null;29            }30            return .{ .descriptor = descriptor, .specialization = specialization };31        },32        .batched_cholesky_solve => |facts| {33            const instance = factor_mod.BatchedCholeskySolve{34                .batch = facts.batch,35                .n = facts.n,36                .threads = threads,37                .layout = factorInstanceLayout(query.layout),38            };39            if (!factor_mod.batchedCholeskySolveInstanceValid(instance)) return null;40            var specialization = try factor_mod.batchedCholeskySolveFamilySpecialization(backing_allocator, instance);41            errdefer specialization.deinit();42            const lifetime_allocator = specialization.allocator();43            const descriptor = Descriptor{44                .name = try factor_mod.batchedCholeskySolveFamilyEntryName(lifetime_allocator, instance),45                .metadata = .{46                    .target = try factor_mod.batchedCholeskySolveFamilyTarget(lifetime_allocator, instance),47                    .version = factor_mod.batched_cholesky_solve_family_version,48                    .layer = .logical,49                    .category = .linalg,50                    .specialization = specialization.value,51                },52            };53            if (!match_mod.factorDescriptorMatches(descriptor, query)) {54                specialization.deinit();55                return null;56            }57            return .{ .descriptor = descriptor, .specialization = specialization };58        },59        .batched_inverse => |facts| {60            const instance = factor_mod.BatchedInverse{61                .batch = facts.batch,62                .n = facts.n,63                .threads = threads,64                .layout = factorInstanceLayout(query.layout),65            };66            if (!factor_mod.batchedInverseInstanceValid(instance)) return null;67            var specialization = try factor_mod.batchedInverseFamilySpecialization(backing_allocator, instance);68            errdefer specialization.deinit();69            const lifetime_allocator = specialization.allocator();70            const descriptor = Descriptor{71                .name = try factor_mod.batchedInverseFamilyEntryName(lifetime_allocator, instance),72                .metadata = .{73                    .target = try factor_mod.batchedInverseFamilyTarget(lifetime_allocator, instance),74                    .version = factor_mod.batched_inverse_family_version,75                    .layer = .logical,76                    .category = .linalg,77                    .specialization = specialization.value,78                },79            };80            if (!match_mod.factorDescriptorMatches(descriptor, query)) {81                specialization.deinit();82                return null;83            }84            return .{ .descriptor = descriptor, .specialization = specialization };85        },86    }87}

1pub fn selectFilterCandidates(2    backing_allocator: std.mem.Allocator,3    query: FilterQuery,4) !FilterCandidateDescriptors {5    var result = FilterCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalFilter(query)) return result;9    if (query.schedule != null) {10        if (try selectFilter(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const thread_candidates = compaction.filterThreadCandidatesForExtent(query.extent);18    for (thread_candidates.slice()) |threads| {19        const instance = compaction.Filter{20            .extent = query.extent,21            .dtype = query.dtype,22            .predicate = filterInstancePredicate(query.predicate),23            .threads = threads,24        };25        if (try filterDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {26            result.items[result.count] = descriptor;27            result.count += 1;28        }29    }30    return result;31}

1pub fn selectGatherCandidates(2    backing_allocator: std.mem.Allocator,3    query: GatherQuery,4) !GatherCandidateDescriptors {5    var result = GatherCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalGather(query)) return result;9    if (query.schedule != null) {10        if (try selectGather(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const total = query.outer * query.gathered * query.inner;18    const thread_candidates = indexing.gatherThreadCandidatesForTotal(total);19    for (thread_candidates.slice()) |threads| {20        const instance = indexing.Gather{21            .outer = query.outer,22            .axis_size = query.axis_size,23            .gathered = query.gathered,24            .inner = query.inner,25            .dtype = query.dtype,26            .threads = threads,27        };28        if (try gatherDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {29            result.items[result.count] = descriptor;30            result.count += 1;31        }32    }33    return result;34}

1pub fn selectHistogramCandidates(2    backing_allocator: std.mem.Allocator,3    query: HistogramQuery,4) !HistogramCandidateDescriptors {5    var result = HistogramCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalHistogram(query)) return result;9    if (query.schedule != null) {10        if (try selectHistogram(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const thread_candidates = histogram_mod.histogramThreadCandidatesForCount(query.count);18    const variants = [_]histogram_mod.HistogramVariant{ .direct, .shared_bins };19    for (variants) |variant| {20        for (thread_candidates.slice()) |threads| {21            const instance = histogram_mod.Histogram{22                .bins = query.bins,23                .count = query.count,24                .dtype = query.dtype,25                .binning = query.binning,26                .variant = variant,27                .threads = threads,28            };29            if (!histogram_mod.histogramInstanceValid(instance)) continue;30            if (result.count >= result.items.len) break;31            if (try histogramDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {32                result.items[result.count] = descriptor;33                result.count += 1;34            }35        }36    }37    return result;38}

1pub fn selectImageCandidates(2    backing_allocator: std.mem.Allocator,3    query: ImageQuery,4) !ImageCandidateDescriptors {5    var result = ImageCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalImage(query)) return result;9    if (query.schedule != null) {10        if (try selectImage(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const thread_candidates = image.imageThreadCandidatesForExtents(query.width, query.height);18    for (thread_candidates.slice()) |threads| {19        if (result.count >= result.items.len) break;20        if (try imageDescriptorForThreads(backing_allocator, query, threads)) |descriptor| {21            result.items[result.count] = descriptor;22            result.count += 1;23        }24    }25    return result;26}

1pub fn selectMatrixProductCandidates(2    backing_allocator: std.mem.Allocator,3    query: MatrixProductQuery,4) !MatrixProductCandidateDescriptors {5    var result = MatrixProductCandidateDescriptors{};6    errdefer result.deinit();78    const accumulation_dtype = linalg.matrixProductAccumulationDType(query.dtype) orelse return result;9    if (!match_mod.canonicalMatrixProduct(query)) return result;10    const shape = match_mod.matrixProductShape(query.lhs_dims, query.rhs_dims, query.output_dims) orelse return result;11    if (query.schedule != null) {12        if (try selectMatrixProduct(backing_allocator, query)) |descriptor| {13            result.items[result.count] = descriptor;14            result.count += 1;15        }16        return result;17    }1819    const thread_candidates = linalg.matrixProductThreadCandidatesForExtents(shape.m, shape.n);20    for (thread_candidates.slice()) |threads| {21        const instance = linalg.MatrixProduct{22            .m = shape.m,23            .n = shape.n,24            .k = shape.k,25            .dtype = query.dtype,26            .accumulation_dtype = accumulation_dtype,27            .threads = threads,28        };29        if (try matrixProductDescriptorForInstance(backing_allocator, instance, shape, null, query.dtype)) |descriptor| {30            result.items[result.count] = descriptor;31            result.count += 1;32        }33    }34    return result;35}

1pub fn selectRandomCandidates(2    backing_allocator: std.mem.Allocator,3    query: RandomQuery,4) !RandomCandidateDescriptors {5    var result = RandomCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalRandom(query)) return result;9    if (query.schedule != null) {10        if (try selectRandom(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    switch (query.algorithm) {18        .philox => try philox.appendCandidates(backing_allocator, query, &result),19        .threefry => try threefry.appendCandidates(backing_allocator, query, &result),20        .squares => try squares.appendCandidates(backing_allocator, query, &result),21    }22    return result;23}

1pub fn selectPrefixSumCandidates(2    backing_allocator: std.mem.Allocator,3    query: ScanQuery,4) !PrefixSumCandidateDescriptors {5    var result = PrefixSumCandidateDescriptors{};6    errdefer result.deinit();78    if (query.schedule != null) {9        if (try selectPrefixSum(backing_allocator, query)) |descriptor| {10            result.items[result.count] = descriptor;11            result.count += 1;12        }13        return result;14    }1516    if (canonicalPrefixSum(query)) {17        const thread_candidates = scan.prefixSumThreadCandidatesForExtent(query.extent);18        for (thread_candidates.slice()) |threads| {19            const instance = scan.PrefixSum{20                .extent = query.extent,21                .dtype = query.dtype,22                .mode = scan.PrefixSumMode.fromOperation(query.kind),23                .threads = threads,24            };25            if (try prefixSumDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {26                result.items[result.count] = descriptor;27                result.count += 1;28            }29        }30        return result;31    }3233    if (canonicalDeviceScan(query)) {34        const thread_candidates = scan.deviceScanThreadCandidatesForExtent(query.extent);35        for (thread_candidates.slice()) |threads| {36            const instance = scan.DeviceScan{37                .extent = query.extent,38                .dtype = query.dtype,39                .mode = scan.PrefixSumMode.fromOperation(query.kind),40                .threads = threads,41            };42            if (try deviceScanDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {43                result.items[result.count] = descriptor;44                result.count += 1;45            }46        }47    }48    return result;49}

1pub fn selectScatterAddCandidates(2    backing_allocator: std.mem.Allocator,3    query: ScatterAddQuery,4) !ScatterAddCandidateDescriptors {5    var result = ScatterAddCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalScatterAdd(query)) return result;9    if (query.schedule != null) {10        if (try selectScatterAdd(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const query_total = scatterAddQueryTotal(query) orelse return result;18    const thread_candidates = indexing.scatterAddThreadCandidatesForTotal(query_total);19    const variants = [_]indexing.ScatterAddVariant{ .direct, .shared_bins };20    for (variants) |variant| {21        for (thread_candidates.slice()) |threads| {22            const instance = indexing.ScatterAdd{23                .outer = query.outer,24                .axis_size = query.axis_size,25                .updates = query.updates,26                .inner = query.inner,27                .dtype = query.dtype,28                .variant = variant,29                .threads = threads,30            };31            if (!indexing.scatterAddInstanceValid(instance)) continue;32            if (result.count >= result.items.len) break;33            if (try scatterAddDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {34                result.items[result.count] = descriptor;35                result.count += 1;36            }37        }38    }39    return result;40}

1pub fn selectScatterCandidates(2    backing_allocator: std.mem.Allocator,3    query: ScatterQuery,4) !ScatterCandidateDescriptors {5    var result = ScatterCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalScatter(query)) return result;9    if (query.schedule != null) {10        if (try selectScatter(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const total = query.outer * query.axis_size * query.inner;18    const thread_candidates = indexing.scatterThreadCandidatesForTotal(total);19    for (thread_candidates.slice()) |threads| {20        const instance = indexing.Scatter{21            .outer = query.outer,22            .axis_size = query.axis_size,23            .updates = query.updates,24            .inner = query.inner,25            .dtype = query.dtype,26            .threads = threads,27        };28        if (try scatterDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {29            result.items[result.count] = descriptor;30            result.count += 1;31        }32    }33    return result;34}

1pub fn selectSegmentSumCandidates(2    backing_allocator: std.mem.Allocator,3    query: SegmentedQuery,4) !SegmentSumCandidateDescriptors {5    var result = SegmentSumCandidateDescriptors{};6    errdefer result.deinit();78    if (!canonicalSegmentSum(query)) return result;9    if (query.schedule != null) {10        if (try selectSegmentSum(backing_allocator, query)) |descriptor| {11            result.items[result.count] = descriptor;12            result.count += 1;13        }14        return result;15    }1617    const thread_candidates = segmented.segmentSumThreadCandidatesForSegments(query.segments);18    for (thread_candidates.slice()) |threads| {19        const instance = segmented.SegmentSum{20            .segments = query.segments,21            .total = query.total,22            .dtype = query.dtype,23            .threads = threads,24        };25        if (try segmentSumDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {26            result.items[result.count] = descriptor;27            result.count += 1;28        }29    }3031    const warp_extent = query.segments * segmented.segment_sum_warp_size;32    const warp_candidates = segmented.segmentSumThreadCandidatesForSegments(warp_extent);33    for (warp_candidates.slice()) |threads| {34        if (!segmented.segmentSumGranularityValid(.warp, threads)) continue;35        const instance = segmented.SegmentSum{36            .segments = query.segments,37            .total = query.total,38            .dtype = query.dtype,39            .granularity = .warp,40            .threads = threads,41        };42        if (try segmentSumDescriptorForInstance(backing_allocator, instance, query)) |descriptor| {43            result.items[result.count] = descriptor;44            result.count += 1;45        }46    }47    return result;48}

1pub fn selectRadixSortWithTuning(2    backing_allocator: std.mem.Allocator,3    query: SortQuery,4    reader: tuning_mod.FamilyTuningReader,5) !?OwnedDescriptor {6    if (query.structure != null) return selectRadixSort(backing_allocator, query);7    if (query.dtype != .i32 or query.kind != .radix_ascending) return null;8    const threads = sort_mod.radixSplitThreadsForExtent(query.extent) orelse return null;9    var instance = sort_mod.RadixSplit{ .extent = query.extent, .threads = threads };10    if (query.schedule) |requested| {11        switch (requested) {12            .thread_blocks => |requested_threads| instance.threads = requested_threads,13        }14    }15    const resolved = try sort_mod.resolveRadixSplitStructure(backing_allocator, reader, instance);16    var tuned = query;17    tuned.structure = if (resolved) |structure| switch (structure) {18        .radix_split => .radix_split,19        .radix_digit => .radix_digit,20    } else null;21    return selectRadixSort(backing_allocator, tuned);22}