lib/accy/src/executable/loaded.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const gpu = @import("gpu");
3 const choir_abi = @import("choir_abi");
4 const sys = @import("sys");
5 const accy_root = @import("../root.zig");
6 const artifact_product = @import("../artifact/root.zig");
7 const binding_mod = @import("binding.zig");
8 const candidate_mod = @import("candidate.zig");
9 const exec_product = @import("plan.zig");
10 const phase_mod = @import("phase.zig");
11 const tuning_mod = @import("tuning.zig");
12
13 const ElementCountBufferBinding = binding_mod.ElementCountBufferBinding;
14 const LaunchCandidateBenchmarkOptions = candidate_mod.LaunchCandidateBenchmarkOptions;
15 const LaunchCandidateSynchronization = candidate_mod.LaunchCandidateSynchronization;
16 const LaunchGraphPlan = exec_product.LaunchGraphPlan;
17 const LaunchGraphLoop = exec_product.LaunchGraphLoop;
18 const LaunchOptions = exec_product.LaunchOptions;
19 const LaunchTuning = tuning_mod.LaunchTuning;
20 const LaunchCandidateMeasurement = tuning_mod.LaunchCandidateMeasurement;
21 const SlotBinding = binding_mod.SlotBinding;
22 const FragmentInstrumentation = phase_mod.FragmentInstrumentation;
23 const createDataflowLaunchGraphPlan = exec_product.createDataflowLaunchGraphPlan;
24 const incomingDependencyCount = exec_product.incomingDependencyCount;
25 const launchGraphNeedsDependencyEvents = exec_product.launchGraphNeedsDependencyEvents;
26
27 pub const LoadedKernel = struct {
28 kernel_id: usize,
29 work_item_id: usize,
30 loaded_artifact: gpu.LoadedArtifact,
31
32 fn deinit(self: *LoadedKernel, handle: gpu.BackendHandle) void {
33 handle.destroyObject(self.loaded_artifact.id);
34 self.* = undefined;
35 }
36 };
37
38 const max_launch_arguments = 32;
39
40 pub const LoadedKernels = struct {
41 allocator: std.mem.Allocator,
42 handle: gpu.BackendHandle,
43 kernels: []LoadedKernel,
44
45 pub fn deinit(self: *LoadedKernels) void {
46 for (self.kernels) |*kernel| kernel.deinit(self.handle);
47 self.allocator.free(self.kernels);
48 self.* = undefined;
49 }
50
51 pub fn kernelCount(self: LoadedKernels) usize {
52 return self.kernels.len;
53 }
54
55 pub fn launchKernel(
56 self: *const LoadedKernels,
57 artifact_plan: *const artifact_product.BackendArtifactPlan,
58 kernel_index: usize,
59 slot_bindings: []const SlotBinding,
60 element_count_buffer: ?gpu.BufferBinding,
61 launch_options: LaunchOptions,
62 ) gpu.BackendError!void {
63 if (kernel_index >= self.kernels.len or kernel_index >= artifact_plan.kernels.items.len) {
64 return error.InvalidArtifact;
65 }
66 const loaded = self.kernels[kernel_index];
67 const planned = artifact_plan.kernels.items[kernel_index];
68 if (loaded.kernel_id != planned.kernel_id or loaded.work_item_id != planned.work_item_id) {
69 return error.InvalidArtifact;
70 }
71
72 const has_count_buffer = planned.element_count_argument == .device_buffer_u32;
73 const buffer_count = 1 + planned.input_slot_ids.len + @intFromBool(has_count_buffer);
74 if (buffer_count > max_launch_arguments) return error.LaunchArgumentMismatch;
75 var binding_storage: [max_launch_arguments]gpu.BufferBinding = undefined;
76 const bindings = binding_storage[0..buffer_count];
77
78 var binding_index: usize = 0;
79 bindings[binding_index] = try binding_mod.bindingForSlot(slot_bindings, planned.output_slot_id, outputAccessForKernel(planned));
80 if (planned.output_fill_pattern) |pattern| {
81 try self.fillOutputBinding(artifact_plan, planned, bindings[binding_index], pattern);
82 }
83 binding_index += 1;
84 for (planned.input_slot_ids, 0..) |slot_id, input_index| {
85 const writable = planned.scratch_fill_pattern != null and input_index + 1 == planned.input_slot_ids.len;
86 bindings[binding_index] = try binding_mod.bindingForSlot(slot_bindings, slot_id, if (writable) .read_write else .read_only);
87 if (planned.scratch_fill_pattern) |pattern| {
88 if (input_index + 1 == planned.input_slot_ids.len) {
89 try self.fillSlotBinding(artifact_plan, slot_id, bindings[binding_index], pattern);
90 }
91 }
92 binding_index += 1;
93 }
94 if (has_count_buffer) {
95 bindings[binding_index] = binding_mod.countBinding(element_count_buffer orelse return error.LaunchArgumentMismatch);
96 binding_index += 1;
97 }
98
99 const runtime_scalar_count: usize = @intCast(planned.runtime_scalar_argument_count);
100 const runtime_scalar_arguments: []const choir_abi.ScalarArgument = if (runtime_scalar_count == 0)
101 &.{}
102 else if (launch_options.runtime_scalar_arguments.len != 0)
103 launch_options.runtime_scalar_arguments
104 else
105 planned.runtime_scalar_defaults;
106 if (runtime_scalar_arguments.len != runtime_scalar_count) return error.LaunchArgumentMismatch;
107 const count_scalar_count: usize = @intFromBool(planned.element_count_argument == .scalar_u32);
108 const scalar_count = count_scalar_count + runtime_scalar_count + planned.static_arguments.len;
109 if (scalar_count > max_launch_arguments) return error.LaunchArgumentMismatch;
110 var scalar_argument_storage: [max_launch_arguments]choir_abi.ScalarArgument = undefined;
111 const scalar_storage = scalar_argument_storage[0..scalar_count];
112
113 var scalar_index: usize = 0;
114 switch (planned.element_count_argument) {
115 .none, .device_buffer_u32 => {},
116 .scalar_u32 => {
117 if (planned.element_count_argument_value > std.math.maxInt(u32)) return error.LaunchArgumentMismatch;
118 scalar_storage[scalar_index] = .{ .u32 = @intCast(planned.element_count_argument_value) };
119 scalar_index += 1;
120 },
121 }
122 if (runtime_scalar_count != 0) {
123 @memcpy(scalar_storage[scalar_index..][0..runtime_scalar_count], runtime_scalar_arguments);
124 scalar_index += runtime_scalar_count;
125 }
126 if (planned.static_arguments.len != 0) {
127 @memcpy(scalar_storage[scalar_index..][0..planned.static_arguments.len], planned.static_arguments);
128 scalar_index += planned.static_arguments.len;
129 }
130
131 try self.handle.launch(.{
132 .artifact = &planned.artifact,
133 .loaded_artifact = loaded.loaded_artifact,
134 .buffers = bindings[0..binding_index],
135 .scalar_arguments = scalar_storage[0..scalar_index],
136 .geometry = try launchGeometryForPlanned(planned, runtime_scalar_arguments, launch_options.tuning),
137 .diagnostic_id = planned.artifact.diagnostic_id,
138 .stream = launch_options.stream,
139 .wait_events = launch_options.wait_events,
140 .signal_event = launch_options.signal_event,
141 });
142 }
143
144 pub fn launchKernelWithArguments(
145 self: *const LoadedKernels,
146 artifact_plan: *const artifact_product.BackendArtifactPlan,
147 kernel_index: usize,
148 buffers: []const gpu.BufferBinding,
149 scalar_arguments: []const choir_abi.ScalarArgument,
150 launch_options: LaunchOptions,
151 ) gpu.BackendError!void {
152 if (kernel_index >= self.kernels.len or kernel_index >= artifact_plan.kernels.items.len) {
153 return error.InvalidArtifact;
154 }
155 const loaded = self.kernels[kernel_index];
156 const planned = artifact_plan.kernels.items[kernel_index];
157 if (loaded.kernel_id != planned.kernel_id or loaded.work_item_id != planned.work_item_id) {
158 return error.InvalidArtifact;
159 }
160 if (planned.compile.source != .choir_kernel) return error.UnsupportedOperation;
161 const candidate_index = try launch_options.tuning.selectedCandidateIndex(planned);
162 const geometry = if (planned.launch_resources.candidate_count == 0)
163 planned.launch_resources.geometry
164 else if (candidate_index < planned.launch_resources.candidate_count)
165 planned.launch_resources.candidates[candidate_index].geometry
166 else
167 return error.LaunchArgumentMismatch;
168
169 try self.handle.launch(.{
170 .artifact = &planned.artifact,
171 .loaded_artifact = loaded.loaded_artifact,
172 .buffers = buffers,
173 .scalar_arguments = scalar_arguments,
174 .geometry = geometry,
175 .stream = launch_options.stream,
176 .wait_events = launch_options.wait_events,
177 .signal_event = launch_options.signal_event,
178 .diagnostic_id = planned.artifact.diagnostic_id,
179 });
180 }
181
182 pub fn measureLaunchCandidates(
183 self: *const LoadedKernels,
184 result_allocator: std.mem.Allocator,
185 scratch: std.mem.Allocator,
186 artifact_plan: *const artifact_product.BackendArtifactPlan,
187 kernel_index: usize,
188 slot_bindings: []const SlotBinding,
189 element_count_buffer: ?gpu.BufferBinding,
190 options: LaunchCandidateBenchmarkOptions,
191 ) gpu.BackendError![]LaunchCandidateMeasurement {
192 if (options.samples == 0) return error.LaunchArgumentMismatch;
193 if (kernel_index >= artifact_plan.kernels.items.len) return error.InvalidArtifact;
194 const planned = artifact_plan.kernels.items[kernel_index];
195 if (planned.launch_resources.candidate_count == 0) return error.LaunchArgumentMismatch;
196
197 const measurements = result_allocator.alloc(LaunchCandidateMeasurement, planned.launch_resources.candidate_count) catch return error.OutOfMemory;
198 errdefer result_allocator.free(measurements);
199
200 const sample_times = scratch.alloc(u64, options.samples) catch return error.OutOfMemory;
201 defer scratch.free(sample_times);
202
203 for (measurements, 0..) |*measurement, candidate_index| {
204 var warmup_index: u32 = 0;
205 while (warmup_index < options.warmup) : (warmup_index += 1) {
206 try self.launchKernelCandidate(
207 artifact_plan,
208 kernel_index,
209 slot_bindings,
210 element_count_buffer,
211 options.base_options,
212 planned.kernel_id,
213 candidate_index,
214 );
215 try self.synchronizeMeasuredLaunch(options.base_options, options.synchronize);
216 }
217
218 for (sample_times) |*sample| {
219 const start = nowNs();
220 try self.launchKernelCandidate(
221 artifact_plan,
222 kernel_index,
223 slot_bindings,
224 element_count_buffer,
225 options.base_options,
226 planned.kernel_id,
227 candidate_index,
228 );
229 try self.synchronizeMeasuredLaunch(options.base_options, options.synchronize);
230 sample.* = elapsedNs(start);
231 }
232
233 std.mem.sort(u64, sample_times, {}, std.sort.asc(u64));
234 measurement.* = .{
235 .kernel_id = planned.kernel_id,
236 .candidate_index = candidate_index,
237 .median_ns = sample_times[sample_times.len / 2],
238 .sample_count = options.samples,
239 };
240 }
241
242 return measurements;
243 }
244
245 fn launchKernelCandidate(
246 self: *const LoadedKernels,
247 artifact_plan: *const artifact_product.BackendArtifactPlan,
248 kernel_index: usize,
249 slot_bindings: []const SlotBinding,
250 element_count_buffer: ?gpu.BufferBinding,
251 base_options: LaunchOptions,
252 kernel_id: usize,
253 candidate_index: usize,
254 ) gpu.BackendError!void {
255 const forced_measurement = [_]LaunchCandidateMeasurement{.{
256 .kernel_id = kernel_id,
257 .candidate_index = candidate_index,
258 .median_ns = 0,
259 }};
260 var launch_options = base_options;
261 launch_options.tuning = .{ .measurements = &forced_measurement };
262 try self.launchKernel(
263 artifact_plan,
264 kernel_index,
265 slot_bindings,
266 element_count_buffer,
267 launch_options,
268 );
269 }
270
271 const max_output_fill_bytes = 4096;
272
273 fn fillOutputBinding(
274 self: *const LoadedKernels,
275 artifact_plan: *const artifact_product.BackendArtifactPlan,
276 planned: artifact_product.PlannedKernel,
277 binding: gpu.BufferBinding,
278 pattern: u32,
279 ) gpu.BackendError!void {
280 return self.fillSlotBinding(artifact_plan, planned.output_slot_id, binding, pattern);
281 }
282
283 fn fillSlotBinding(
284 self: *const LoadedKernels,
285 artifact_plan: *const artifact_product.BackendArtifactPlan,
286 slot_id: usize,
287 binding: gpu.BufferBinding,
288 pattern: u32,
289 ) gpu.BackendError!void {
290 const slot = artifact_plan.slotById(slot_id) orelse return error.InvalidArtifact;
291 const byte_size_u64 = slot.byte_size orelse return error.UnsupportedOperation;
292 if (byte_size_u64 == 0 or byte_size_u64 % @sizeOf(u32) != 0) return error.InvalidArtifact;
293 if (self.handle.fillBuffer(.{ .handle = binding.handle, .pattern = pattern })) |_| {
294 return;
295 } else |err| switch (err) {
296 error.UnsupportedOperation => {},
297 else => return err,
298 }
299 if (byte_size_u64 > max_output_fill_bytes) return error.UnsupportedOperation;
300 const byte_size: usize = @intCast(byte_size_u64);
301
302 var storage: [max_output_fill_bytes]u8 = undefined;
303 const pattern_bytes = std.mem.asBytes(&pattern);
304 var offset: usize = 0;
305 while (offset < byte_size) : (offset += @sizeOf(u32)) {
306 @memcpy(storage[offset..][0..@sizeOf(u32)], pattern_bytes);
307 }
308 try self.handle.writeBuffer(.{
309 .handle = binding.handle,
310 .bytes = storage[0..byte_size],
311 });
312 }
313
314 fn synchronizeMeasuredLaunch(
315 self: *const LoadedKernels,
316 launch_options: LaunchOptions,
317 synchronization: LaunchCandidateSynchronization,
318 ) gpu.BackendError!void {
319 switch (synchronization) {
320 .none => {},
321 .device => try self.handle.synchronize(.{ .scope = .device }),
322 .stream => try self.handle.synchronize(.{
323 .scope = .stream,
324 .stream = launch_options.stream orelse return error.LaunchArgumentMismatch,
325 }),
326 .event => try self.handle.synchronize(.{
327 .scope = .event,
328 .event = launch_options.signal_event orelse return error.LaunchArgumentMismatch,
329 }),
330 }
331 }
332
333 pub fn launchAll(
334 self: *const LoadedKernels,
335 scratch: std.mem.Allocator,
336 artifact_plan: *const artifact_product.BackendArtifactPlan,
337 slot_bindings: []const SlotBinding,
338 element_count_buffers: []const ElementCountBufferBinding,
339 ) gpu.BackendError!void {
340 try self.launchAllWithOptions(
341 scratch,
342 artifact_plan,
343 slot_bindings,
344 element_count_buffers,
345 .{},
346 );
347 }
348
349 pub fn launchAllWithOptions(
350 self: *const LoadedKernels,
351 scratch: std.mem.Allocator,
352 artifact_plan: *const artifact_product.BackendArtifactPlan,
353 slot_bindings: []const SlotBinding,
354 element_count_buffers: []const ElementCountBufferBinding,
355 launch_options: LaunchOptions,
356 ) gpu.BackendError!void {
357 if (self.kernels.len != artifact_plan.kernels.items.len) return error.InvalidArtifact;
358 var graph = try createDataflowLaunchGraphPlan(scratch, artifact_plan, launch_options);
359 defer graph.deinit();
360 try self.launchGraph(
361 scratch,
362 artifact_plan,
363 slot_bindings,
364 element_count_buffers,
365 graph.plan(),
366 );
367 }
368
369 pub fn launchGraph(
370 self: *const LoadedKernels,
371 scratch: std.mem.Allocator,
372 artifact_plan: *const artifact_product.BackendArtifactPlan,
373 slot_bindings: []const SlotBinding,
374 element_count_buffers: []const ElementCountBufferBinding,
375 graph: LaunchGraphPlan,
376 ) gpu.BackendError!void {
377 if (self.kernels.len != artifact_plan.kernels.items.len) return error.InvalidArtifact;
378 if (launchGraphNeedsDependencyEvents(graph)) {
379 if (graph.loops.len != 0) return error.UnsupportedOperation;
380 try self.launchGraphWithDependencyEvents(
381 scratch,
382 artifact_plan,
383 slot_bindings,
384 element_count_buffers,
385 graph,
386 );
387 return;
388 }
389 try self.launchGraphHostOrder(
390 scratch,
391 artifact_plan,
392 slot_bindings,
393 element_count_buffers,
394 graph,
395 );
396 }
397
398 fn launchGraphHostOrder(
399 self: *const LoadedKernels,
400 scratch: std.mem.Allocator,
401 artifact_plan: *const artifact_product.BackendArtifactPlan,
402 slot_bindings: []const SlotBinding,
403 element_count_buffers: []const ElementCountBufferBinding,
404 graph: LaunchGraphPlan,
405 ) gpu.BackendError!void {
406 if (!graph.validated) try exec_product.validateLaunchGraph(scratch, artifact_plan, graph, false);
407
408 var node_index: usize = 0;
409 while (node_index < graph.nodes.len) {
410 if (loopAtNode(graph.loops, node_index)) |loop_desc| {
411 try self.launchGraphLoopHostOrder(
412 scratch,
413 artifact_plan,
414 slot_bindings,
415 element_count_buffers,
416 graph,
417 loop_desc,
418 );
419 node_index += loop_desc.node_count;
420 continue;
421 }
422 try self.launchGraphNodeHostOrder(
423 artifact_plan,
424 slot_bindings,
425 element_count_buffers,
426 graph.nodes[node_index],
427 );
428 node_index += 1;
429 }
430 }
431
432 fn launchGraphLoopHostOrder(
433 self: *const LoadedKernels,
434 scratch: std.mem.Allocator,
435 artifact_plan: *const artifact_product.BackendArtifactPlan,
436 slot_bindings: []const SlotBinding,
437 element_count_buffers: []const ElementCountBufferBinding,
438 graph: LaunchGraphPlan,
439 loop_desc: LaunchGraphLoop,
440 ) gpu.BackendError!void {
441 if (loop_desc.trip_count == 0) return;
442 const loop_end = loop_desc.first_node_index + loop_desc.node_count;
443 if (loop_desc.carries.len == 0) {
444 var iteration: u64 = 0;
445 while (iteration < loop_desc.trip_count) : (iteration += 1) {
446 for (graph.nodes[loop_desc.first_node_index..loop_end]) |node| {
447 try self.launchGraphNodeHostOrder(
448 artifact_plan,
449 slot_bindings,
450 element_count_buffers,
451 node,
452 );
453 }
454 }
455 return;
456 }
457
458 const remapped_slot_bindings = scratch.alloc(SlotBinding, slot_bindings.len) catch return error.OutOfMemory;
459 defer scratch.free(remapped_slot_bindings);
460
461 var iteration: u64 = 0;
462 while (iteration < loop_desc.trip_count) : (iteration += 1) {
463 @memcpy(remapped_slot_bindings, slot_bindings);
464 for (loop_desc.carries) |carry| {
465 const input_physical_slot = if (iteration % 2 == 0) carry.initial_slot_id else carry.output_slot_id;
466 const output_physical_slot = if (iteration % 2 == 0) carry.output_slot_id else carry.initial_slot_id;
467 const input_binding = try binding_mod.bindingForSlot(slot_bindings, input_physical_slot, .read_write);
468 const output_binding = try binding_mod.bindingForSlot(slot_bindings, output_physical_slot, .read_write);
469 try remapSlotBinding(remapped_slot_bindings, carry.input_slot_id, input_binding);
470 try remapSlotBinding(remapped_slot_bindings, carry.output_slot_id, output_binding);
471 }
472 for (graph.nodes[loop_desc.first_node_index..loop_end]) |node| {
473 try self.launchGraphNodeHostOrder(
474 artifact_plan,
475 remapped_slot_bindings,
476 element_count_buffers,
477 node,
478 );
479 }
480 }
481 }
482
483 fn launchGraphNodeHostOrder(
484 self: *const LoadedKernels,
485 artifact_plan: *const artifact_product.BackendArtifactPlan,
486 slot_bindings: []const SlotBinding,
487 element_count_buffers: []const ElementCountBufferBinding,
488 node: exec_product.LaunchGraphNode,
489 ) gpu.BackendError!void {
490 const kernel = self.kernels[node.kernel_index];
491 try self.launchKernel(
492 artifact_plan,
493 node.kernel_index,
494 slot_bindings,
495 binding_mod.elementCountBufferForKernel(element_count_buffers, kernel.kernel_id),
496 .{
497 .stream = node.stream,
498 .wait_events = node.wait_events,
499 .signal_event = node.signal_event,
500 .tuning = node.tuning,
501 .runtime_scalar_arguments = node.runtime_scalar_arguments,
502 },
503 );
504 }
505
506 pub fn launchGraphWithDependencyEvents(
507 self: *const LoadedKernels,
508 scratch: std.mem.Allocator,
509 artifact_plan: *const artifact_product.BackendArtifactPlan,
510 slot_bindings: []const SlotBinding,
511 element_count_buffers: []const ElementCountBufferBinding,
512 graph: LaunchGraphPlan,
513 ) gpu.BackendError!void {
514 if (graph.loops.len != 0) return error.UnsupportedOperation;
515 if (!graph.validated) try exec_product.validateLaunchGraph(scratch, artifact_plan, graph, true);
516
517 const dependency_events = scratch.alloc(gpu.EventHandle, graph.dependencies.len) catch return error.OutOfMemory;
518 defer scratch.free(dependency_events);
519 var created_event_count: usize = 0;
520 errdefer {
521 for (dependency_events[0..created_event_count]) |event| self.handle.destroyObject(event.id);
522 }
523 for (graph.dependencies, 0..) |_, index| {
524 dependency_events[index] = try self.handle.createEvent(.{});
525 created_event_count += 1;
526 }
527 defer {
528 for (dependency_events[0..created_event_count]) |event| self.handle.destroyObject(event.id);
529 }
530
531 for (graph.nodes, 0..) |node, node_index| {
532 const incoming_count = incomingDependencyCount(graph, node_index);
533 const wait_events = scratch.alloc(gpu.EventHandle, node.wait_events.len + incoming_count) catch return error.OutOfMemory;
534 defer scratch.free(wait_events);
535 @memcpy(wait_events[0..node.wait_events.len], node.wait_events);
536 var wait_index = node.wait_events.len;
537 for (graph.dependencies, 0..) |dependency, dependency_index| {
538 if (dependency.consumer_node_index != node_index) continue;
539 wait_events[wait_index] = dependency_events[dependency_index];
540 wait_index += 1;
541 }
542
543 const kernel = self.kernels[node.kernel_index];
544 try self.launchKernel(
545 artifact_plan,
546 node.kernel_index,
547 slot_bindings,
548 binding_mod.elementCountBufferForKernel(element_count_buffers, kernel.kernel_id),
549 .{
550 .stream = node.stream,
551 .wait_events = wait_events,
552 .signal_event = node.signal_event,
553 .tuning = node.tuning,
554 .runtime_scalar_arguments = node.runtime_scalar_arguments,
555 },
556 );
557
558 for (graph.dependencies, 0..) |dependency, dependency_index| {
559 if (dependency.producer_node_index != node_index) continue;
560 try self.handle.recordEvent(.{
561 .stream = node.stream.?,
562 .event = dependency_events[dependency_index],
563 });
564 }
565 }
566 }
567 };
568
569 fn loopAtNode(loops: []const LaunchGraphLoop, node_index: usize) ?LaunchGraphLoop {
570 for (loops) |loop_desc| {
571 if (loop_desc.first_node_index == node_index) return loop_desc;
572 }
573 return null;
574 }
575
576 fn remapSlotBinding(
577 slot_bindings: []SlotBinding,
578 slot_id: usize,
579 binding: gpu.BufferBinding,
580 ) gpu.BackendError!void {
581 for (slot_bindings) |*slot_binding| {
582 if (slot_binding.slot_id != slot_id) continue;
583 slot_binding.binding = binding;
584 return;
585 }
586 return error.InvalidBuffer;
587 }
588
589 pub fn loadKernels(
590 allocator: std.mem.Allocator,
591 handle: gpu.BackendHandle,
592 artifact_plan: *const artifact_product.BackendArtifactPlan,
593 ) !LoadedKernels {
594 return try loadKernelsWithInstrumentation(allocator, handle, artifact_plan, .{});
595 }
596
597 pub fn loadKernelsWithInstrumentation(
598 allocator: std.mem.Allocator,
599 handle: gpu.BackendHandle,
600 artifact_plan: *const artifact_product.BackendArtifactPlan,
601 instrumentation: FragmentInstrumentation,
602 ) !LoadedKernels {
603 const load_start = nowNs();
604 const kernels = allocator.alloc(LoadedKernel, artifact_plan.kernels.items.len) catch return error.OutOfMemory;
605 var loaded_count: usize = 0;
606 errdefer {
607 for (kernels[0..loaded_count]) |*kernel| kernel.deinit(handle);
608 allocator.free(kernels);
609 }
610
611 for (artifact_plan.kernels.items, 0..) |planned, index| {
612 const loaded = try handle.loadArtifact(&planned.artifact);
613 kernels[index] = .{
614 .kernel_id = planned.kernel_id,
615 .work_item_id = planned.work_item_id,
616 .loaded_artifact = loaded,
617 };
618 loaded_count += 1;
619 }
620
621 try instrumentation.record(.load_backend_artifacts, load_start);
622 return .{
623 .allocator = allocator,
624 .handle = handle,
625 .kernels = kernels,
626 };
627 }
628
629 fn outputAccessForKernel(planned: artifact_product.PlannedKernel) gpu.BufferAccess {
630 for (planned.input_slot_ids) |slot_id| {
631 if (slot_id == planned.output_slot_id) return .read_write;
632 }
633 return .write_only;
634 }
635
636 fn launchGeometryForPlanned(
637 planned: artifact_product.PlannedKernel,
638 runtime_scalar_arguments: []const choir_abi.ScalarArgument,
639 tuning: LaunchTuning,
640 ) gpu.BackendError!choir_abi.LaunchGeometry {
641 if (planned.kernel_call_launch) |launch| {
642 switch (launch) {
643 .derived => |derived| return derived.geometry(runtime_scalar_arguments),
644 .fixed => {},
645 }
646 }
647 const candidate_index = try tuning.selectedCandidateIndex(planned);
648 if (candidate_index >= planned.launch_resources.candidate_count) return error.LaunchArgumentMismatch;
649 return planned.launch_resources.candidates[candidate_index].geometry;
650 }
651
652 fn nowNs() i128 {
653 return sys.time.nanoTimestamp();
654 }
655
656 fn elapsedNs(start: i128) u64 {
657 const end = nowNs();
658 if (end <= start) return 0;
659 const elapsed = end - start;
660 if (elapsed > std.math.maxInt(u64)) return std.math.maxInt(u64);
661 return @intCast(elapsed);
662 }