lib/choir/src/backends/x64/jit.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

   1 const std = @import("std");
   2 const alloc_arena = @import("alloc_arena");
   3 const ir = @import("../../core/root.zig");
   4 const capacity = @import("capacity.zig");
   5 const emit = @import("emit.zig");
   6 const failures = @import("failures.zig");
   7 const abi = @import("abi.zig");
   8 const invoke = @import("invoke.zig");
   9 const dialects = @import("../../dialects/root.zig");
  10 const backends = @import("../root.zig");
  11 const artifact = backends.artifact;
  12 const debug_info = backends.debug_info;
  13 const machine = backends.machine_code;
  14 const sys = @import("sys");
  15 const diagnostics = @import("../../root.zig").diagnostics;
  16 
  17 const Mapping = []align(std.heap.page_size_min) u8;
  18 
  19 comptime {
  20     std.debug.assert(machine.max_data_alignment <= std.heap.page_size_min);
  21 }
  22 
  23 /// The bytes the kernel reserves for a mapping of `byte_count`. `sys.memory.mapAnonymous` rounds
  24 /// a nonzero request up to a page, and zero reserves nothing because `link` skips the call rather
  25 /// than mapping a page for no data. A request too large to page-align saturates to the whole
  26 /// address space. A budget short of that refuses the saturated charge in `admitMapping`, and a
  27 /// budget equal to it admits the charge and lets the mapping call fail.
  28 fn mappedSize(byte_count: usize) u64 {
  29     if (byte_count == 0) return 0;
  30     const aligned = sys.memory.pageAlign(byte_count) orelse return std.math.maxInt(u64);
  31     return aligned;
  32 }
  33 
  34 /// Names one module compiled or loaded into a `JitRuntime`.
  35 /// Releasing the module retires its generation, so the handle stops resolving.
  36 pub const ModuleHandle = struct {
  37     index: u32,
  38     generation: u32,
  39 };
  40 
  41 pub const HandleError = error{InvalidModuleHandle};
  42 pub const LookupError = HandleError || error{FunctionNotFound};
  43 pub const CallError = LookupError || invoke.Error;
  44 
  45 pub const LoadError = error{ InvalidArtifact, SymbolNotFound, JitRuntimeFull };
  46 pub const MachineCodeArtifactError = std.mem.Allocator.Error ||
  47     sys.memory.MapError ||
  48     sys.memory.ProtectError ||
  49     LoadError;
  50 
  51 const FunctionEntry = struct {
  52     offset: usize,
  53     size: usize,
  54     signature: artifact.Signature,
  55     line_table: ?debug_info.LineTable = null,
  56     debug_object: ?debug_info.JitDebugHandle = null,
  57 
  58     fn deinit(self: *FunctionEntry, allocator: std.mem.Allocator) void {
  59         if (self.debug_object) |*handle| handle.deinit(allocator);
  60         if (self.line_table) |*table| table.deinit();
  61         self.* = undefined;
  62     }
  63 };
  64 
  65 const FoundFunction = struct {
  66     address: usize,
  67     entry: *const FunctionEntry,
  68 };
  69 
  70 const Module = struct {
  71     generation: u32,
  72     live: bool,
  73     code: ?Mapping,
  74     /// Read-only bytes that the code addresses through absolute relocations.
  75     data: ?Mapping,
  76     functions: std.StringHashMapUnmanaged(FunctionEntry),
  77 
  78     const staged: Module = .{
  79         .generation = 0,
  80         .live = true,
  81         .code = null,
  82         .data = null,
  83         .functions = .empty,
  84     };
  85 
  86     fn functionAddress(self: *const Module, entry: *const FunctionEntry) usize {
  87         const code = self.code.?;
  88         std.debug.assert(entry.offset + entry.size <= code.len);
  89         return @intFromPtr(code.ptr) + entry.offset;
  90     }
  91 
  92     /// The bytes this module holds mapped, code and read-only data together. `unload` unmaps
  93     /// exactly those bytes, so `release` credits back the figure `install` charged.
  94     fn mappedBytes(self: *const Module) u64 {
  95         var total: u64 = 0;
  96         if (self.code) |mapping| total += mapping.len;
  97         if (self.data) |mapping| total += mapping.len;
  98         return total;
  99     }
 100 
 101     /// Unregisters debug objects before unmapping the code they describe, then unmaps the data.
 102     /// Keeps the generation.
 103     fn unload(self: *Module, allocator: std.mem.Allocator) void {
 104         std.debug.assert(self.live);
 105         var iterator = self.functions.iterator();
 106         while (iterator.next()) |entry| {
 107             entry.value_ptr.deinit(allocator);
 108             allocator.free(entry.key_ptr.*);
 109         }
 110         self.functions.deinit(allocator);
 111         if (self.code) |mapping| sys.memory.unmap(mapping);
 112         if (self.data) |mapping| sys.memory.unmap(mapping);
 113         self.* = .{
 114             .generation = self.generation,
 115             .live = false,
 116             .code = null,
 117             .data = null,
 118             .functions = .empty,
 119         };
 120     }
 121 };
 122 
 123 /// One module's code and data symbols, with every relocation still unpatched.
 124 const Image = struct {
 125     code: []const u8,
 126     calls: []const emit.CallRelocation,
 127     data: *const machine.DataSymbolSet,
 128     data_relocations: []const machine.DataRelocation,
 129 };
 130 
 131 /// Concatenates emitted functions into one image.
 132 const ImageBuilder = struct {
 133     code: std.ArrayListUnmanaged(u8) = .empty,
 134     calls: std.ArrayListUnmanaged(emit.CallRelocation) = .empty,
 135     data: machine.DataSymbolSet = .{},
 136     data_relocations: std.ArrayListUnmanaged(machine.DataRelocation) = .empty,
 137 
 138     fn deinit(self: *ImageBuilder, allocator: std.mem.Allocator) void {
 139         self.code.deinit(allocator);
 140         self.calls.deinit(allocator);
 141         self.data.deinit(allocator);
 142         self.data_relocations.deinit(allocator);
 143         self.* = undefined;
 144     }
 145 
 146     /// Appends the emitter's current function, rebases its relocations, and merges its data
 147     /// symbols. Returns the function's offset in the image.
 148     fn append(
 149         self: *ImageBuilder,
 150         allocator: std.mem.Allocator,
 151         emitter: *const emit.Emitter,
 152     ) (std.mem.Allocator.Error || machine.DataSymbolError)!usize {
 153         const code = emitter.getCode();
 154         std.debug.assert(code.len != 0);
 155         const offset = self.code.items.len;
 156         try self.code.appendSlice(allocator, code);
 157         for (emitter.call_relocations.items) |relocation| {
 158             try self.calls.append(allocator, .{
 159                 .offset = offset + relocation.offset,
 160                 .target = relocation.target,
 161             });
 162         }
 163         for (emitter.data_symbols.items()) |symbol| try self.data.put(allocator, symbol);
 164         for (emitter.data_relocations.items) |relocation| {
 165             var rebased = relocation;
 166             rebased.offset += offset;
 167             try self.data_relocations.append(allocator, rebased);
 168         }
 169         std.debug.assert(self.code.items.len == offset + code.len);
 170         return offset;
 171     }
 172 
 173     fn image(self: *const ImageBuilder) Image {
 174         return .{
 175             .code = self.code.items,
 176             .calls = self.calls.items,
 177             .data = &self.data,
 178             .data_relocations = self.data_relocations.items,
 179         };
 180     }
 181 };
 182 
 183 /// Executable modules addressed by generation-checked handles.
 184 /// Each module owns its code and data until `release`. Compiling or loading another module never
 185 /// moves or frees them.
 186 pub const JitRuntime = struct {
 187     allocator: std.mem.Allocator,
 188     limits: Limits,
 189     live_modules: u32,
 190     /// The bytes live modules hold mapped, as the kernel reserved them. The running total moves
 191     /// alongside `live_modules`, at `install` and `release`. A compile that fails partway spends
 192     /// none of the budget, because the module never reaches `install` and its `errdefer` unmaps
 193     /// whatever it had mapped.
 194     mapped_bytes: u64,
 195     modules: std.ArrayListUnmanaged(Module),
 196     external_symbols: std.StringHashMapUnmanaged(usize),
 197     emitter: emit.Emitter,
 198 
 199     pub const Limits: type = capacity.Limits;
 200 
 201     pub fn init(allocator: std.mem.Allocator, limits: Limits) JitRuntime {
 202         std.debug.assert(limits.live_modules != 0);
 203         std.debug.assert(limits.mapped_bytes != 0);
 204         return .{
 205             .allocator = allocator,
 206             .limits = limits,
 207             .live_modules = 0,
 208             .mapped_bytes = 0,
 209             .modules = .empty,
 210             .external_symbols = .empty,
 211             .emitter = emit.Emitter.init(allocator),
 212         };
 213     }
 214 
 215     pub fn deinit(self: *JitRuntime) void {
 216         for (self.modules.items) |*module| {
 217             if (module.live) module.unload(self.allocator);
 218         }
 219         self.modules.deinit(self.allocator);
 220 
 221         var names = self.external_symbols.keyIterator();
 222         while (names.next()) |name| self.allocator.free(name.*);
 223         self.external_symbols.deinit(self.allocator);
 224 
 225         self.emitter.deinit();
 226         self.* = undefined;
 227     }
 228 
 229     /// Compiles every function definition in `module` into a new module. Calls resolve against the
 230     /// module's own functions first, then registered external symbols. Data symbols resolve against
 231     /// the module's own read-only data. Fails with `error.ConflictingDataSymbol` when two constants
 232     /// give one data symbol name different bytes or alignment.
 233     /// Reports one error diagnostic when a function fails to compile, located at the innermost
 234     /// operation whose emission failed, or otherwise at the function.
 235     /// Fails with `error.SymbolNotFound` when neither the module nor a registered external symbol
 236     /// defines a call's target, and reports one error diagnostic at the first `func.call` naming
 237     /// the target, or at the calling function when no `func.call` names it.
 238     /// Fails with `error.OutOfMemory` when an allocation fails, and reports nothing.
 239     /// Fails with `error.JitRuntimeFull`, and reports nothing, when the runtime already holds
 240     /// `limits.live_modules` modules or when this module's pages would carry it past
 241     /// `limits.mapped_bytes`. A `release` admits the next module under either bound.
 242     /// A failed compile leaves every existing module untouched and spends none of either budget.
 243     pub fn compile(self: *JitRuntime, module: *ir.Operation) !ModuleHandle {
 244         const region = module.getRegion(0) orelse return error.NoModuleBody;
 245         const block = region.getEntryBlock() orelse return error.NoEntryBlock;
 246         const index = try self.reserveModule();
 247 
 248         var staged: Module = .staged;
 249         errdefer staged.unload(self.allocator);
 250         var builder: ImageBuilder = .{};
 251         defer builder.deinit(self.allocator);
 252 
 253         var op_iter = block.operations.head;
 254         while (op_iter) |op_ptr| {
 255             const op: *ir.Operation = @ptrCast(@alignCast(op_ptr));
 256             op_iter = op.next_op;
 257             if (!std.mem.eql(u8, op.name.name, dialects.FuncDialect.FuncOp.operation_name)) continue;
 258             if (op.getRegion(0) == null) continue;
 259             try self.emitFunction(&staged, &builder, op);
 260         }
 261         if (builder.code.items.len != 0) {
 262             try self.resolveCalls(module, &staged, builder.calls.items);
 263             try self.link(&staged, builder.image());
 264         }
 265         return self.install(index, staged);
 266     }
 267 
 268     /// Emits one function into the module image. Relocations stay unpatched until `link`.
 269     fn emitFunction(
 270         self: *JitRuntime,
 271         staged: *Module,
 272         builder: *ImageBuilder,
 273         func: *ir.Operation,
 274     ) !void {
 275         defer self.emitter.discardFunctionState();
 276         self.stageFunction(staged, builder, func) catch |err| {
 277             failures.Failure.report(.{
 278                 .stage = .emit,
 279                 .operation = self.emitter.failed_operation orelse func,
 280                 .err = err,
 281             });
 282             return err;
 283         };
 284     }
 285 
 286     fn stageFunction(
 287         self: *JitRuntime,
 288         staged: *Module,
 289         builder: *ImageBuilder,
 290         func: *ir.Operation,
 291     ) !void {
 292         const name = ir.SymbolTable.getSymbolName(func) orelse return error.NoFunctionName;
 293         if (staged.functions.contains(name)) return error.DuplicateSymbol;
 294         const signature = try signatureOf(func);
 295 
 296         try self.emitter.emitFunction(func);
 297         const size = self.emitter.getCode().len;
 298         if (size == 0) return error.EmptyFunction;
 299         const offset = try builder.append(self.allocator, &self.emitter);
 300 
 301         try staged.functions.ensureUnusedCapacity(self.allocator, 1);
 302         const owned_name = try self.allocator.dupe(u8, name);
 303         errdefer self.allocator.free(owned_name);
 304         var entry = FunctionEntry{ .offset = offset, .size = size, .signature = signature };
 305         errdefer entry.deinit(self.allocator);
 306         entry.line_table = try self.emitter.takeLineTable();
 307         staged.functions.putAssumeCapacityNoClobber(owned_name, entry);
 308     }
 309 
 310     fn resolveCalls(
 311         self: *const JitRuntime,
 312         module: *ir.Operation,
 313         staged: *const Module,
 314         calls: []const emit.CallRelocation,
 315     ) error{SymbolNotFound}!void {
 316         for (calls) |call_site| {
 317             if (staged.functions.contains(call_site.target)) continue;
 318             if (self.externalAddress(call_site.target) != null) continue;
 319             failures.Failure.report(.{
 320                 .stage = .link,
 321                 .operation = callerOf(module, staged, call_site),
 322                 .err = error.SymbolNotFound,
 323                 .symbol = call_site.target,
 324             });
 325             return error.SymbolNotFound;
 326         }
 327     }
 328 
 329     /// Maps the image's data writable, copies it, seals it as its symbols require, then maps
 330     /// the code writable, patches every call and data address, and seals the code executable.
 331     ///
 332     /// A module's data is one mapping, so one writable global leaves the whole mapping
 333     /// writable, read only symbols included. An object file places each section on its own and
 334     /// carries no such weakening.
 335     fn link(self: *JitRuntime, staged: *Module, image: Image) !void {
 336         std.debug.assert(staged.code == null);
 337         std.debug.assert(staged.data == null);
 338         std.debug.assert(image.code.len != 0);
 339         const symbols = image.data.items();
 340         var layout = machine.DataLayout.init(self.allocator, symbols) catch |err| switch (err) {
 341             error.InvalidDataSymbol => unreachable,
 342             else => |narrow| return narrow,
 343         };
 344         defer layout.deinit(self.allocator);
 345         try self.admitMapping(mappedSize(image.code.len) +| mappedSize(layout.size));
 346 
 347         const writable: sys.memory.Protection = .{ .read = true, .write = true };
 348         if (layout.size != 0) {
 349             const data = try sys.memory.mapAnonymous(layout.size, writable);
 350             std.debug.assert(data.len == mappedSize(layout.size));
 351             staged.data = data;
 352             layout.write(symbols, data);
 353             if (!anyWritable(symbols)) try sys.memory.protect(data, .{ .read = true });
 354         }
 355 
 356         const mapping = try sys.memory.mapAnonymous(image.code.len, writable);
 357         std.debug.assert(mapping.len == mappedSize(image.code.len));
 358         staged.code = mapping;
 359         const code = mapping[0..image.code.len];
 360         @memcpy(code, image.code);
 361         for (image.calls) |call_site| {
 362             const target = if (staged.functions.getPtr(call_site.target)) |callee|
 363                 staged.functionAddress(callee)
 364             else
 365                 self.externalAddress(call_site.target) orelse return error.SymbolNotFound;
 366             writeAddress(code, call_site.offset, target);
 367         }
 368         for (image.data_relocations) |relocation| {
 369             std.debug.assert(relocation.width_bits == 64);
 370             const symbol = image.data.indexOf(relocation.target).?;
 371             const base: u64 = @intFromPtr(staged.data.?.ptr) + layout.offsets[symbol];
 372             const addend: u64 = @bitCast(relocation.addend);
 373             writeAddress(code, relocation.offset, base +% addend);
 374         }
 375         try sys.memory.protect(mapping, .{ .read = true, .execute = true });
 376     }
 377 
 378     /// Reports whether any symbol asks for storage the code may write.
 379     fn anyWritable(symbols: []const machine.DataSymbol) bool {
 380         for (symbols) |symbol| {
 381             if (symbol.section != .rodata) return true;
 382         }
 383         return false;
 384     }
 385 
 386     /// Reserves a slot so that installing a finished module cannot fail.
 387     fn reserveModule(self: *JitRuntime) error{ OutOfMemory, JitRuntimeFull }!u32 {
 388         std.debug.assert(self.live_modules <= self.limits.live_modules);
 389         if (self.live_modules >= self.limits.live_modules) return error.JitRuntimeFull;
 390         for (self.modules.items, 0..) |*module, index| {
 391             if (module.live) continue;
 392             if (module.generation == std.math.maxInt(u32)) continue;
 393             return @intCast(index);
 394         }
 395         const index = std.math.cast(u32, self.modules.items.len) orelse return error.JitRuntimeFull;
 396         try self.modules.ensureUnusedCapacity(self.allocator, 1);
 397         return index;
 398     }
 399 
 400     /// Refuses `charge` bytes before the runtime maps code or data. `reserveModule` checks the
 401     /// slot count at the top of a compile, but a module's size is not known until its code is
 402     /// emitted, so `link` checks the byte budget instead. Both checks answer the same capacity
 403     /// question and fail with `error.JitRuntimeFull`.
 404     ///
 405     /// The sum saturates rather than the budget being subtracted from, so a runtime already over
 406     /// its budget refuses instead of wrapping. A host reaches that state by lowering
 407     /// `limits.mapped_bytes` under what is already live, and the slot count answers the same
 408     /// move the same way.
 409     fn admitMapping(self: *const JitRuntime, charge: u64) error{JitRuntimeFull}!void {
 410         if (self.mapped_bytes +| charge > self.limits.mapped_bytes) return error.JitRuntimeFull;
 411     }
 412 
 413     fn install(self: *JitRuntime, index: u32, staged: Module) ModuleHandle {
 414         std.debug.assert(staged.live);
 415         std.debug.assert(self.live_modules < self.limits.live_modules);
 416         self.live_modules += 1;
 417         const charge = staged.mappedBytes();
 418         std.debug.assert(self.mapped_bytes +| charge <= self.limits.mapped_bytes);
 419         self.mapped_bytes += charge;
 420         var module = staged;
 421         if (index == self.modules.items.len) {
 422             module.generation = 1;
 423             self.modules.appendAssumeCapacity(module);
 424         } else {
 425             const slot = &self.modules.items[index];
 426             std.debug.assert(!slot.live);
 427             module.generation = slot.generation + 1;
 428             slot.* = module;
 429         }
 430         return .{ .index = index, .generation = module.generation };
 431     }
 432 
 433     /// Unmaps the module's code and data. Function pointers and data addresses taken from it dangle
 434     /// afterward.
 435     pub fn release(self: *JitRuntime, handle: ModuleHandle) HandleError!void {
 436         const module = try self.liveModule(handle);
 437         const credit = module.mappedBytes();
 438         module.unload(self.allocator);
 439         std.debug.assert(self.live_modules != 0);
 440         self.live_modules -= 1;
 441         std.debug.assert(self.mapped_bytes >= credit);
 442         self.mapped_bytes -= credit;
 443     }
 444 
 445     fn liveModule(self: *const JitRuntime, handle: ModuleHandle) HandleError!*Module {
 446         if (handle.index >= self.modules.items.len) return error.InvalidModuleHandle;
 447         const module = &self.modules.items[handle.index];
 448         if (!module.live) return error.InvalidModuleHandle;
 449         if (module.generation != handle.generation) return error.InvalidModuleHandle;
 450         return module;
 451     }
 452 
 453     fn findFunction(
 454         self: *const JitRuntime,
 455         handle: ModuleHandle,
 456         name: []const u8,
 457     ) LookupError!FoundFunction {
 458         const module = try self.liveModule(handle);
 459         const entry = module.functions.getPtr(name) orelse return error.FunctionNotFound;
 460         return .{ .address = module.functionAddress(entry), .entry = entry };
 461     }
 462 
 463     /// Returns the address of `name` without checking how a caller will call it.
 464     /// Fails with `error.InvalidModuleHandle` when `handle` no longer resolves, and with
 465     /// `error.FunctionNotFound` when its module has no function `name`.
 466     pub fn functionAddress(
 467         self: *const JitRuntime,
 468         handle: ModuleHandle,
 469         name: []const u8,
 470     ) LookupError!usize {
 471         const found = try self.findFunction(handle, name);
 472         return found.address;
 473     }
 474 
 475     /// Returns the parameter and result types recorded for `name`.
 476     /// Fails with `error.InvalidModuleHandle` when `handle` no longer resolves, and with
 477     /// `error.FunctionNotFound` when its module has no function `name`.
 478     pub fn functionSignature(
 479         self: *const JitRuntime,
 480         handle: ModuleHandle,
 481         name: []const u8,
 482     ) LookupError!artifact.Signature {
 483         const found = try self.findFunction(handle, name);
 484         return found.entry.signature;
 485     }
 486 
 487     /// Returns `name` as `FunctionPointer` when the signature derived from that type equals the
 488     /// recorded one. `backends.signature.ofZigFunction` gives the type mapping.
 489     /// Fails with `error.InvalidModuleHandle` or `error.FunctionNotFound` when the lookup fails,
 490     /// and with `error.SignatureMismatch` when the signatures differ.
 491     pub fn getFunction(
 492         self: *const JitRuntime,
 493         handle: ModuleHandle,
 494         name: []const u8,
 495         comptime FunctionPointer: type,
 496     ) CallError!FunctionPointer {
 497         const expected = backends.signature.ofZigFunction(FunctionPointer);
 498         const found = try self.findFunction(handle, name);
 499         if (!found.entry.signature.eql(&expected)) return error.SignatureMismatch;
 500         return @ptrFromInt(found.address);
 501     }
 502 
 503     /// Calls `name` with `args` and stores its results in `results`.
 504     /// Fails with `error.InvalidModuleHandle` or `error.FunctionNotFound` when the lookup fails.
 505     /// Fails with `error.SignatureMismatch`, without calling, when an argument's type, the argument
 506     /// count, or the number of result slots differs from the recorded signature.
 507     pub fn call(
 508         self: *const JitRuntime,
 509         handle: ModuleHandle,
 510         name: []const u8,
 511         args: []const invoke.Value,
 512         results: []invoke.Value,
 513     ) CallError!void {
 514         const found = try self.findFunction(handle, name);
 515         try invoke.call(found.address, &found.entry.signature, args, results);
 516     }
 517 
 518     /// Binds `name` for modules compiled or loaded afterward. Linked modules keep their targets.
 519     /// Replacing a name keeps its entry and owned name. A new name past `limits.external_symbols`
 520     /// fails with `error.JitRuntimeFull` before allocating. Either refusal or allocation failure
 521     /// leaves the table, its retained storage, and all existing bindings unchanged.
 522     pub fn registerExternalSymbol(
 523         self: *JitRuntime,
 524         name: []const u8,
 525         addr: usize,
 526     ) error{ OutOfMemory, JitRuntimeFull }!void {
 527         if (self.external_symbols.getPtr(name)) |existing| {
 528             existing.* = addr;
 529             return;
 530         }
 531 
 532         const count = self.external_symbols.count();
 533         if (count >= self.limits.external_symbols) return error.JitRuntimeFull;
 534         const owned_name = try self.allocator.dupe(u8, name);
 535         errdefer self.allocator.free(owned_name);
 536         try self.external_symbols.putNoClobber(self.allocator, owned_name, addr);
 537         std.debug.assert(self.external_symbols.count() == count + 1);
 538         std.debug.assert(self.external_symbols.count() <= self.limits.external_symbols);
 539     }
 540 
 541     fn externalAddress(self: *const JitRuntime, name: []const u8) ?usize {
 542         const address = self.external_symbols.get(name) orelse return null;
 543         if (address == 0) return null;
 544         return address;
 545     }
 546 
 547     /// Loads one machine-code function and its data buffers into a new module.
 548     /// Call relocations resolve against the function itself, then registered external symbols.
 549     /// Absolute relocations resolve against the artifact's data buffers.
 550     /// Fails with `error.InvalidArtifact` before mapping anything when the artifact is malformed or
 551     /// its function symbol lacks a signature this backend passes and returns.
 552     /// Fails with `error.JitRuntimeFull` when the runtime already holds `limits.live_modules`
 553     /// modules or when this artifact's pages would carry it past `limits.mapped_bytes`.
 554     /// A `release` admits the next module under either bound.
 555     pub fn loadMachineCodeArtifact(
 556         self: *JitRuntime,
 557         serialized: *const artifact.Artifact,
 558     ) MachineCodeArtifactError!ModuleHandle {
 559         var view = try ArtifactView.init(self.allocator, serialized);
 560         defer view.deinit(self.allocator);
 561         const index = try self.reserveModule();
 562 
 563         var staged: Module = .staged;
 564         errdefer staged.unload(self.allocator);
 565         try staged.functions.ensureUnusedCapacity(self.allocator, 1);
 566         const owned_name = try self.allocator.dupe(u8, view.export_name);
 567         staged.functions.putAssumeCapacityNoClobber(owned_name, .{
 568             .offset = 0,
 569             .size = view.code.len,
 570             .signature = view.signature,
 571         });
 572         self.link(&staged, view.image()) catch |err| switch (err) {
 573             error.DataTooLarge => return error.InvalidArtifact,
 574             else => |narrow| return narrow,
 575         };
 576         return self.install(index, staged);
 577     }
 578 
 579     /// Writes every live function's address range and line table.
 580     /// Registers debug objects with the debugger first where the platform supports it.
 581     pub fn writeDebugMap(self: *JitRuntime, writer: anytype) !bool {
 582         if (!self.hasFunctions()) return false;
 583         self.ensureDebugInfo(writer);
 584 
 585         try writer.writeAll("JIT debug map\n");
 586         for (self.modules.items) |*module| {
 587             if (!module.live) continue;
 588             var iterator = module.functions.iterator();
 589             while (iterator.next()) |entry| {
 590                 const name = entry.key_ptr.*;
 591                 const function = entry.value_ptr;
 592                 const start = module.functionAddress(function);
 593                 try writer.print("fn {s} 0x{x}..0x{x}\n", .{ name, start, start + function.size });
 594 
 595                 const table = function.line_table orelse {
 596                     try writer.writeAll("  (no line info)\n");
 597                     continue;
 598                 };
 599                 if (table.isEmpty()) {
 600                     try writer.writeAll("  (no line info)\n");
 601                     continue;
 602                 }
 603                 for (table.entries) |line| {
 604                     const info = line.info;
 605                     const file = info.file orelse "<unknown>";
 606                     const name_label = info.name orelse "_";
 607                     const addr = start + line.offset;
 608                     try writer.print(
 609                         "  +0x{x:0>4} 0x{x} {s}:{d}:{d} name={s} op={s}\n",
 610                         .{ line.offset, addr, file, info.line, info.column, name_label, info.op_name },
 611                     );
 612                 }
 613             }
 614         }
 615         return true;
 616     }
 617 
 618     fn hasFunctions(self: *const JitRuntime) bool {
 619         for (self.modules.items) |*module| {
 620             if (!module.live) continue;
 621             if (module.functions.count() != 0) return true;
 622         }
 623         return false;
 624     }
 625 
 626     fn ensureDebugInfo(self: *JitRuntime, writer: anytype) void {
 627         const arch = sys.capabilities.current.arch;
 628         const support = debug_info.jitDebugSupport(arch);
 629         if (!support.supported) {
 630             if (support.reason) |reason| {
 631                 writer.print("note: {s}; emitting debug map without debugger registration\n", .{reason}) catch {};
 632             }
 633             return;
 634         }
 635         if (support.note) |note| {
 636             writer.print("note: {s}\n", .{note}) catch {};
 637         }
 638 
 639         for (self.modules.items) |*module| {
 640             if (!module.live) continue;
 641             var iterator = module.functions.iterator();
 642             while (iterator.next()) |entry| {
 643                 const function = entry.value_ptr;
 644                 if (function.debug_object != null) continue;
 645                 const table = if (function.line_table) |*line_table| line_table else continue;
 646                 const code = module.code.?[function.offset..][0..function.size];
 647                 function.debug_object = debug_info.registerJitDebugInfo(
 648                     self.allocator,
 649                     arch,
 650                     @intFromPtr(code.ptr),
 651                     code,
 652                     table,
 653                     entry.key_ptr.*,
 654                 ) catch |err| {
 655                     writer.print("note: JIT debug registration failed: {s}\n", .{@errorName(err)}) catch {};
 656                     return;
 657                 };
 658             }
 659         }
 660     }
 661 };
 662 
 663 fn callerOf(
 664     module: *ir.Operation,
 665     staged: *const Module,
 666     call_site: emit.CallRelocation,
 667 ) *ir.Operation {
 668     var finder = CallFinder{ .callee = call_site.target };
 669     _ = module.walk(.{ .order = .pre_order }, &finder, CallFinder.visit) catch unreachable;
 670     if (finder.call) |call| return call;
 671     var functions = staged.functions.iterator();
 672     while (functions.next()) |entry| {
 673         const function = entry.value_ptr;
 674         if (call_site.offset < function.offset) continue;
 675         if (call_site.offset - function.offset >= function.size) continue;
 676         return ir.inspection.functionDefinitionByName(module, entry.key_ptr.*).?;
 677     }
 678     unreachable;
 679 }
 680 
 681 const CallFinder = struct {
 682     callee: []const u8,
 683     call: ?*ir.Operation = null,
 684 
 685     fn visit(self: *CallFinder, op: *ir.Operation) ir.WalkResult {
 686         std.debug.assert(self.call == null);
 687         if (!std.mem.eql(u8, op.name.name, dialects.FuncDialect.CallOp.operation_name)) {
 688             return .advance;
 689         }
 690         const callee = dialects.FuncDialect.CallOp.getCallee(.{ .op = op }) orelse return .advance;
 691         if (!std.mem.eql(u8, callee, self.callee)) return .advance;
 692         self.call = op;
 693         return .interrupt;
 694     }
 695 };
 696 
 697 /// Writes a little-endian `address` into the 8-byte window in `code` at `offset`.
 698 fn writeAddress(code: []u8, offset: usize, address: u64) void {
 699     std.debug.assert(offset <= code.len);
 700     std.debug.assert(code.len - offset >= 8);
 701     std.mem.writeInt(u64, code[offset..][0..8], address, .little);
 702 }
 703 
 704 fn signatureOf(func: *ir.Operation) backends.signature.Error!artifact.Signature {
 705     const signature = try backends.signature.ofFunction(func);
 706     if (!abi.admitsSignature(&signature)) return error.UnsupportedType;
 707     return signature;
 708 }
 709 
 710 const ArtifactView = struct {
 711     export_name: []const u8,
 712     code: []const u8,
 713     signature: artifact.Signature,
 714     data: machine.DataSymbolSet = .{},
 715     calls: std.ArrayListUnmanaged(emit.CallRelocation) = .empty,
 716     data_relocations: std.ArrayListUnmanaged(machine.DataRelocation) = .empty,
 717 
 718     const Error = std.mem.Allocator.Error || error{InvalidArtifact};
 719 
 720     fn init(allocator: std.mem.Allocator, serialized: *const artifact.Artifact) Error!ArtifactView {
 721         try checkMetadata(serialized.metadata);
 722         const buffers = serialized.payload.buffers.items;
 723         const provided = serialized.linkage.provided_symbols.items;
 724         if (buffers.len == 0 or provided.len != buffers.len) return error.InvalidArtifact;
 725         const code = buffers[0];
 726         if (code.format != .machine_code or code.bytes.len == 0) return error.InvalidArtifact;
 727         const export_symbol = provided[0];
 728         if (export_symbol.kind != .function) return error.InvalidArtifact;
 729         if (export_symbol.binding != .external) return error.InvalidArtifact;
 730         if (export_symbol.name.len == 0) return error.InvalidArtifact;
 731         if (!std.mem.eql(u8, export_symbol.name, code.name)) return error.InvalidArtifact;
 732         const signature = export_symbol.signature orelse return error.InvalidArtifact;
 733         if (!abi.admitsSignature(&signature)) return error.InvalidArtifact;
 734 
 735         var view = ArtifactView{
 736             .export_name = export_symbol.name,
 737             .code = code.bytes,
 738             .signature = signature,
 739         };
 740         errdefer view.deinit(allocator);
 741         for (1..buffers.len) |index| try view.addData(allocator, serialized, index);
 742 
 743         const relocations = serialized.linkage.relocations.items;
 744         const offsets = try allocator.alloc(usize, relocations.len);
 745         defer allocator.free(offsets);
 746         for (relocations, offsets) |relocation, *offset| {
 747             offset.* = try view.addRelocation(allocator, relocation);
 748         }
 749         std.mem.sortUnstable(usize, offsets, {}, std.sort.asc(usize));
 750         if (offsets.len > 1) {
 751             for (offsets[0 .. offsets.len - 1], offsets[1..]) |previous, next| {
 752                 if (next - previous < 8) return error.InvalidArtifact;
 753             }
 754         }
 755         std.debug.assert(view.data.items().len + 1 == buffers.len);
 756         std.debug.assert(view.calls.items.len + view.data_relocations.items.len == relocations.len);
 757         return view;
 758     }
 759 
 760     fn deinit(self: *ArtifactView, allocator: std.mem.Allocator) void {
 761         self.data.deinit(allocator);
 762         self.calls.deinit(allocator);
 763         self.data_relocations.deinit(allocator);
 764         self.* = undefined;
 765     }
 766 
 767     fn image(self: *const ArtifactView) Image {
 768         return .{
 769             .code = self.code,
 770             .calls = self.calls.items,
 771             .data = &self.data,
 772             .data_relocations = self.data_relocations.items,
 773         };
 774     }
 775 
 776     fn checkMetadata(metadata: artifact.Metadata) error{InvalidArtifact}!void {
 777         if (metadata.kind != .machine_code) return error.InvalidArtifact;
 778         if (metadata.target.architecture != .x86_64) return error.InvalidArtifact;
 779         if (metadata.abi.pointer_width_bits != 64) return error.InvalidArtifact;
 780         if (metadata.abi.endianness != .little) return error.InvalidArtifact;
 781         const abi_name = metadata.abi.name orelse return error.InvalidArtifact;
 782         if (!std.mem.eql(u8, abi_name, "sysv")) return error.InvalidArtifact;
 783         const convention = metadata.abi.calling_convention orelse return error.InvalidArtifact;
 784         if (!std.mem.eql(u8, convention, "c")) return error.InvalidArtifact;
 785     }
 786 
 787     /// Adds the raw buffer at `index` as a data symbol under the name its provided symbol gives.
 788     fn addData(
 789         self: *ArtifactView,
 790         allocator: std.mem.Allocator,
 791         serialized: *const artifact.Artifact,
 792         index: usize,
 793     ) Error!void {
 794         std.debug.assert(index != 0);
 795         std.debug.assert(index < serialized.linkage.provided_symbols.items.len);
 796         const buffer = serialized.payload.buffers.items[index];
 797         const symbol = serialized.linkage.provided_symbols.items[index];
 798         if (buffer.format != .raw or symbol.kind != .data) return error.InvalidArtifact;
 799         if (!std.mem.eql(u8, buffer.name, symbol.name)) return error.InvalidArtifact;
 800         if (std.mem.eql(u8, buffer.name, self.export_name)) return error.InvalidArtifact;
 801         const binding: machine.DataSymbolBinding = switch (symbol.binding) {
 802             .local => .local,
 803             .external => .global,
 804             .weak => return error.InvalidArtifact,
 805         };
 806         const count = self.data.items().len;
 807         self.data.put(allocator, .{
 808             .name = buffer.name,
 809             .bytes = buffer.bytes,
 810             .alignment = buffer.alignment,
 811             .binding = binding,
 812         }) catch |err| switch (err) {
 813             error.OutOfMemory => return error.OutOfMemory,
 814             error.InvalidDataSymbol, error.ConflictingDataSymbol => return error.InvalidArtifact,
 815         };
 816         if (self.data.items().len == count) return error.InvalidArtifact;
 817         std.debug.assert(self.data.indexOf(buffer.name) == count);
 818     }
 819 
 820     /// Records one call or data relocation and returns its code offset.
 821     fn addRelocation(
 822         self: *ArtifactView,
 823         allocator: std.mem.Allocator,
 824         relocation: artifact.Relocation,
 825     ) Error!usize {
 826         const offset = std.math.cast(usize, relocation.offset) orelse return error.InvalidArtifact;
 827         if (offset > self.code.len or self.code.len - offset < 8) return error.InvalidArtifact;
 828         if (relocation.width_bits) |width| if (width != 64) return error.InvalidArtifact;
 829         switch (relocation.kind) {
 830             .call => {
 831                 if (relocation.symbol.len == 0) return error.InvalidArtifact;
 832                 if (relocation.addend != 0) return error.InvalidArtifact;
 833                 const call = emit.CallRelocation{ .offset = offset, .target = relocation.symbol };
 834                 try self.calls.append(allocator, call);
 835             },
 836             .absolute => {
 837                 if (self.data.indexOf(relocation.symbol) == null) return error.InvalidArtifact;
 838                 try self.data_relocations.append(allocator, .{
 839                     .offset = offset,
 840                     .target = relocation.symbol,
 841                     .addend = relocation.addend,
 842                 });
 843             },
 844             else => return error.InvalidArtifact,
 845         }
 846         std.debug.assert(offset + 8 <= self.code.len);
 847         return offset;
 848     }
 849 };
 850 
 851 fn choirDeclAdd1(value: i64) callconv(.c) i64 {
 852     return value + 1;
 853 }
 854 
 855 const artifact_target = artifact.Target{ .architecture = .x86_64 };
 856 const artifact_abi = artifact.Abi{
 857     .name = "sysv",
 858     .calling_convention = "c",
 859     .pointer_width_bits = 64,
 860     .endianness = .little,
 861 };
 862 const nullary_integer = artifact.Signature.init(&.{}, &.{.{ .scalar = .i64 }}) catch unreachable;
 863 
 864 const MalformedDataArtifact = struct {
 865     data: []const artifact.DataBuffer = &.{table},
 866     relocations: []const artifact.Relocation = &.{reference},
 867     orphan_buffer: bool = false,
 868 
 869     const table = artifact.DataBuffer{ .name = "table", .bytes = "\x01\x02" };
 870     const reference = artifact.Relocation{ .offset = 0, .symbol = "table", .kind = .absolute };
 871 };
 872 
 873 const JitIrResourceCounts = struct {
 874     operations: usize,
 875 
 876     fn capture(ctx: *const ir.Context) JitIrResourceCounts {
 877         return .{
 878             .operations = ctx.operationCount(),
 879         };
 880     }
 881 
 882     fn expectEqual(self: JitIrResourceCounts, ctx: *const ir.Context) !void {
 883         try std.testing.expectEqual(self.operations, ctx.operationCount());
 884     }
 885 };
 886 
 887 fn expectEmitterFunctionStateDiscarded(runtime: *const JitRuntime) !void {
 888     const emitter = &runtime.emitter;
 889     try std.testing.expectEqual(@as(usize, 0), emitter.code.items.len);
 890     try std.testing.expectEqual(@as(usize, 0), emitter.call_relocations.items.len);
 891     try std.testing.expectEqual(@as(usize, 0), emitter.data_symbols.items().len);
 892     try std.testing.expectEqual(@as(usize, 0), emitter.data_relocations.items.len);
 893     try std.testing.expectEqual(@as(usize, 0), emitter.slot_map.count());
 894     try std.testing.expectEqual(@as(usize, 0), emitter.vector_slot_map.count());
 895     try std.testing.expectEqual(@as(usize, 0), emitter.alloca_payload_slots.count());
 896     try std.testing.expectEqual(@as(usize, 0), emitter.allocation_kinds.count());
 897     try std.testing.expectEqual(@as(usize, 0), emitter.value_locations.count());
 898     try std.testing.expectEqual(@as(usize, 0), emitter.value_location_ranges.items.len);
 899     try std.testing.expectEqual(@as(usize, 0), emitter.value_location_index.value_keys.items.len);
 900     try std.testing.expectEqual(@as(usize, 0), emitter.value_location_index.value_heads.items.len);
 901     try std.testing.expectEqual(@as(usize, 0), emitter.value_location_index.value_links.items.len);
 902     try std.testing.expectEqual(@as(usize, 0), emitter.xmm_locations.count());
 903     try std.testing.expectEqual(@as(usize, 0), emitter.xmm_location_ranges.items.len);
 904     try std.testing.expectEqual(@as(usize, 0), emitter.xmm_location_index.value_keys.items.len);
 905     try std.testing.expectEqual(@as(usize, 0), emitter.xmm_location_index.value_heads.items.len);
 906     try std.testing.expectEqual(@as(usize, 0), emitter.xmm_location_index.value_links.items.len);
 907     try std.testing.expectEqual(@as(usize, 0), emitter.operation_positions.count());
 908 }
 909 
 910 fn createJitConstantFunction(ctx: *ir.Context, name: []const u8, value: i64) !*ir.Operation {
 911     const ArithDialect = dialects.ArithDialect;
 912     const FuncDialect = dialects.FuncDialect;
 913     const loc = ir.Location.getFile("temporary-jit.choir", 1, 1);
 914     const i64_type = try ArithDialect.getScalarType(ctx, .i64);
 915     const func = try FuncDialect.FuncOp.create(ctx, loc, name, &.{}, &.{i64_type});
 916     errdefer func.op.erase();
 917     const entry = func.getEntryBlock();
 918     const constant = try ArithDialect.ConstantOp.createInt(ctx, loc, i64_type, value);
 919     try entry.addOperation(constant.op);
 920     const ret = try FuncDialect.ReturnOp.create(ctx, loc, &.{constant.getResult()});
 921     try entry.addOperation(ret.op);
 922     return func.op;
 923 }
 924 
 925 fn createJitCallingFunction(ctx: *ir.Context, name: []const u8, callee: []const u8) !*ir.Operation {
 926     const ArithDialect = dialects.ArithDialect;
 927     const FuncDialect = dialects.FuncDialect;
 928     const loc = ir.Location.getFile("temporary-jit.choir", 3, 1);
 929     const i64_type = try ArithDialect.getScalarType(ctx, .i64);
 930     const func = try FuncDialect.FuncOp.create(ctx, loc, name, &.{}, &.{i64_type});
 931     errdefer func.op.erase();
 932     const entry = func.getEntryBlock();
 933     const call = try FuncDialect.CallOp.create(ctx, loc, callee, &.{}, &.{i64_type});
 934     try entry.addOperation(call.op);
 935     const ret = try FuncDialect.ReturnOp.create(ctx, loc, &.{call.getResult(0).?});
 936     try entry.addOperation(ret.op);
 937     return func.op;
 938 }
 939 
 940 fn createJitConstantModule(ctx: *ir.Context, name: []const u8, value: i64) !*ir.Operation {
 941     const module = try dialects.BuiltinDialect.ModuleOp.create(ctx, ir.Location.getUnknown());
 942     errdefer module.op.erase();
 943     const function = try createJitConstantFunction(ctx, name, value);
 944     errdefer function.erase();
 945     try module.getBodyBlock().addOperation(function);
 946     return module.op;
 947 }
 948 
 949 fn createJitMemrefIdentityModule(ctx: *ir.Context, name: []const u8) !*ir.Operation {
 950     const i32_type = try dialects.ArithDialect.getScalarType(ctx, .i32);
 951     const memref_type = try dialects.MemrefDialect.getMemrefType1D(ctx, 4, i32_type, .host);
 952     const loc = ir.Location.getFile("temporary-jit.choir", 2, 1);
 953     const types = [_]ir.Type{memref_type};
 954     const function = try dialects.FuncDialect.FuncOp.create(ctx, loc, name, &types, &types);
 955     errdefer function.op.erase();
 956     const ret = try dialects.FuncDialect.ReturnOp.create(ctx, loc, &.{function.getArgument(0)});
 957     try function.getEntryBlock().addOperation(ret.op);
 958     const module = try dialects.BuiltinDialect.ModuleOp.create(ctx, loc);
 959     errdefer module.op.erase();
 960     try module.getBodyBlock().addOperation(function.op);
 961     return module.op;
 962 }
 963 
 964 /// Appends an integer constant that carries the attributes of `symbol`, so it evaluates to the
 965 /// address of the bytes instead of its value, 7.
 966 fn appendJitDataConstant(
 967     ctx: *ir.Context,
 968     block: *ir.Block,
 969     symbol: machine.DataSymbol,
 970 ) !*ir.Value {
 971     const ArithDialect = dialects.ArithDialect;
 972     const names = machine.data_symbol_attr_names;
 973     const loc = ir.Location.getFile("data-jit.choir", 2, 1);
 974     const i64_type = try ArithDialect.getScalarType(ctx, .i64);
 975     const constant = try ArithDialect.ConstantOp.createInt(ctx, loc, i64_type, 7);
 976     try block.addOperation(constant.op);
 977     try constant.op.setAttr(names.name, try ctx.getStringAttr(symbol.name));
 978     try constant.op.setAttr(names.bytes, try ctx.getStringAttr(symbol.bytes));
 979     try constant.op.setAttr(names.alignment, try ctx.getI64Attr(@intCast(symbol.alignment)));
 980     return constant.getResult();
 981 }
 982 
 983 /// Builds `name` returning the address of `symbol`.
 984 fn createJitDataFunction(
 985     ctx: *ir.Context,
 986     name: []const u8,
 987     symbol: machine.DataSymbol,
 988 ) !*ir.Operation {
 989     const FuncDialect = dialects.FuncDialect;
 990     const loc = ir.Location.getFile("data-jit.choir", 1, 1);
 991     const i64_type = try dialects.ArithDialect.getScalarType(ctx, .i64);
 992     const func = try FuncDialect.FuncOp.create(ctx, loc, name, &.{}, &.{i64_type});
 993     errdefer func.op.erase();
 994     const entry = func.getEntryBlock();
 995     const address = try appendJitDataConstant(ctx, entry, symbol);
 996     const ret = try FuncDialect.ReturnOp.create(ctx, loc, &.{address});
 997     try entry.addOperation(ret.op);
 998     return func.op;
 999 }
1000 
1001 /// Builds `name` returning `(address + delta) - address` for the address of `symbol`.
1002 fn createJitDataDeltaFunction(
1003     ctx: *ir.Context,
1004     name: []const u8,
1005     symbol: machine.DataSymbol,
1006     delta: i64,
1007 ) !*ir.Operation {
1008     const ArithDialect = dialects.ArithDialect;
1009     const FuncDialect = dialects.FuncDialect;
1010     const loc = ir.Location.getFile("data-jit.choir", 3, 1);
1011     const i64_type = try ArithDialect.getScalarType(ctx, .i64);
1012     const func = try FuncDialect.FuncOp.create(ctx, loc, name, &.{}, &.{i64_type});
1013     errdefer func.op.erase();
1014     const entry = func.getEntryBlock();
1015     const address = try appendJitDataConstant(ctx, entry, symbol);
1016     const offset = try ArithDialect.ConstantOp.createInt(ctx, loc, i64_type, delta);
1017     try entry.addOperation(offset.op);
1018     const sum = try ArithDialect.AddOp.create(ctx, loc, address, offset.getResult());
1019     try entry.addOperation(sum.op);
1020     const difference = try ArithDialect.SubOp.create(ctx, loc, sum.getResult(), address);
1021     try entry.addOperation(difference.op);
1022     const ret = try FuncDialect.ReturnOp.create(ctx, loc, &.{difference.getResult()});
1023     try entry.addOperation(ret.op);
1024     return func.op;
1025 }
1026 
1027 fn createJitDataModule(
1028     ctx: *ir.Context,
1029     name: []const u8,
1030     symbol: machine.DataSymbol,
1031 ) !*ir.Operation {
1032     const module = try dialects.BuiltinDialect.ModuleOp.create(ctx, ir.Location.getUnknown());
1033     errdefer module.op.erase();
1034     const function = try createJitDataFunction(ctx, name, symbol);
1035     errdefer function.erase();
1036     try module.getBodyBlock().addOperation(function);
1037     return module.op;
1038 }
1039 
1040 fn addJitFunction(module: *ir.Operation, function: *ir.Operation) !void {
1041     errdefer function.erase();
1042     try module.getRegion(0).?.getEntryBlock().?.addOperation(function);
1043 }
1044 
1045 fn addressFrom(runtime: *const JitRuntime, handle: ModuleHandle, name: []const u8) !usize {
1046     const Address = *const fn () callconv(.c) i64;
1047     return @intCast((try runtime.getFunction(handle, name, Address))());
1048 }
1049 
1050 fn bytesAt(address: usize, len: usize) []const u8 {
1051     const bytes: [*]const u8 = @ptrFromInt(address);
1052     return bytes[0..len];
1053 }
1054 
1055 fn readProcSelfMaps(allocator: std.mem.Allocator) ![]u8 {
1056     var file = try std.Io.Dir.openFileAbsolute(std.Options.debug_io, "/proc/self/maps", .{});
1057     defer file.close(std.Options.debug_io);
1058     var maps: std.ArrayList(u8) = .empty;
1059     errdefer maps.deinit(allocator);
1060     var buffer: [4096]u8 = undefined;
1061     while (true) {
1062         const count = try sys.fd.read(file.handle, &buffer);
1063         if (count == 0) break;
1064         try maps.appendSlice(allocator, buffer[0..count]);
1065     }
1066     return try maps.toOwnedSlice(allocator);
1067 }
1068 
1069 /// Returns the permission field of the `/proc/self/maps` line whose range holds `address`.
1070 fn mappingPermissions(maps: []const u8, address: usize) ?[]const u8 {
1071     var lines = std.mem.splitScalar(u8, maps, '\n');
1072     while (lines.next()) |line| {
1073         var fields = std.mem.tokenizeScalar(u8, line, ' ');
1074         const range = fields.next() orelse continue;
1075         const permissions = fields.next() orelse continue;
1076         const dash = std.mem.indexOfScalar(u8, range, '-') orelse continue;
1077         const start = std.fmt.parseInt(usize, range[0..dash], 16) catch continue;
1078         const end = std.fmt.parseInt(usize, range[dash + 1 ..], 16) catch continue;
1079         if (start <= address and address < end) return permissions;
1080     }
1081     return null;
1082 }
1083 
1084 test "JitRuntime machine-code artifact failures leave no loaded state" {
1085     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1086 
1087     const testing = std.testing;
1088     const code = @as([16]u8, @splat(0));
1089 
1090     var malformed = try artifact.machineCodeArtifact(
1091         testing.allocator,
1092         artifact_target,
1093         artifact_abi,
1094         "malformed_entry",
1095         nullary_integer,
1096         &code,
1097         &.{},
1098         &.{.{ .offset = 12, .symbol = "choir_decl_add1", .kind = .call }},
1099     );
1100     defer malformed.deinit();
1101     var runtime = JitRuntime.init(testing.allocator, .testing);
1102     defer runtime.deinit();
1103     try runtime.registerExternalSymbol("choir_decl_add1", @intFromPtr(&choirDeclAdd1));
1104     try testing.expectError(error.InvalidArtifact, runtime.loadMachineCodeArtifact(&malformed));
1105     try testing.expectEqual(@as(usize, 0), runtime.modules.items.len);
1106 
1107     var unresolved = try artifact.machineCodeArtifact(
1108         testing.allocator,
1109         artifact_target,
1110         artifact_abi,
1111         "unresolved_entry",
1112         nullary_integer,
1113         &code,
1114         &.{},
1115         &.{.{ .offset = 0, .symbol = "missing_external", .kind = .call }},
1116     );
1117     defer unresolved.deinit();
1118     var unresolved_runtime = JitRuntime.init(testing.allocator, .testing);
1119     defer unresolved_runtime.deinit();
1120     try testing.expectError(error.SymbolNotFound, unresolved_runtime.loadMachineCodeArtifact(&unresolved));
1121     try testing.expectEqual(@as(usize, 0), unresolved_runtime.modules.items.len);
1122 
1123     var valid = try artifact.machineCodeArtifact(
1124         testing.allocator,
1125         artifact_target,
1126         artifact_abi,
1127         "allocation_entry",
1128         nullary_integer,
1129         &code,
1130         &.{.{ .name = "allocation_table", .bytes = "\x01\x02", .alignment = 8 }},
1131         &.{
1132             .{ .offset = 0, .symbol = "choir_decl_add1", .kind = .call },
1133             .{ .offset = 8, .symbol = "allocation_table", .kind = .absolute },
1134         },
1135     );
1136     defer valid.deinit();
1137     var reached_success = false;
1138     var fail_offset: usize = 0;
1139     while (fail_offset < 32) : (fail_offset += 1) {
1140         var failing = testing.FailingAllocator.init(testing.allocator, .{});
1141         var allocation_runtime = JitRuntime.init(failing.allocator(), .testing);
1142         defer allocation_runtime.deinit();
1143         try allocation_runtime.registerExternalSymbol("choir_decl_add1", @intFromPtr(&choirDeclAdd1));
1144         failing.fail_index = failing.alloc_index + fail_offset;
1145         if (allocation_runtime.loadMachineCodeArtifact(&valid)) |_| {
1146             reached_success = true;
1147             break;
1148         } else |err| {
1149             try testing.expectEqual(error.OutOfMemory, err);
1150             try testing.expectEqual(@as(usize, 0), allocation_runtime.modules.items.len);
1151         }
1152     }
1153     try testing.expect(reached_success);
1154 }
1155 
1156 test "JitRuntime patches data addresses into loaded machine code" {
1157     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1158 
1159     const testing = std.testing;
1160     const movabs_rax = [2]u8{ 0x48, 0xb8 };
1161     const movabs_rdx = [2]u8{ 0x48, 0xba };
1162     const ret_opcode = 0xc3;
1163     const unreached_movabs_rcx = [2]u8{ 0x48, 0xb9 };
1164     var code = @as([31]u8, @splat(0));
1165     code[0..2].* = movabs_rax;
1166     code[10..12].* = movabs_rdx;
1167     code[20] = ret_opcode;
1168     code[21..23].* = unreached_movabs_rcx;
1169     const table = "\x10\x20\x30\x40\x50";
1170     const page = "page bytes";
1171     const addresses_signature = try artifact.Signature.init(
1172         &.{},
1173         &.{ .{ .scalar = .index }, .{ .scalar = .index } },
1174     );
1175     var loadable = try artifact.machineCodeArtifact(
1176         testing.allocator,
1177         artifact_target,
1178         artifact_abi,
1179         "data_entry",
1180         addresses_signature,
1181         &code,
1182         &.{
1183             .{ .name = "table", .bytes = table, .alignment = 4 },
1184             .{ .name = "page", .bytes = page, .alignment = machine.max_data_alignment },
1185         },
1186         &.{
1187             .{ .offset = 2, .symbol = "table", .kind = .absolute, .addend = 3 },
1188             .{ .offset = 12, .symbol = "page", .kind = .absolute },
1189             .{ .offset = 23, .symbol = "data_entry", .kind = .call },
1190         },
1191     );
1192     defer loadable.deinit();
1193 
1194     var runtime = JitRuntime.init(testing.allocator, .testing);
1195     defer runtime.deinit();
1196     const handle = try runtime.loadMachineCodeArtifact(&loadable);
1197     const Addresses = extern struct { table: usize, page: usize };
1198     const Entry = *const fn () callconv(.c) Addresses;
1199     const entry = try runtime.getFunction(handle, "data_entry", Entry);
1200     const addresses = entry();
1201     const table_address = addresses.table - 3;
1202     try testing.expect(std.mem.isAligned(table_address, 4));
1203     try testing.expectEqualSlices(u8, table, bytesAt(table_address, table.len));
1204     try testing.expect(std.mem.isAligned(addresses.page, machine.max_data_alignment));
1205     try testing.expectEqualSlices(u8, page, bytesAt(addresses.page, page.len));
1206 
1207     const entry_address = try runtime.functionAddress(handle, "data_entry");
1208     const self_call = bytesAt(entry_address + 23, 8)[0..8];
1209     try testing.expectEqual(@as(u64, entry_address), std.mem.readInt(u64, self_call, .little));
1210 }
1211 
1212 test "JitRuntime rejects malformed data artifacts" {
1213     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1214 
1215     const testing = std.testing;
1216     const code = @as([16]u8, @splat(0));
1217     const table = MalformedDataArtifact.table;
1218     const reference = MalformedDataArtifact.reference;
1219     const narrow_window = artifact.Relocation{
1220         .offset = 0,
1221         .symbol = "table",
1222         .kind = .absolute,
1223         .width_bits = 32,
1224     };
1225     const call_with_addend = artifact.Relocation{
1226         .offset = 0,
1227         .symbol = "choir_decl_add1",
1228         .kind = .call,
1229         .addend = 1,
1230     };
1231     const cases = [_]MalformedDataArtifact{
1232         .{ .relocations = &.{.{ .offset = 0, .symbol = "missing", .kind = .absolute }} },
1233         .{ .relocations = &.{.{ .offset = 0, .symbol = "malformed_entry", .kind = .absolute }} },
1234         .{ .data = &.{ table, .{ .name = "malformed_entry", .bytes = "\x03" } } },
1235         .{ .data = &.{.{ .name = "table", .bytes = "\x01\x02", .alignment = 3 }} },
1236         .{ .data = &.{.{ .name = "table", .bytes = "" }} },
1237         .{ .data = &.{.{ .name = "table", .bytes = "\x01\x02", .binding = .weak }} },
1238         .{ .data = &.{ table, table } },
1239         .{ .relocations = &.{ reference, .{ .offset = 4, .symbol = "table", .kind = .absolute } } },
1240         .{ .relocations = &.{.{ .offset = 9, .symbol = "table", .kind = .absolute }} },
1241         .{ .relocations = &.{narrow_window} },
1242         .{ .relocations = &.{call_with_addend} },
1243         .{ .relocations = &.{.{ .offset = 0, .symbol = "table", .kind = .unknown }} },
1244         .{ .orphan_buffer = true },
1245     };
1246 
1247     var runtime = JitRuntime.init(testing.allocator, .testing);
1248     defer runtime.deinit();
1249     try runtime.registerExternalSymbol("choir_decl_add1", @intFromPtr(&choirDeclAdd1));
1250     for (cases) |case| {
1251         var malformed = try artifact.machineCodeArtifact(
1252             testing.allocator,
1253             artifact_target,
1254             artifact_abi,
1255             "malformed_entry",
1256             nullary_integer,
1257             &code,
1258             case.data,
1259             case.relocations,
1260         );
1261         defer malformed.deinit();
1262         if (case.orphan_buffer) {
1263             try malformed.payload.addBuffer(.{ .name = "orphan", .format = .raw, .bytes = "\x03" });
1264         }
1265         try testing.expectError(error.InvalidArtifact, runtime.loadMachineCodeArtifact(&malformed));
1266         try testing.expectEqual(@as(usize, 0), runtime.modules.items.len);
1267     }
1268 }
1269 
1270 test "JitRuntime refuses artifacts without a signature it can call" {
1271     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1272 
1273     const testing = std.testing;
1274     const code = @as([16]u8, @splat(0xc3));
1275     const lanes = artifact.ValueType{ .vector = .{ .element = .f32, .lanes = 4 } };
1276     var no_lanes = try artifact.Signature.init(&.{}, &.{lanes});
1277     no_lanes.result_types[0].vector.lanes = 0;
1278     const refused = [_]?artifact.Signature{
1279         null,
1280         try artifact.Signature.init(&.{lanes}, &.{}),
1281         try artifact.Signature.init(&.{}, &.{ lanes, .{ .scalar = .f32 } }),
1282         no_lanes,
1283     };
1284     var runtime = JitRuntime.init(testing.allocator, .testing);
1285     defer runtime.deinit();
1286     for (refused) |signature| {
1287         var unsigned = try artifact.machineCodeArtifact(
1288             testing.allocator,
1289             artifact_target,
1290             artifact_abi,
1291             "entry",
1292             nullary_integer,
1293             &code,
1294             &.{},
1295             &.{},
1296         );
1297         defer unsigned.deinit();
1298         unsigned.linkage.provided_symbols.items[0].signature = signature;
1299         try testing.expectError(error.InvalidArtifact, runtime.loadMachineCodeArtifact(&unsigned));
1300         try testing.expectEqual(@as(usize, 0), runtime.modules.items.len);
1301     }
1302 }
1303 
1304 test "JitRuntime failed compiles keep previously compiled modules" {
1305     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1306 
1307     const testing = std.testing;
1308     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1309     defer ctx.deinit(testing.allocator);
1310     try dialects.registerAllDialects(&ctx);
1311     const kept_module = try createJitConstantModule(&ctx, "kept", 7);
1312     defer kept_module.erase();
1313     const staged_module = try createJitConstantModule(&ctx, "staged", 8);
1314     defer staged_module.erase();
1315     const staged_table = machine.DataSymbol{
1316         .name = "staged_table",
1317         .bytes = "staged bytes",
1318         .alignment = 8,
1319     };
1320     const table_function = try createJitDataFunction(&ctx, "staged_table", staged_table);
1321     try addJitFunction(staged_module, table_function);
1322 
1323     var reports = diagnostics.CaptureBuffer.init(testing.allocator);
1324     defer reports.deinit();
1325     var capture = ctx.captureDiagnostics(&reports);
1326     var guard = capture.enter();
1327     defer guard.deinit();
1328 
1329     const Constant = *const fn () callconv(.c) i64;
1330     var failing = testing.FailingAllocator.init(testing.allocator, .{});
1331     var runtime = JitRuntime.init(failing.allocator(), .testing);
1332     defer runtime.deinit();
1333     const kept = try runtime.compile(kept_module);
1334     const kept_value = try runtime.getFunction(kept, "kept", Constant);
1335 
1336     var reached_success = false;
1337     var fail_offset: usize = 0;
1338     while (fail_offset < 128) : (fail_offset += 1) {
1339         failing.fail_index = failing.alloc_index + fail_offset;
1340         if (runtime.compile(staged_module)) |staged| {
1341             failing.fail_index = std.math.maxInt(usize);
1342             const staged_value = try runtime.getFunction(staged, "staged", Constant);
1343             try testing.expectEqual(@as(i64, 8), staged_value());
1344             const table_address = try addressFrom(&runtime, staged, "staged_table");
1345             const table_bytes = bytesAt(table_address, staged_table.bytes.len);
1346             try testing.expectEqualStrings(staged_table.bytes, table_bytes);
1347             reached_success = true;
1348             break;
1349         } else |err| {
1350             try testing.expectEqual(error.OutOfMemory, err);
1351             try expectEmitterFunctionStateDiscarded(&runtime);
1352             try testing.expectEqual(@as(usize, 1), runtime.modules.items.len);
1353             try testing.expectEqual(@as(i64, 7), kept_value());
1354         }
1355     }
1356     try testing.expect(reached_success);
1357     try testing.expectEqual(@as(i64, 7), kept_value());
1358     try testing.expectEqual(@as(usize, 0), reports.diagnostics.items.len);
1359 }
1360 
1361 test "JitRuntime data pages stay read-only and outlive released modules" {
1362     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1363 
1364     const testing = std.testing;
1365     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1366     defer ctx.deinit(testing.allocator);
1367     try dialects.registerAllDialects(&ctx);
1368     const first_table = machine.DataSymbol{ .name = "table", .bytes = "first", .alignment = 16 };
1369     const first_module = try createJitDataModule(&ctx, "table", first_table);
1370     defer first_module.erase();
1371     const second_table = machine.DataSymbol{ .name = "table", .bytes = "second" };
1372     const second_module = try createJitDataModule(&ctx, "table", second_table);
1373     defer second_module.erase();
1374 
1375     var runtime = JitRuntime.init(testing.allocator, .testing);
1376     defer runtime.deinit();
1377     const first = try runtime.compile(first_module);
1378     const second = try runtime.compile(second_module);
1379     const first_address = try addressFrom(&runtime, first, "table");
1380     const second_address = try addressFrom(&runtime, second, "table");
1381     try testing.expect(std.mem.isAligned(first_address, 16));
1382     try testing.expectEqualStrings("first", bytesAt(first_address, 5));
1383     try testing.expectEqualStrings("second", bytesAt(second_address, 6));
1384 
1385     if (comptime @import("builtin").os.tag == .linux) {
1386         const maps = try readProcSelfMaps(testing.allocator);
1387         defer testing.allocator.free(maps);
1388         try testing.expectEqualStrings("r--p", mappingPermissions(maps, first_address).?);
1389         const code_address = try runtime.functionAddress(first, "table");
1390         try testing.expectEqualStrings("r-xp", mappingPermissions(maps, code_address).?);
1391     }
1392 
1393     try runtime.release(first);
1394     try testing.expectEqualStrings("second", bytesAt(second_address, 6));
1395     const reloaded = try runtime.compile(first_module);
1396     try testing.expectEqual(first.index, reloaded.index);
1397     const reloaded_address = try addressFrom(&runtime, reloaded, "table");
1398     try testing.expectEqualStrings("first", bytesAt(reloaded_address, 5));
1399 }
1400 
1401 test "JitRuntime modules share equal data symbols and refuse conflicting ones" {
1402     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1403 
1404     const testing = std.testing;
1405     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1406     defer ctx.deinit(testing.allocator);
1407     try dialects.registerAllDialects(&ctx);
1408     const table = machine.DataSymbol{ .name = "table", .bytes = "shared" };
1409     const shared = try createJitDataModule(&ctx, "first", table);
1410     defer shared.erase();
1411     try addJitFunction(shared, try createJitDataFunction(&ctx, "second", table));
1412     try addJitFunction(shared, try createJitDataDeltaFunction(&ctx, "delta", table, 5));
1413 
1414     const Value = *const fn () callconv(.c) i64;
1415     var runtime = JitRuntime.init(testing.allocator, .testing);
1416     defer runtime.deinit();
1417     const handle = try runtime.compile(shared);
1418     const first = (try runtime.getFunction(handle, "first", Value))();
1419     try testing.expectEqual(first, (try runtime.getFunction(handle, "second", Value))());
1420     try testing.expectEqual(@as(i64, 5), (try runtime.getFunction(handle, "delta", Value))());
1421 
1422     const conflicting = try createJitDataModule(&ctx, "first", table);
1423     defer conflicting.erase();
1424     const different = machine.DataSymbol{ .name = "table", .bytes = "different" };
1425     const second = try createJitDataFunction(&ctx, "second", different);
1426     try addJitFunction(conflicting, second);
1427 
1428     var reports = diagnostics.CaptureBuffer.init(testing.allocator);
1429     defer reports.deinit();
1430     var capture = ctx.captureDiagnostics(&reports);
1431     var guard = capture.enter();
1432     defer guard.deinit();
1433     try testing.expectError(error.ConflictingDataSymbol, runtime.compile(conflicting));
1434     try testing.expectEqual(@as(usize, 1), runtime.modules.items.len);
1435     try expectEmitterFunctionStateDiscarded(&runtime);
1436     try testing.expectEqual(@as(usize, 1), reports.diagnostics.items.len);
1437     try testing.expectEqual(second, reports.diagnostics.items[0].operation.?);
1438 }
1439 
1440 test "JitRuntime modules outlive their IR until released" {
1441     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1442 
1443     const testing = std.testing;
1444     var runtime = JitRuntime.init(testing.allocator, .testing);
1445     defer runtime.deinit();
1446 
1447     var constants: ModuleHandle = undefined;
1448     var identity: ModuleHandle = undefined;
1449     const identity_name = "temporary_memref_identity";
1450     {
1451         var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1452         defer ctx.deinit(testing.allocator);
1453         try dialects.registerAllDialects(&ctx);
1454         const resource_baseline = JitIrResourceCounts.capture(&ctx);
1455 
1456         const module = try createJitConstantModule(&ctx, "temporary_first", 41);
1457         var module_owned = true;
1458         errdefer if (module_owned) module.erase();
1459         const caller = try createJitCallingFunction(&ctx, "temporary_caller", "temporary_first");
1460         try module.getRegion(0).?.getEntryBlock().?.addOperation(caller);
1461         constants = try runtime.compile(module);
1462         try expectEmitterFunctionStateDiscarded(&runtime);
1463         module.erase();
1464         module_owned = false;
1465         try resource_baseline.expectEqual(&ctx);
1466 
1467         const identity_module = try createJitMemrefIdentityModule(&ctx, identity_name);
1468         var identity_owned = true;
1469         errdefer if (identity_owned) identity_module.erase();
1470         identity = try runtime.compile(identity_module);
1471         try expectEmitterFunctionStateDiscarded(&runtime);
1472         identity_module.erase();
1473         identity_owned = false;
1474         try resource_baseline.expectEqual(&ctx);
1475     }
1476 
1477     const Constant = *const fn () callconv(.c) i64;
1478     const first_entry = try runtime.getFunction(constants, "temporary_first", Constant);
1479     try testing.expectEqual(@as(i64, 41), first_entry());
1480     const caller_entry = try runtime.getFunction(constants, "temporary_caller", Constant);
1481     try testing.expectEqual(@as(i64, 41), caller_entry());
1482     const first_signature = try runtime.functionSignature(constants, "temporary_first");
1483     try testing.expect(first_signature.eql(&nullary_integer));
1484 
1485     const memref_identity = try artifact.Signature.init(&.{.memref}, &.{.memref});
1486     const identity_signature = try runtime.functionSignature(identity, identity_name);
1487     try testing.expect(identity_signature.eql(&memref_identity));
1488     const Identity = *const fn (*anyopaque) callconv(.c) *anyopaque;
1489     const identity_entry = try runtime.getFunction(identity, identity_name, Identity);
1490     const address: *anyopaque = @ptrFromInt(0x1234_5678);
1491     try testing.expectEqual(address, identity_entry(address));
1492 
1493     try runtime.release(constants);
1494     const released = runtime.getFunction(constants, "temporary_first", Constant);
1495     try testing.expectError(error.InvalidModuleHandle, released);
1496     try testing.expectError(error.InvalidModuleHandle, runtime.release(constants));
1497     const low: *anyopaque = @ptrFromInt(0x42);
1498     try testing.expectEqual(low, identity_entry(low));
1499     try runtime.release(identity);
1500     for (runtime.modules.items) |module| try testing.expect(!module.live);
1501 }
1502 
1503 test "JitRuntime recompiles keep handed-out code until release" {
1504     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1505 
1506     const testing = std.testing;
1507     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1508     defer ctx.deinit(testing.allocator);
1509     try dialects.registerAllDialects(&ctx);
1510     const one = try createJitConstantModule(&ctx, "value", 1);
1511     defer one.erase();
1512     const two = try createJitConstantModule(&ctx, "value", 2);
1513     defer two.erase();
1514 
1515     const Value = *const fn () callconv(.c) i64;
1516     var runtime = JitRuntime.init(testing.allocator, .testing);
1517     defer runtime.deinit();
1518     const first = try runtime.compile(one);
1519     const first_value = try runtime.getFunction(first, "value", Value);
1520     const second = try runtime.compile(two);
1521     try testing.expectEqual(@as(i64, 1), first_value());
1522     try testing.expectEqual(@as(i64, 2), (try runtime.getFunction(second, "value", Value))());
1523 
1524     try runtime.release(first);
1525     const third = try runtime.compile(one);
1526     try testing.expectEqual(first.index, third.index);
1527     try testing.expect(first.generation != third.generation);
1528     try testing.expectError(error.InvalidModuleHandle, runtime.functionAddress(first, "value"));
1529     try testing.expectEqual(@as(i64, 1), (try runtime.getFunction(third, "value", Value))());
1530     try testing.expectEqual(@as(i64, 2), (try runtime.getFunction(second, "value", Value))());
1531     try testing.expectError(error.FunctionNotFound, runtime.functionAddress(third, "missing"));
1532     const unissued = ModuleHandle{ .index = 2, .generation = 1 };
1533     try testing.expectError(error.InvalidModuleHandle, runtime.functionAddress(unissued, "value"));
1534 }
1535 
1536 test "JIT patches extern call targets for memref alloc/free" {
1537     const testing = std.testing;
1538     const ArithDialect = dialects.ArithDialect;
1539     const BuiltinDialect = dialects.BuiltinDialect;
1540     const FuncDialect = dialects.FuncDialect;
1541     const MemrefDialect = dialects.MemrefDialect;
1542 
1543     var arena = alloc_arena.Arena.init(std.testing.allocator);
1544     defer arena.deinit();
1545     const allocator = arena.allocator();
1546 
1547     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1548     defer ir_ctx.deinit(allocator);
1549     try dialects.registerAllDialects(&ir_ctx);
1550 
1551     const loc = ir.Location.getUnknown();
1552     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
1553     const index_type = try ArithDialect.getIndexType(&ir_ctx);
1554     const memref_type = try MemrefDialect.getMemrefTypeDynamic(&ir_ctx, i64_type, .host);
1555 
1556     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
1557     const module_block = module.getBodyBlock();
1558 
1559     var func = try FuncDialect.FuncOp.create(&ir_ctx, loc, "memref_roundtrip", &.{}, &.{i64_type});
1560     try module_block.addOperation(func.op);
1561 
1562     const entry = func.getEntryBlock();
1563     var size = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, index_type, 1);
1564     try entry.addOperation(size.op);
1565 
1566     var alloc = try MemrefDialect.AllocOp.createDynamic(&ir_ctx, loc, size.getResult(), memref_type);
1567     try entry.addOperation(alloc.op);
1568 
1569     const dealloc = try MemrefDialect.DeallocOp.create(&ir_ctx, loc, alloc.getResult());
1570     try entry.addOperation(dealloc.op);
1571 
1572     var zero = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, 0);
1573     try entry.addOperation(zero.op);
1574     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{zero.getResult()});
1575     try entry.addOperation(ret.op);
1576 
1577     var reports = diagnostics.CaptureBuffer.init(testing.allocator);
1578     defer reports.deinit();
1579     var capture = ir_ctx.captureDiagnostics(&reports);
1580     var guard = capture.enter();
1581     defer guard.deinit();
1582 
1583     var runtime = JitRuntime.init(allocator, .testing);
1584     defer runtime.deinit();
1585     try testing.expectError(error.SymbolNotFound, runtime.compile(module.op));
1586     try testing.expectEqual(@as(usize, 1), reports.diagnostics.items.len);
1587     try testing.expectEqual(func.op, reports.diagnostics.items[0].operation.?);
1588     try runtime.registerExternalSymbol("malloc", @intFromPtr(&sys.heap.malloc));
1589     try runtime.registerExternalSymbol("free", @intFromPtr(&sys.heap.free));
1590     const handle = try runtime.compile(module.op);
1591 
1592     const compiled = runtime.modules.items[handle.index];
1593     const function = compiled.functions.get("memref_roundtrip") orelse return error.FunctionNotFound;
1594     const code = compiled.code.?[function.offset..][0..function.size];
1595 
1596     var malloc_bytes: [8]u8 = undefined;
1597     std.mem.writeInt(u64, malloc_bytes[0..], @intCast(@intFromPtr(&sys.heap.malloc)), .little);
1598     var free_bytes: [8]u8 = undefined;
1599     std.mem.writeInt(u64, free_bytes[0..], @intCast(@intFromPtr(&sys.heap.free)), .little);
1600 
1601     try testing.expect(std.mem.indexOf(u8, code, malloc_bytes[0..]) != null);
1602     try testing.expect(std.mem.indexOf(u8, code, free_bytes[0..]) != null);
1603 }
1604 
1605 test "JIT treats bodyless func declarations as extern symbols" {
1606     if (!sys.capabilities.current.supportsX86_64Execution()) {
1607         return error.SkipZigTest;
1608     }
1609 
1610     const testing = std.testing;
1611     const ArithDialect = dialects.ArithDialect;
1612     const BuiltinDialect = dialects.BuiltinDialect;
1613     const FuncDialect = dialects.FuncDialect;
1614 
1615     var arena = alloc_arena.Arena.init(std.testing.allocator);
1616     defer arena.deinit();
1617     const allocator = arena.allocator();
1618 
1619     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1620     defer ir_ctx.deinit(allocator);
1621     try dialects.registerAllDialects(&ir_ctx);
1622 
1623     const loc = ir.Location.getUnknown();
1624     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
1625 
1626     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
1627     const module_block = module.getBodyBlock();
1628 
1629     const decl_name = "choir_decl_add1";
1630     const declaration = try FuncDialect.FuncOp.createDeclaration(
1631         &ir_ctx,
1632         loc,
1633         decl_name,
1634         &.{i64_type},
1635         &.{i64_type},
1636     );
1637     try module_block.addOperation(declaration.op);
1638 
1639     var caller = try FuncDialect.FuncOp.create(&ir_ctx, loc, "call_decl", &.{}, &.{i64_type});
1640     try module_block.addOperation(caller.op);
1641 
1642     const entry = caller.getEntryBlock();
1643     var forty_one = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, 41);
1644     try entry.addOperation(forty_one.op);
1645 
1646     var call = try FuncDialect.CallOp.create(&ir_ctx, loc, decl_name, &.{forty_one.getResult()}, &.{i64_type});
1647     try entry.addOperation(call.op);
1648 
1649     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{call.getResult(0).?});
1650     try entry.addOperation(ret.op);
1651 
1652     var runtime = JitRuntime.init(allocator, .testing);
1653     defer runtime.deinit();
1654     try runtime.registerExternalSymbol(decl_name, @intFromPtr(&choirDeclAdd1));
1655     const handle = try runtime.compile(module.op);
1656 
1657     try testing.expectError(error.FunctionNotFound, runtime.functionAddress(handle, decl_name));
1658     const entry_fn = try runtime.getFunction(handle, "call_decl", *const fn () callconv(.c) i64);
1659     try testing.expectEqual(@as(i64, 42), entry_fn());
1660 }
1661 
1662 test "JIT requires explicit extern symbol registration" {
1663     if (!sys.capabilities.current.supportsX86_64Execution()) {
1664         return error.SkipZigTest;
1665     }
1666 
1667     const testing = std.testing;
1668     const ArithDialect = dialects.ArithDialect;
1669     const BuiltinDialect = dialects.BuiltinDialect;
1670     const FuncDialect = dialects.FuncDialect;
1671 
1672     var arena = alloc_arena.Arena.init(std.testing.allocator);
1673     defer arena.deinit();
1674     const allocator = arena.allocator();
1675 
1676     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1677     defer ir_ctx.deinit(allocator);
1678     try dialects.registerAllDialects(&ir_ctx);
1679 
1680     const loc = ir.Location.getUnknown();
1681     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
1682 
1683     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
1684     const module_block = module.getBodyBlock();
1685 
1686     const decl_name = "choir_unregistered_external";
1687     const declaration = try FuncDialect.FuncOp.createDeclaration(
1688         &ir_ctx,
1689         loc,
1690         decl_name,
1691         &.{i64_type},
1692         &.{i64_type},
1693     );
1694     try module_block.addOperation(declaration.op);
1695 
1696     var caller = try FuncDialect.FuncOp.create(&ir_ctx, loc, "call_unregistered_decl", &.{}, &.{i64_type});
1697     try module_block.addOperation(caller.op);
1698 
1699     const entry = caller.getEntryBlock();
1700     var forty_one = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, 41);
1701     try entry.addOperation(forty_one.op);
1702 
1703     var call = try FuncDialect.CallOp.create(&ir_ctx, loc, decl_name, &.{forty_one.getResult()}, &.{i64_type});
1704     try entry.addOperation(call.op);
1705 
1706     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{call.getResult(0).?});
1707     try entry.addOperation(ret.op);
1708 
1709     var reports = diagnostics.CaptureBuffer.init(testing.allocator);
1710     defer reports.deinit();
1711     var capture = ir_ctx.captureDiagnostics(&reports);
1712     var guard = capture.enter();
1713     defer guard.deinit();
1714 
1715     var runtime = JitRuntime.init(allocator, .testing);
1716     defer runtime.deinit();
1717     try testing.expectError(error.SymbolNotFound, runtime.compile(module.op));
1718     try testing.expectEqual(@as(usize, 0), runtime.modules.items.len);
1719     try testing.expectEqual(@as(usize, 1), reports.diagnostics.items.len);
1720     const unresolved = reports.diagnostics.items[0];
1721     try testing.expectEqual(call.op, unresolved.operation.?);
1722     try testing.expectEqualStrings(failures.Stage.link.name(), unresolved.metadata[0].value);
1723 
1724     try runtime.registerExternalSymbol(decl_name, @intFromPtr(&choirDeclAdd1));
1725     const handle = try runtime.compile(module.op);
1726     const entry_fn = try runtime.getFunction(handle, "call_unregistered_decl", *const fn () callconv(.c) i64);
1727     try testing.expectEqual(@as(i64, 42), entry_fn());
1728 }
1729 
1730 test "JitRuntime rejects functions beyond the result record bound" {
1731     const testing = std.testing;
1732     const ArithDialect = dialects.ArithDialect;
1733     const FuncDialect = dialects.FuncDialect;
1734 
1735     var arena = alloc_arena.Arena.init(testing.allocator);
1736     defer arena.deinit();
1737     const allocator = arena.allocator();
1738     var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1739     defer ctx.deinit(allocator);
1740     try dialects.registerAllDialects(&ctx);
1741 
1742     const loc = ir.Location.getUnknown();
1743     const i64_type = try ArithDialect.getScalarType(&ctx, .i64);
1744     const module = try dialects.BuiltinDialect.ModuleOp.create(&ctx, loc);
1745     var results: [abi.max_result_count + 1]ir.Type = undefined;
1746     @memset(&results, i64_type);
1747     const wide = try FuncDialect.FuncOp.create(&ctx, loc, "wide", &.{i64_type}, &results);
1748     try module.getBodyBlock().addOperation(wide.op);
1749     var operands: [results.len]*ir.Value = undefined;
1750     @memset(&operands, wide.getArgument(0));
1751     const ret = try FuncDialect.ReturnOp.create(&ctx, loc, &operands);
1752     try wide.getEntryBlock().addOperation(ret.op);
1753 
1754     var reports = diagnostics.CaptureBuffer.init(testing.allocator);
1755     defer reports.deinit();
1756     var capture = ctx.captureDiagnostics(&reports);
1757     var guard = capture.enter();
1758     defer guard.deinit();
1759 
1760     var runtime = JitRuntime.init(testing.allocator, .testing);
1761     defer runtime.deinit();
1762     try testing.expectError(error.TooManyResults, runtime.compile(module.op));
1763     try testing.expectEqual(@as(usize, 0), runtime.modules.items.len);
1764     try testing.expectEqual(@as(usize, 1), reports.diagnostics.items.len);
1765     const rejected = reports.diagnostics.items[0];
1766     try testing.expectEqual(wide.op, rejected.operation.?);
1767     try testing.expectEqualStrings(failures.Stage.emit.name(), rejected.metadata[0].value);
1768 }
1769 
1770 test "JitRuntime locates emission failures at the innermost failing operation" {
1771     const testing = std.testing;
1772     const ArithDialect = dialects.ArithDialect;
1773     const FuncDialect = dialects.FuncDialect;
1774     const ScfDialect = dialects.ScfDialect;
1775 
1776     var arena = alloc_arena.Arena.init(testing.allocator);
1777     defer arena.deinit();
1778     const allocator = arena.allocator();
1779     var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1780     defer ctx.deinit(allocator);
1781     try dialects.registerAllDialects(&ctx);
1782 
1783     const loc = ir.Location.getFile("failing-jit.choir", 5, 3);
1784     const u64_type = try ArithDialect.getScalarType(&ctx, .u64);
1785     const module = try createJitConstantModule(&ctx, "kept", 7);
1786     const function = try FuncDialect.FuncOp.create(&ctx, loc, "shift", &.{}, &.{u64_type});
1787     try addJitFunction(module, function.op);
1788     const entry = function.getEntryBlock();
1789     const value = try ArithDialect.ConstantOp.createInt(&ctx, loc, u64_type, 16);
1790     try entry.addOperation(value.op);
1791     const condition = try ArithDialect.ConstantOp.createBool(&ctx, loc, true);
1792     try entry.addOperation(condition.op);
1793     const branch = try ScfDialect.IfOp.create(&ctx, loc, condition.getResult(), &.{u64_type});
1794     try entry.addOperation(branch.op);
1795     const shift = try ArithDialect.ShrOp.create(&ctx, loc, value.getResult(), value.getResult());
1796     try branch.getThenBlock().addOperation(shift.op);
1797     const shifted = try ScfDialect.YieldOp.create(&ctx, loc, &.{shift.getResult()});
1798     try branch.getThenBlock().addOperation(shifted.op);
1799     const unshifted = try ScfDialect.YieldOp.create(&ctx, loc, &.{value.getResult()});
1800     try branch.getElseBlock().?.addOperation(unshifted.op);
1801     const ret = try FuncDialect.ReturnOp.create(&ctx, loc, &.{branch.op.getResult(0).?});
1802     try entry.addOperation(ret.op);
1803 
1804     var reports = diagnostics.CaptureBuffer.init(testing.allocator);
1805     defer reports.deinit();
1806     var capture = ctx.captureDiagnostics(&reports);
1807     var guard = capture.enter();
1808     defer guard.deinit();
1809 
1810     var runtime = JitRuntime.init(testing.allocator, .testing);
1811     defer runtime.deinit();
1812     try testing.expectError(error.UnsupportedType, runtime.compile(module));
1813     try expectEmitterFunctionStateDiscarded(&runtime);
1814     try testing.expectEqual(@as(usize, 0), runtime.modules.items.len);
1815     try testing.expectEqual(@as(u64, 0), runtime.mapped_bytes);
1816     try testing.expectEqual(@as(usize, 1), reports.diagnostics.items.len);
1817     const rejected = reports.diagnostics.items[0];
1818     try testing.expectEqual(shift.op, rejected.operation.?);
1819     try testing.expectEqualStrings(failures.Stage.emit.name(), rejected.metadata[0].value);
1820 }
1821 
1822 test "JitRuntime function addresses survive module list growth" {
1823     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1824 
1825     const testing = std.testing;
1826     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1827     defer ctx.deinit(testing.allocator);
1828     try dialects.registerAllDialects(&ctx);
1829     const module = try createJitConstantModule(&ctx, "entry", 5);
1830     defer module.erase();
1831 
1832     var runtime = JitRuntime.init(testing.allocator, .testing);
1833     defer runtime.deinit();
1834     const first = try runtime.compile(module);
1835     const before = try runtime.functionAddress(first, "entry");
1836     for (0..8) |_| _ = try runtime.compile(module);
1837     try testing.expectEqual(before, try runtime.functionAddress(first, "entry"));
1838     const entry = try runtime.getFunction(first, "entry", *const fn () callconv(.c) i64);
1839     try testing.expectEqual(@as(i64, 5), entry());
1840 }
1841 
1842 test "JitRuntime admits modules up to its module count and rejects the next" {
1843     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1844 
1845     const testing = std.testing;
1846     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1847     defer ctx.deinit(testing.allocator);
1848     try dialects.registerAllDialects(&ctx);
1849     const module = try createJitConstantModule(&ctx, "entry", 7);
1850     defer module.erase();
1851 
1852     const limit = 3;
1853     var runtime = JitRuntime.init(testing.allocator, .{
1854         .live_modules = limit,
1855         .mapped_bytes = JitRuntime.Limits.testing.mapped_bytes,
1856     });
1857     defer runtime.deinit();
1858 
1859     var handles: [limit]ModuleHandle = undefined;
1860     for (&handles) |*handle| handle.* = try runtime.compile(module);
1861     try testing.expectEqual(@as(u32, limit), runtime.live_modules);
1862     try testing.expectError(error.JitRuntimeFull, runtime.compile(module));
1863 
1864     try runtime.release(handles[1]);
1865     try testing.expectEqual(@as(u32, limit - 1), runtime.live_modules);
1866     handles[1] = try runtime.compile(module);
1867     try testing.expectError(error.JitRuntimeFull, runtime.compile(module));
1868 
1869     try testing.expectEqual(@as(u32, 1), handles[1].index);
1870     try testing.expectEqual(@as(u32, 2), handles[1].generation);
1871     try testing.expectEqual(@as(usize, limit), runtime.modules.items.len);
1872 
1873     const entry = try runtime.getFunction(handles[0], "entry", *const fn () callconv(.c) i64);
1874     try testing.expectEqual(@as(i64, 7), entry());
1875 }
1876 
1877 test "JitRuntime admits modules up to its mapped-byte budget and rejects the next" {
1878     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1879 
1880     const testing = std.testing;
1881     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1882     defer ctx.deinit(testing.allocator);
1883     try dialects.registerAllDialects(&ctx);
1884     const module = try createJitConstantModule(&ctx, "entry", 9);
1885     defer module.erase();
1886 
1887     const bytes_per_module = sys.memory.pageAlign(1).?;
1888     const admitted_modules = 3;
1889     const budget: u64 = admitted_modules * bytes_per_module;
1890     var runtime = JitRuntime.init(testing.allocator, .{
1891         .live_modules = JitRuntime.Limits.testing.live_modules,
1892         .mapped_bytes = budget,
1893     });
1894     defer runtime.deinit();
1895 
1896     var handles: [admitted_modules]ModuleHandle = undefined;
1897     for (&handles) |*handle| handle.* = try runtime.compile(module);
1898     try testing.expectEqual(budget, runtime.mapped_bytes);
1899     try testing.expectEqual(@as(u32, admitted_modules), runtime.live_modules);
1900 
1901     try testing.expectError(error.JitRuntimeFull, runtime.compile(module));
1902     try testing.expectEqual(budget, runtime.mapped_bytes);
1903     try testing.expectEqual(@as(u32, admitted_modules), runtime.live_modules);
1904 
1905     try runtime.release(handles[1]);
1906     try testing.expectEqual(budget - bytes_per_module, runtime.mapped_bytes);
1907     handles[1] = try runtime.compile(module);
1908     try testing.expectEqual(budget, runtime.mapped_bytes);
1909     try testing.expectError(error.JitRuntimeFull, runtime.compile(module));
1910 
1911     const entry = try runtime.getFunction(handles[0], "entry", *const fn () callconv(.c) i64);
1912     try testing.expectEqual(@as(i64, 9), entry());
1913 }
1914 
1915 test "JitRuntime charges a module's data mapping alongside its code" {
1916     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1917 
1918     const testing = std.testing;
1919     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1920     defer ctx.deinit(testing.allocator);
1921     try dialects.registerAllDialects(&ctx);
1922     const table = machine.DataSymbol{ .name = "table", .bytes = "payload" };
1923     const module = try createJitDataModule(&ctx, "table", table);
1924     defer module.erase();
1925 
1926     const page = sys.memory.pageAlign(1).?;
1927     var runtime = JitRuntime.init(testing.allocator, .testing);
1928     defer runtime.deinit();
1929 
1930     const handle = try runtime.compile(module);
1931     try testing.expectEqual(@as(u64, 2 * page), runtime.mapped_bytes);
1932     try runtime.release(handle);
1933     try testing.expectEqual(@as(u64, 0), runtime.mapped_bytes);
1934 }
1935 
1936 test "JitRuntime counts a data mapping against the mapped-byte budget" {
1937     if (!sys.capabilities.current.supportsX86_64Execution()) return error.SkipZigTest;
1938 
1939     const testing = std.testing;
1940     var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
1941     defer ctx.deinit(testing.allocator);
1942     try dialects.registerAllDialects(&ctx);
1943     const table = machine.DataSymbol{ .name = "table", .bytes = "payload" };
1944     const module = try createJitDataModule(&ctx, "table", table);
1945     defer module.erase();
1946 
1947     const page = sys.memory.pageAlign(1).?;
1948     var runtime = JitRuntime.init(testing.allocator, .{
1949         .live_modules = JitRuntime.Limits.testing.live_modules,
1950         .mapped_bytes = 2 * page,
1951     });
1952     defer runtime.deinit();
1953 
1954     const handle = try runtime.compile(module);
1955     try testing.expectEqual(@as(u64, 2 * page), runtime.mapped_bytes);
1956     try testing.expectError(error.JitRuntimeFull, runtime.compile(module));
1957 
1958     try runtime.release(handle);
1959     try testing.expectEqual(@as(u64, 0), runtime.mapped_bytes);
1960     const again = try runtime.compile(module);
1961     try testing.expectEqual(@as(u64, 2 * page), runtime.mapped_bytes);
1962     try runtime.release(again);
1963 }
1964 
1965 test "JIT debug map includes line locations" {
1966     const testing = std.testing;
1967     const ArithDialect = dialects.ArithDialect;
1968     const BuiltinDialect = dialects.BuiltinDialect;
1969     const FuncDialect = dialects.FuncDialect;
1970 
1971     var arena = alloc_arena.Arena.init(std.testing.allocator);
1972     defer arena.deinit();
1973     const allocator = arena.allocator();
1974 
1975     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1976     defer ir_ctx.deinit(allocator);
1977     try dialects.registerAllDialects(&ir_ctx);
1978 
1979     const loc = ir.Location.getFile("test.choir", 4, 2);
1980     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
1981 
1982     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
1983     const module_block = module.getBodyBlock();
1984 
1985     var func = try FuncDialect.FuncOp.create(&ir_ctx, loc, "debug_map", &.{}, &.{i64_type});
1986     try module_block.addOperation(func.op);
1987 
1988     const entry = func.getEntryBlock();
1989     var constant = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, 7);
1990     try entry.addOperation(constant.op);
1991     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{constant.getResult()});
1992     try entry.addOperation(ret.op);
1993 
1994     var runtime = JitRuntime.init(allocator, .testing);
1995     defer runtime.deinit();
1996     const handle = try runtime.compile(module.op);
1997 
1998     var buf: [2048]u8 = undefined;
1999     var writer = std.Io.Writer.fixed(&buf);
2000 
2001     const emitted = try runtime.writeDebugMap(&writer);
2002     const output = writer.buffered();
2003 
2004     try testing.expect(emitted);
2005     try testing.expect(std.mem.containsAtLeast(u8, output, 1, "debug_map"));
2006     try testing.expect(std.mem.containsAtLeast(u8, output, 1, "test.choir:4:2"));
2007     try testing.expect(std.mem.containsAtLeast(u8, output, 1, "arith.constant"));
2008 
2009     try runtime.release(handle);
2010     var released_writer = std.Io.Writer.fixed(&buf);
2011     try testing.expect(!try runtime.writeDebugMap(&released_writer));
2012 }
2013 
2014 test "JIT debug map is best-effort when registration fails" {
2015     const testing = std.testing;
2016     const ArithDialect = dialects.ArithDialect;
2017     const BuiltinDialect = dialects.BuiltinDialect;
2018     const FuncDialect = dialects.FuncDialect;
2019 
2020     if (!debug_info.supportsJitDebugInfo(sys.capabilities.current.arch)) {
2021         return error.SkipZigTest;
2022     }
2023 
2024     var arena = alloc_arena.Arena.init(testing.allocator);
2025     defer arena.deinit();
2026 
2027     var failing = std.testing.FailingAllocator.init(arena.allocator(), .{});
2028     const allocator = failing.allocator();
2029 
2030     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2031     defer ir_ctx.deinit(allocator);
2032     try dialects.registerAllDialects(&ir_ctx);
2033 
2034     const loc = ir.Location.getFile("test.choir", 1, 1);
2035     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
2036 
2037     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
2038     const module_block = module.getBodyBlock();
2039 
2040     var func = try FuncDialect.FuncOp.create(&ir_ctx, loc, "debug_map_fail", &.{}, &.{i64_type});
2041     try module_block.addOperation(func.op);
2042 
2043     const entry = func.getEntryBlock();
2044     var constant = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, 1);
2045     try entry.addOperation(constant.op);
2046     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{constant.getResult()});
2047     try entry.addOperation(ret.op);
2048 
2049     var runtime = JitRuntime.init(allocator, .testing);
2050     defer runtime.deinit();
2051     _ = try runtime.compile(module.op);
2052 
2053     failing.fail_index = failing.alloc_index;
2054 
2055     var buf: [2048]u8 = undefined;
2056     var writer = std.Io.Writer.fixed(&buf);
2057 
2058     const emitted = try runtime.writeDebugMap(&writer);
2059     const output = writer.buffered();
2060 
2061     try testing.expect(emitted);
2062     try testing.expect(failing.has_induced_failure);
2063     try testing.expect(std.mem.containsAtLeast(u8, output, 1, "debug_map_fail"));
2064 }
2065 
2066 /// Linux x86-64 kernel entry numbers and arguments the two tests below enter the kernel with.
2067 /// They are named here rather than read from a host namespace because Choir owns no host
2068 /// surface, and because a wrong number fails the test that uses it: `getpid` answers a pid this
2069 /// process can check, and `rt_sigprocmask` answers zero only when it read a whole signal set.
2070 const linux_getpid: i64 = 39;
2071 const linux_rt_sigprocmask: i64 = 14;
2072 const linux_sig_setmask: i64 = 2;
2073 const linux_signal_set_bytes: i64 = 8;
2074 const linux_efault: i64 = -14;
2075 const linux_einval: i64 = -22;
2076 
2077 test "JIT enters the kernel through a func.syscall" {
2078     if (!sys.capabilities.current.supportsX86_64Execution()) {
2079         return error.SkipZigTest;
2080     }
2081     if (!sys.capabilities.current.isLinux()) {
2082         return error.SkipZigTest;
2083     }
2084 
2085     const testing = std.testing;
2086     const ArithDialect = dialects.ArithDialect;
2087     const BuiltinDialect = dialects.BuiltinDialect;
2088     const FuncDialect = dialects.FuncDialect;
2089 
2090     var arena = alloc_arena.Arena.init(std.testing.allocator);
2091     defer arena.deinit();
2092     const allocator = arena.allocator();
2093 
2094     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2095     defer ir_ctx.deinit(allocator);
2096     try dialects.registerAllDialects(&ir_ctx);
2097 
2098     const loc = ir.Location.getUnknown();
2099     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
2100 
2101     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
2102     const module_block = module.getBodyBlock();
2103 
2104     var func = try FuncDialect.FuncOp.create(&ir_ctx, loc, "syscall_getpid", &.{}, &.{i64_type});
2105     try module_block.addOperation(func.op);
2106 
2107     const entry = func.getEntryBlock();
2108     var number = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, linux_getpid);
2109     try entry.addOperation(number.op);
2110     const entered = try FuncDialect.SyscallOp.create(&ir_ctx, loc, number.getResult(), &.{}, i64_type);
2111     try entry.addOperation(entered.op);
2112     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{entered.getResult()});
2113     try entry.addOperation(ret.op);
2114 
2115     var runtime = JitRuntime.init(allocator, .testing);
2116     defer runtime.deinit();
2117     const handle = try runtime.compile(module.op);
2118     const entry_fn = try runtime.getFunction(handle, "syscall_getpid", *const fn () callconv(.c) i64);
2119 
2120     const expected: i64 = try sys.process.currentProcessId();
2121     try testing.expectEqual(expected, entry_fn());
2122 }
2123 
2124 test "JIT passes a func.syscall argument three in r10" {
2125     if (!sys.capabilities.current.supportsX86_64Execution()) {
2126         return error.SkipZigTest;
2127     }
2128     if (!sys.capabilities.current.isLinux()) {
2129         return error.SkipZigTest;
2130     }
2131 
2132     const testing = std.testing;
2133     const ArithDialect = dialects.ArithDialect;
2134     const BuiltinDialect = dialects.BuiltinDialect;
2135     const FuncDialect = dialects.FuncDialect;
2136 
2137     var arena = alloc_arena.Arena.init(std.testing.allocator);
2138     defer arena.deinit();
2139     const allocator = arena.allocator();
2140 
2141     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2142     defer ir_ctx.deinit(allocator);
2143     try dialects.registerAllDialects(&ir_ctx);
2144 
2145     const loc = ir.Location.getUnknown();
2146     const i64_type = try ArithDialect.getScalarType(&ir_ctx, .i64);
2147 
2148     const module = try BuiltinDialect.ModuleOp.create(&ir_ctx, loc);
2149     const module_block = module.getBodyBlock();
2150 
2151     var func = try FuncDialect.FuncOp.create(
2152         &ir_ctx,
2153         loc,
2154         "syscall_read_signal_mask",
2155         &.{ i64_type, i64_type },
2156         &.{i64_type},
2157     );
2158     try module_block.addOperation(func.op);
2159 
2160     const entry = func.getEntryBlock();
2161     var number = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, linux_rt_sigprocmask);
2162     try entry.addOperation(number.op);
2163     var how = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, linux_sig_setmask);
2164     try entry.addOperation(how.op);
2165     var no_new_mask = try ArithDialect.ConstantOp.createInt(&ir_ctx, loc, i64_type, 0);
2166     try entry.addOperation(no_new_mask.op);
2167 
2168     const arguments = [_]*ir.Value{
2169         how.getResult(),
2170         no_new_mask.getResult(),
2171         func.getArgument(0),
2172         func.getArgument(1),
2173     };
2174     const entered = try FuncDialect.SyscallOp.create(&ir_ctx, loc, number.getResult(), &arguments, i64_type);
2175     try entry.addOperation(entered.op);
2176     const ret = try FuncDialect.ReturnOp.create(&ir_ctx, loc, &.{entered.getResult()});
2177     try entry.addOperation(ret.op);
2178 
2179     var runtime = JitRuntime.init(allocator, .testing);
2180     defer runtime.deinit();
2181     const handle = try runtime.compile(module.op);
2182     const entry_fn = try runtime.getFunction(
2183         handle,
2184         "syscall_read_signal_mask",
2185         *const fn (i64, i64) callconv(.c) i64,
2186     );
2187 
2188     var observed: u64 = 0;
2189     const address: i64 = @intCast(@intFromPtr(&observed));
2190 
2191     try testing.expectEqual(@as(i64, 0), entry_fn(address, linux_signal_set_bytes));
2192     try testing.expectEqual(linux_einval, entry_fn(address, linux_signal_set_bytes - 1));
2193     try testing.expectEqual(linux_efault, entry_fn(1, linux_signal_set_bytes));
2194 }
2195 
2196 /// A constant global's bytes, as two words whose bytes a wrong offset would not reproduce.
2197 const pin_table_words = [_]u64{ 0x0102030405060708, 0x1122334455667788 };
2198 const pin_table_bytes: []const u8 = std.mem.asBytes(&pin_table_words);
2199 
2200 /// Builds a module with one zeroed global the code writes and one constant global it reads.
2201 fn buildGlobalModule(ir_ctx: *ir.Context) !dialects.BuiltinDialect.ModuleOp {
2202     const ArithDialect = dialects.ArithDialect;
2203     const BuiltinDialect = dialects.BuiltinDialect;
2204     const FuncDialect = dialects.FuncDialect;
2205     const MemrefDialect = dialects.MemrefDialect;
2206 
2207     const loc = ir.Location.getUnknown();
2208     const i64_type = try ArithDialect.getScalarType(ir_ctx, .i64);
2209     const index_type = try ArithDialect.getIndexType(ir_ctx);
2210     const word_type = try MemrefDialect.getMemrefType1D(ir_ctx, 1, i64_type, .host);
2211     const table_type = try MemrefDialect.getMemrefType1D(ir_ctx, 2, i64_type, .host);
2212 
2213     const module = try BuiltinDialect.ModuleOp.create(ir_ctx, loc);
2214     const module_block = module.getBodyBlock();
2215 
2216     const count = try MemrefDialect.GlobalOp.create(ir_ctx, loc, .{
2217         .sym_name = "count",
2218         .memref_type = word_type,
2219         .alignment = 8,
2220     });
2221     try module_block.addOperation(count.op);
2222     const table = try MemrefDialect.GlobalOp.create(ir_ctx, loc, .{
2223         .sym_name = "table",
2224         .memref_type = table_type,
2225         .alignment = 8,
2226         .constant = true,
2227         .initial = pin_table_bytes,
2228     });
2229     try module_block.addOperation(table.op);
2230 
2231     var next = try FuncDialect.FuncOp.create(ir_ctx, loc, "next", &.{}, &.{i64_type});
2232     try module_block.addOperation(next.op);
2233     {
2234         const entry = next.getEntryBlock();
2235         var zero = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, index_type, 0);
2236         try entry.addOperation(zero.op);
2237         const count_ref = try MemrefDialect.GetGlobalOp.create(ir_ctx, loc, "count", word_type);
2238         try entry.addOperation(count_ref.op);
2239         var used = try MemrefDialect.LoadOp.create(
2240             ir_ctx,
2241             loc,
2242             count_ref.getResult(),
2243             zero.getResult(),
2244             i64_type,
2245         );
2246         try entry.addOperation(used.op);
2247         var one = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, i64_type, 1);
2248         try entry.addOperation(one.op);
2249         const raised = try ArithDialect.AddOp.create(ir_ctx, loc, used.getResult(), one.getResult());
2250         try entry.addOperation(raised.op);
2251         const stored = try MemrefDialect.StoreOp.create(
2252             ir_ctx,
2253             loc,
2254             raised.getResult(),
2255             count_ref.getResult(),
2256             zero.getResult(),
2257         );
2258         try entry.addOperation(stored.op);
2259         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{used.getResult()});
2260         try entry.addOperation(ret.op);
2261     }
2262 
2263     var entry_of = try FuncDialect.FuncOp.create(ir_ctx, loc, "entry_of", &.{i64_type}, &.{i64_type});
2264     try module_block.addOperation(entry_of.op);
2265     {
2266         const entry = entry_of.getEntryBlock();
2267         const table_ref = try MemrefDialect.GetGlobalOp.create(ir_ctx, loc, "table", table_type);
2268         try entry.addOperation(table_ref.op);
2269         var word = try MemrefDialect.LoadOp.create(
2270             ir_ctx,
2271             loc,
2272             table_ref.getResult(),
2273             entry_of.getArgument(0),
2274             i64_type,
2275         );
2276         try entry.addOperation(word.op);
2277         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{word.getResult()});
2278         try entry.addOperation(ret.op);
2279     }
2280 
2281     return module;
2282 }
2283 
2284 test "JIT resolves a zeroed global and a constant one to storage" {
2285     if (!sys.capabilities.current.supportsX86_64Execution()) {
2286         return error.SkipZigTest;
2287     }
2288 
2289     const testing = std.testing;
2290     var arena = alloc_arena.Arena.init(std.testing.allocator);
2291     defer arena.deinit();
2292     const allocator = arena.allocator();
2293 
2294     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2295     defer ir_ctx.deinit(allocator);
2296     try dialects.registerAllDialects(&ir_ctx);
2297 
2298     const module = try buildGlobalModule(&ir_ctx);
2299 
2300     var runtime = JitRuntime.init(allocator, .testing);
2301     defer runtime.deinit();
2302     const handle = try runtime.compile(module.op);
2303     const next = try runtime.getFunction(handle, "next", *const fn () callconv(.c) i64);
2304     const entry_of = try runtime.getFunction(handle, "entry_of", *const fn (i64) callconv(.c) i64);
2305 
2306     try testing.expectEqual(@as(i64, 0), next());
2307     try testing.expectEqual(@as(i64, 1), next());
2308     try testing.expectEqual(@as(i64, 2), next());
2309 
2310     for (pin_table_words, 0..) |word, index| {
2311         try testing.expectEqual(word, @as(u64, @bitCast(entry_of(@intCast(index)))));
2312     }
2313 }
2314 
2315 /// The region the module-region pin declares, in bytes.
2316 const pin_arena_capacity: u64 = 4096;
2317 /// The width of the cursor word and of each value the pin promotes.
2318 const pin_word_bytes: i64 = 8;
2319 /// Values whose every byte differs, so reading the region back one byte at a time says which
2320 /// bytes the store reached and where they landed.
2321 const pin_values = [_]i64{
2322     @bitCast(@as(u64, 0x1122334455667788)),
2323     @bitCast(@as(u64, 0x99aabbccddeeff01)),
2324     @bitCast(@as(u64, 0x0123456789abcdef)),
2325 };
2326 
2327 /// Builds the module the module-region pin runs: two zero filled globals and three functions
2328 /// over them.
2329 ///
2330 /// `bump` is the promotion. `peek` and `cursor` read the result back without going through
2331 /// `memref.view`, so a view that computed the wrong address cannot agree with itself and pass.
2332 /// `peek` loads from the byte base, which the backend emits as a zero extending one byte load,
2333 /// so the caller reassembles a word from eight of them rather than trusting one load to be wide.
2334 fn buildRegionModule(ir_ctx: *ir.Context) !dialects.BuiltinDialect.ModuleOp {
2335     const ArithDialect = dialects.ArithDialect;
2336     const BuiltinDialect = dialects.BuiltinDialect;
2337     const FuncDialect = dialects.FuncDialect;
2338     const MemrefDialect = dialects.MemrefDialect;
2339 
2340     const loc = ir.Location.getUnknown();
2341     const i64_type = try ArithDialect.getScalarType(ir_ctx, .i64);
2342     const u8_type = try ArithDialect.getScalarType(ir_ctx, .u8);
2343     const index_type = try ArithDialect.getIndexType(ir_ctx);
2344     const arena_type = try MemrefDialect.getMemrefType1D(ir_ctx, pin_arena_capacity, u8_type, .host);
2345     const word_type = try MemrefDialect.getMemrefType1D(ir_ctx, 1, i64_type, .host);
2346 
2347     const module = try BuiltinDialect.ModuleOp.create(ir_ctx, loc);
2348     const module_block = module.getBodyBlock();
2349 
2350     const backing = try MemrefDialect.GlobalOp.create(ir_ctx, loc, .{
2351         .sym_name = "arena",
2352         .memref_type = arena_type,
2353         .alignment = 16,
2354     });
2355     try module_block.addOperation(backing.op);
2356     const cursor_global = try MemrefDialect.GlobalOp.create(ir_ctx, loc, .{
2357         .sym_name = "arena.cursor",
2358         .memref_type = word_type,
2359         .alignment = 8,
2360     });
2361     try module_block.addOperation(cursor_global.op);
2362 
2363     var bump = try FuncDialect.FuncOp.create(ir_ctx, loc, "bump", &.{i64_type}, &.{i64_type});
2364     try module_block.addOperation(bump.op);
2365     {
2366         const entry = bump.getEntryBlock();
2367         var zero = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, index_type, 0);
2368         try entry.addOperation(zero.op);
2369         const cursor_ref = try MemrefDialect.GetGlobalOp.create(ir_ctx, loc, "arena.cursor", word_type);
2370         try entry.addOperation(cursor_ref.op);
2371         var used = try MemrefDialect.LoadOp.create(
2372             ir_ctx,
2373             loc,
2374             cursor_ref.getResult(),
2375             zero.getResult(),
2376             i64_type,
2377         );
2378         try entry.addOperation(used.op);
2379         const arena_ref = try MemrefDialect.GetGlobalOp.create(ir_ctx, loc, "arena", arena_type);
2380         try entry.addOperation(arena_ref.op);
2381         const slot = try MemrefDialect.ViewOp.create(
2382             ir_ctx,
2383             loc,
2384             arena_ref.getResult(),
2385             used.getResult(),
2386             word_type,
2387         );
2388         try entry.addOperation(slot.op);
2389         const written = try MemrefDialect.StoreOp.create(
2390             ir_ctx,
2391             loc,
2392             bump.getArgument(0),
2393             slot.getResult(),
2394             zero.getResult(),
2395         );
2396         try entry.addOperation(written.op);
2397         var width = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, i64_type, pin_word_bytes);
2398         try entry.addOperation(width.op);
2399         const next = try ArithDialect.AddOp.create(ir_ctx, loc, used.getResult(), width.getResult());
2400         try entry.addOperation(next.op);
2401         const advanced = try MemrefDialect.StoreOp.create(
2402             ir_ctx,
2403             loc,
2404             next.getResult(),
2405             cursor_ref.getResult(),
2406             zero.getResult(),
2407         );
2408         try entry.addOperation(advanced.op);
2409         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{used.getResult()});
2410         try entry.addOperation(ret.op);
2411     }
2412 
2413     var peek = try FuncDialect.FuncOp.create(ir_ctx, loc, "peek", &.{i64_type}, &.{i64_type});
2414     try module_block.addOperation(peek.op);
2415     {
2416         const entry = peek.getEntryBlock();
2417         const arena_ref = try MemrefDialect.GetGlobalOp.create(ir_ctx, loc, "arena", arena_type);
2418         try entry.addOperation(arena_ref.op);
2419         var byte = try MemrefDialect.LoadOp.create(
2420             ir_ctx,
2421             loc,
2422             arena_ref.getResult(),
2423             peek.getArgument(0),
2424             i64_type,
2425         );
2426         try entry.addOperation(byte.op);
2427         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{byte.getResult()});
2428         try entry.addOperation(ret.op);
2429     }
2430 
2431     var cursor = try FuncDialect.FuncOp.create(ir_ctx, loc, "cursor", &.{}, &.{i64_type});
2432     try module_block.addOperation(cursor.op);
2433     {
2434         const entry = cursor.getEntryBlock();
2435         var zero = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, index_type, 0);
2436         try entry.addOperation(zero.op);
2437         const cursor_ref = try MemrefDialect.GetGlobalOp.create(ir_ctx, loc, "arena.cursor", word_type);
2438         try entry.addOperation(cursor_ref.op);
2439         var used = try MemrefDialect.LoadOp.create(
2440             ir_ctx,
2441             loc,
2442             cursor_ref.getResult(),
2443             zero.getResult(),
2444             i64_type,
2445         );
2446         try entry.addOperation(used.op);
2447         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{used.getResult()});
2448         try entry.addOperation(ret.op);
2449     }
2450 
2451     return module;
2452 }
2453 
2454 /// Reassembles the little endian word at `offset` from eight one byte reads of the region.
2455 fn readRegionWord(peek: *const fn (i64) callconv(.c) i64, offset: i64) u64 {
2456     var word: u64 = 0;
2457     for (0..8) |index| {
2458         const byte: u64 = @intCast(peek(offset + @as(i64, @intCast(index))));
2459         std.debug.assert(byte <= 0xff);
2460         word |= byte << @intCast(index * 8);
2461     }
2462     return word;
2463 }
2464 
2465 test "JIT promotes into a module region through writable globals" {
2466     if (!sys.capabilities.current.supportsX86_64Execution()) {
2467         return error.SkipZigTest;
2468     }
2469 
2470     const testing = std.testing;
2471     var arena = alloc_arena.Arena.init(std.testing.allocator);
2472     defer arena.deinit();
2473     const allocator = arena.allocator();
2474 
2475     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2476     defer ir_ctx.deinit(allocator);
2477     try dialects.registerAllDialects(&ir_ctx);
2478 
2479     const module = try buildRegionModule(&ir_ctx);
2480 
2481     var runtime = JitRuntime.init(allocator, .testing);
2482     defer runtime.deinit();
2483     const handle = try runtime.compile(module.op);
2484     const bump = try runtime.getFunction(handle, "bump", *const fn (i64) callconv(.c) i64);
2485     const peek = try runtime.getFunction(handle, "peek", *const fn (i64) callconv(.c) i64);
2486     const cursor = try runtime.getFunction(handle, "cursor", *const fn () callconv(.c) i64);
2487 
2488     try testing.expectEqual(@as(i64, 0), cursor());
2489     for (pin_values, 0..) |value, index| {
2490         const expected: i64 = @intCast(index * @as(usize, @intCast(pin_word_bytes)));
2491         try testing.expectEqual(expected, bump(value));
2492     }
2493     try testing.expectEqual(@as(i64, 24), cursor());
2494 
2495     for (pin_values, 0..) |value, index| {
2496         const offset: i64 = @intCast(index * @as(usize, @intCast(pin_word_bytes)));
2497         try testing.expectEqual(@as(u64, @bitCast(value)), readRegionWord(peek, offset));
2498     }
2499     try testing.expectEqual(@as(u64, 0), readRegionWord(peek, 24));
2500 }
2501 
2502 /// The values the live-across-a-call fixture carries past its call, and the argument it passes.
2503 ///
2504 /// Four rather than one, because one could survive by luck in whichever register the callee
2505 /// happened not to touch. Four is more than the callee's own working set, so a call site that
2506 /// left them in volatile registers would lose at least one of them.
2507 const across_addends = [_]i64{ 7, 9, 11, 13 };
2508 const across_argument: i64 = 5;
2509 
2510 /// Builds a function whose values are live across a call without being arguments to it.
2511 ///
2512 /// `callee` writes every volatile register it can reach by doing arithmetic, so if the call site
2513 /// were responsible for preserving a live value and stopped doing it, the sum below would come
2514 /// back wrong rather than merely slower.
2515 fn buildAcrossModule(ir_ctx: *ir.Context) !dialects.BuiltinDialect.ModuleOp {
2516     const ArithDialect = dialects.ArithDialect;
2517     const BuiltinDialect = dialects.BuiltinDialect;
2518     const FuncDialect = dialects.FuncDialect;
2519 
2520     const loc = ir.Location.getUnknown();
2521     const i64_type = try ArithDialect.getScalarType(ir_ctx, .i64);
2522 
2523     const module = try BuiltinDialect.ModuleOp.create(ir_ctx, loc);
2524     const module_block = module.getBodyBlock();
2525 
2526     var callee = try FuncDialect.FuncOp.create(ir_ctx, loc, "callee", &.{i64_type}, &.{i64_type});
2527     try module_block.addOperation(callee.op);
2528     {
2529         const entry = callee.getEntryBlock();
2530         var running = callee.getArgument(0);
2531         for (across_addends) |addend| {
2532             var step = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, i64_type, addend);
2533             try entry.addOperation(step.op);
2534             const sum = try ArithDialect.AddOp.create(ir_ctx, loc, running, step.getResult());
2535             try entry.addOperation(sum.op);
2536             running = sum.getResult();
2537         }
2538         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{running});
2539         try entry.addOperation(ret.op);
2540     }
2541 
2542     var across = try FuncDialect.FuncOp.create(ir_ctx, loc, "across", &.{i64_type}, &.{i64_type});
2543     try module_block.addOperation(across.op);
2544     {
2545         const entry = across.getEntryBlock();
2546         var live: [across_addends.len]*ir.Value = undefined;
2547         for (across_addends, 0..) |addend, index| {
2548             var step = try ArithDialect.ConstantOp.createInt(ir_ctx, loc, i64_type, addend);
2549             try entry.addOperation(step.op);
2550             const sum = try ArithDialect.AddOp.create(
2551                 ir_ctx,
2552                 loc,
2553                 across.getArgument(0),
2554                 step.getResult(),
2555             );
2556             try entry.addOperation(sum.op);
2557             live[index] = sum.getResult();
2558         }
2559 
2560         const call = try FuncDialect.CallOp.create(
2561             ir_ctx,
2562             loc,
2563             "callee",
2564             &.{across.getArgument(0)},
2565             &.{i64_type},
2566         );
2567         try entry.addOperation(call.op);
2568 
2569         var total = call.getResult(0).?;
2570         for (live) |value| {
2571             const sum = try ArithDialect.AddOp.create(ir_ctx, loc, total, value);
2572             try entry.addOperation(sum.op);
2573             total = sum.getResult();
2574         }
2575         const ret = try FuncDialect.ReturnOp.create(ir_ctx, loc, &.{total});
2576         try entry.addOperation(ret.op);
2577     }
2578 
2579     return module;
2580 }
2581 
2582 test "JIT keeps values live across a call without the call site saving them" {
2583     if (!sys.capabilities.current.supportsX86_64Execution()) {
2584         return error.SkipZigTest;
2585     }
2586 
2587     const testing = std.testing;
2588     var arena = alloc_arena.Arena.init(std.testing.allocator);
2589     defer arena.deinit();
2590     const allocator = arena.allocator();
2591 
2592     var ir_ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2593     defer ir_ctx.deinit(allocator);
2594     try dialects.registerAllDialects(&ir_ctx);
2595 
2596     const module = try buildAcrossModule(&ir_ctx);
2597 
2598     var runtime = JitRuntime.init(allocator, .testing);
2599     defer runtime.deinit();
2600     const handle = try runtime.compile(module.op);
2601     const across = try runtime.getFunction(handle, "across", *const fn (i64) callconv(.c) i64);
2602 
2603     var expected: i64 = across_argument;
2604     for (across_addends) |addend| expected += addend;
2605     for (across_addends) |addend| expected += across_argument + addend;
2606 
2607     try testing.expectEqual(expected, across(across_argument));
2608 }
2609 
2610 test "JitRuntime admits external symbols at each preset and refuses one past" {
2611     const testing = std.testing;
2612     for ([_]JitRuntime.Limits{ .testing, .standard }) |limits| {
2613         var failing = testing.FailingAllocator.init(testing.allocator, .{});
2614         var runtime = JitRuntime.init(failing.allocator(), limits);
2615         defer runtime.deinit();
2616         var name: [32]u8 = undefined;
2617         for (0..limits.external_symbols) |index| {
2618             const symbol = try std.fmt.bufPrint(&name, "external_{d}", .{index});
2619             try runtime.registerExternalSymbol(symbol, index + 1);
2620         }
2621         try testing.expectEqual(limits.external_symbols, runtime.external_symbols.count());
2622         const bytes = failing.allocated_bytes - failing.freed_bytes;
2623         const allocations = failing.alloc_index;
2624         const metadata = runtime.external_symbols.metadata;
2625         const table_capacity = runtime.external_symbols.capacity();
2626         const first = runtime.external_symbols.getEntry("external_0").?;
2627         const first_name = first.key_ptr.*.ptr;
2628 
2629         try testing.expectError(
2630             error.JitRuntimeFull,
2631             runtime.registerExternalSymbol("one_past", 1),
2632         );
2633         try testing.expectEqual(limits.external_symbols, runtime.external_symbols.count());
2634         try testing.expect(!runtime.external_symbols.contains("one_past"));
2635         try testing.expectEqual(bytes, failing.allocated_bytes - failing.freed_bytes);
2636         try testing.expectEqual(allocations, failing.alloc_index);
2637         try testing.expectEqual(metadata, runtime.external_symbols.metadata);
2638         try testing.expectEqual(table_capacity, runtime.external_symbols.capacity());
2639 
2640         failing.fail_index = failing.alloc_index;
2641         try runtime.registerExternalSymbol("external_0", 99);
2642         try testing.expectEqual(@as(?usize, 99), runtime.external_symbols.get("external_0"));
2643         try testing.expectEqual(
2644             first_name,
2645             runtime.external_symbols.getEntry("external_0").?.key_ptr.*.ptr,
2646         );
2647         try testing.expectEqual(limits.external_symbols, runtime.external_symbols.count());
2648         try testing.expectEqual(allocations, failing.alloc_index);
2649         try testing.expectEqual(bytes, failing.allocated_bytes - failing.freed_bytes);
2650     }
2651 }
2652 
2653 test "JitRuntime failed external registration preserves storage and permits retry" {
2654     const testing = std.testing;
2655     for (0..2) |fail_offset| {
2656         var failing = testing.FailingAllocator.init(testing.allocator, .{});
2657         var runtime = JitRuntime.init(failing.allocator(), .testing);
2658         defer runtime.deinit();
2659         try runtime.registerExternalSymbol("external_0", 1);
2660         const initial_entries = runtime.external_symbols.capacity() *
2661             std.hash_map.default_max_load_percentage / 100;
2662         var name: [32]u8 = undefined;
2663         for (1..initial_entries) |index| {
2664             const symbol = try std.fmt.bufPrint(&name, "external_{d}", .{index});
2665             try runtime.registerExternalSymbol(symbol, index + 1);
2666         }
2667         const bytes = failing.allocated_bytes - failing.freed_bytes;
2668         const metadata = runtime.external_symbols.metadata;
2669         const table_capacity = runtime.external_symbols.capacity();
2670         failing.fail_index = failing.alloc_index + fail_offset;
2671         try testing.expectError(
2672             error.OutOfMemory,
2673             runtime.registerExternalSymbol("next", initial_entries + 1),
2674         );
2675         try testing.expect(failing.has_induced_failure);
2676         try testing.expectEqual(@as(u32, initial_entries), runtime.external_symbols.count());
2677         try testing.expect(!runtime.external_symbols.contains("next"));
2678         try testing.expectEqual(bytes, failing.allocated_bytes - failing.freed_bytes);
2679         try testing.expectEqual(metadata, runtime.external_symbols.metadata);
2680         try testing.expectEqual(table_capacity, runtime.external_symbols.capacity());
2681         for (0..initial_entries) |index| {
2682             const symbol = try std.fmt.bufPrint(&name, "external_{d}", .{index});
2683             try testing.expectEqual(@as(?usize, index + 1), runtime.external_symbols.get(symbol));
2684         }
2685         failing.fail_index = std.math.maxInt(usize);
2686         try runtime.registerExternalSymbol("next", initial_entries + 1);
2687         try testing.expectEqual(@as(u32, initial_entries + 1), runtime.external_symbols.count());
2688         try testing.expectEqual(
2689             @as(?usize, initial_entries + 1),
2690             runtime.external_symbols.get("next"),
2691         );
2692     }
2693 }
2694 
2695 test "JitRuntime zero external symbol limit refuses before allocating" {
2696     const testing = std.testing;
2697     var limits: JitRuntime.Limits = .testing;
2698     limits.external_symbols = 0;
2699     var failing = testing.FailingAllocator.init(testing.allocator, .{ .fail_index = 0 });
2700     var runtime = JitRuntime.init(failing.allocator(), limits);
2701     defer runtime.deinit();
2702     try testing.expectError(error.JitRuntimeFull, runtime.registerExternalSymbol("first", 1));
2703     try testing.expectEqual(@as(u32, 0), runtime.external_symbols.count());
2704     try testing.expectEqual(@as(usize, 0), failing.alloc_index);
2705     try testing.expect(!failing.has_induced_failure);
2706 }