lib/choir/src/backends/aarch64/backend.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const alloc_arena = @import("alloc_arena");
3 const ir = @import("../../core/root.zig");
4 const dialects = @import("../../dialects/root.zig");
5 const artifact = @import("../root.zig").artifact;
6 const boundary = @import("../root.zig").signature;
7 const contract = @import("../root.zig").contract;
8 const interface = @import("../root.zig").interface;
9 const encoding = @import("encoding/root.zig");
10 const registers = @import("registers/root.zig");
11 const disasm = @import("disassemble/root.zig");
12 const sys = @import("sys");
13 const control = @import("root.zig").control;
14 const placement = @import("root.zig").placement;
15 const diagnostics = @import("../../root.zig").diagnostics;
16
17 const Allocator = std.mem.Allocator;
18 const BackendError = interface.BackendError;
19 const ExecuteResult = interface.ExecuteResult;
20 const ArithDialect = dialects.ArithDialect;
21 const BuiltinDialect = dialects.BuiltinDialect;
22 const FuncDialect = dialects.FuncDialect;
23 const GPR = registers.GPR;
24 const Instruction = encoding.Instruction;
25
26 pub const supports_native_execution = sys.capabilities.current.supportsAarch64Execution();
27
28 pub const max_arguments: usize = 8;
29 pub const max_results: usize = 1;
30
31 pub const Backend = struct {
32 allocator: Allocator,
33 ctx: *ir.Context,
34 compiled_module: ?*ir.Operation = null,
35
36 pub fn init(allocator: Allocator, ctx: *ir.Context) Allocator.Error!Backend {
37 try contract.loadArithDialect(ctx);
38 return .{
39 .allocator = allocator,
40 .ctx = ctx,
41 };
42 }
43
44 pub fn deinit(_: *Backend) void {}
45
46 pub fn verify(_: *Backend, module: *ir.Operation) BackendError!void {
47 try contract.verifyModule("backend/aarch64/verify", module);
48 _ = module.walk(.{ .order = .pre_order }, {}, refuseOverflow) catch
49 return error.UnsupportedOperation;
50 }
51
52 pub fn lower(self: *Backend, module: *ir.Operation) BackendError!*ir.Operation {
53 try self.verify(module);
54 return module;
55 }
56
57 pub fn compile(self: *Backend, module: *ir.Operation) BackendError!void {
58 self.compiled_module = try self.lower(module);
59 }
60
61 pub fn compileFunctionToMachineCode(
62 self: *Backend,
63 module: *ir.Operation,
64 function_name: []const u8,
65 ) BackendError![]u8 {
66 const lowered = try self.lower(module);
67 const func = ir.inspection.functionDefinitionByName(lowered, function_name) orelse return BackendError.FunctionNotFound;
68
69 var emitter = Emitter.init(self.allocator);
70 defer emitter.deinit();
71 try emitter.emitFunction(func);
72
73 const code = emitter.code.items;
74 if (code.len == 0) return BackendError.CodeGenFailed;
75 return self.allocator.dupe(u8, code) catch BackendError.OutOfMemory;
76 }
77
78 pub fn compileFunctionToArtifact(
79 self: *Backend,
80 module: *ir.Operation,
81 function_name: []const u8,
82 ) BackendError!artifact.Artifact {
83 const func = ir.inspection.functionDefinitionByName(module, function_name) orelse
84 return BackendError.FunctionNotFound;
85 const signature = boundary.ofFunction(func) catch return BackendError.UnsupportedOperation;
86 const code = try self.compileFunctionToMachineCode(module, function_name);
87 defer self.allocator.free(code);
88
89 return artifact.machineCodeArtifact(
90 self.allocator,
91 .{
92 .architecture = .aarch64,
93 .triple = "aarch64-unknown-unknown",
94 .cpu = "generic",
95 },
96 .{
97 .name = "AAPCS64",
98 .calling_convention = "aapcs64",
99 .pointer_width_bits = 64,
100 .endianness = .little,
101 },
102 function_name,
103 signature,
104 code,
105 &.{},
106 &.{},
107 ) catch BackendError.OutOfMemory;
108 }
109
110 pub fn emitFunction(self: *Backend, module: *ir.Operation, function_name: []const u8) BackendError![]u8 {
111 return self.compileFunctionToMachineCode(module, function_name);
112 }
113
114 pub fn emit(
115 self: *Backend,
116 module: *ir.Operation,
117 options: interface.EmitOptions,
118 writer: *std.Io.Writer,
119 ) BackendError!void {
120 const entry = options.entry orelse return BackendError.FunctionNotFound;
121 const code = try self.compileFunctionToMachineCode(module, entry);
122 defer self.allocator.free(code);
123 writer.writeAll(code) catch return BackendError.CodeGenFailed;
124 }
125
126 pub fn executeJit(self: *Backend, entry: []const u8, args: []const i64) BackendError!ExecuteResult {
127 if (!supports_native_execution) return BackendError.UnsupportedArchitecture;
128
129 const module = self.compiled_module orelse return BackendError.NoCompiledModule;
130 const func = ir.inspection.functionDefinitionByName(module, entry) orelse return BackendError.FunctionNotFound;
131 const function = FuncDialect.FuncOp{ .op = func };
132 if (args.len != function.getNumArguments()) {
133 return refuse(func, "aarch64 call argument count does not match the function signature", BackendError.ExecutionFailed);
134 }
135 const code = try self.compileFunctionToMachineCode(module, entry);
136 defer self.allocator.free(code);
137
138 return executeMachineCode(code, args, function.getNumResults() != 0);
139 }
140
141 pub fn disassemble(_: *Backend, code: []const u8, writer: *std.Io.Writer) BackendError!void {
142 if (code.len % Instruction.size != 0) return BackendError.CodeGenFailed;
143
144 var offset: usize = 0;
145 while (offset < code.len) : (offset += Instruction.size) {
146 if (offset != 0) writer.writeByte('\n') catch return BackendError.CodeGenFailed;
147 const raw = std.mem.readInt(u32, code[offset..][0..4], .little);
148 var buf: [96]u8 = undefined;
149 const text = disasm.formatInstructionRawWithDetail(raw, &buf, true);
150 writer.writeAll(text) catch return BackendError.CodeGenFailed;
151 }
152 }
153 };
154
155 /// Refuse overflow roots even when their results are unused or outside the selected function.
156 fn refuseOverflow(_: void, op: *ir.Operation) error{UnsupportedOperation}!ir.WalkResult {
157 inline for (.{ ArithDialect.AddoOp, ArithDialect.SuboOp, ArithDialect.MuloOp }) |Op| {
158 if (std.mem.eql(u8, op.name.name, Op.operation_name)) return error.UnsupportedOperation;
159 }
160 return .advance;
161 }
162
163 pub fn initHandle(allocator: Allocator, ctx: *ir.Context) BackendError!interface.BackendHandle {
164 return interface.initHandle(
165 Backend,
166 allocator,
167 ctx,
168 interface.BackendTarget.aarch64,
169 "aarch64",
170 .{
171 .artifact = .{ .machine_code = true },
172 .cpu = .{ .disassemble = true },
173 },
174 );
175 }
176
177 const Emitter = struct {
178 allocator: Allocator,
179 code: std.ArrayListUnmanaged(u8) = .empty,
180 plan: placement.Plan = .{},
181
182 fn init(allocator: Allocator) Emitter {
183 return .{ .allocator = allocator };
184 }
185
186 fn deinit(self: *Emitter) void {
187 self.code.deinit(self.allocator);
188 }
189
190 /// Emits the verified finite block in order, including unused effects.
191 fn emitFunction(self: *Emitter, func: *ir.Operation) BackendError!void {
192 const function = FuncDialect.FuncOp{ .op = func };
193 if (!function.hasBody() or function.getBody().blocks.size != 1) {
194 return refuse(func, "aarch64 emission requires a single defined block", BackendError.UnsupportedOperation);
195 }
196 if (function.getNumArguments() > max_arguments) {
197 return refuse(func, std.fmt.comptimePrint("aarch64 emission exceeds {d} arguments", .{max_arguments}), BackendError.UnsupportedOperation);
198 }
199 if (function.getNumResults() > max_results) {
200 return refuse(func, std.fmt.comptimePrint("aarch64 emission exceeds {d} result", .{max_results}), BackendError.UnsupportedOperation);
201 }
202 self.plan.build(function.getEntryBlock()) catch |err| {
203 const message = switch (err) {
204 error.ValueCapacity => std.fmt.comptimePrint("aarch64 placement exceeds {d} tracked values", .{placement.max_values}),
205 error.FrameCapacity => std.fmt.comptimePrint("aarch64 placement exceeds {d} frame slots", .{placement.max_frame_slots}),
206 error.OperationCapacity => std.fmt.comptimePrint("aarch64 placement exceeds {d} operations", .{placement.max_operations}),
207 error.ControlShape => "aarch64 structured control requires matching single-block regions and terminal yield/condition",
208 error.ControlType => "aarch64 structured control requires scalar integer values, bool conditions and index induction",
209 error.CarriedCapacity => std.fmt.comptimePrint("aarch64 control exceeds {d} carried or region-result values", .{control.max_carried_values}),
210 error.DepthCapacity => std.fmt.comptimePrint("aarch64 control exceeds {d} nested constructs", .{control.max_depth}),
211 error.MissingValue => "aarch64 operand has no preceding definition",
212 };
213 return refuse(self.plan.refused_operation.?, message, BackendError.UnsupportedOperation);
214 };
215 if (self.plan.frameBytes() != 0) {
216 try self.emitInstruction(encoding.AddSubImmediate.sub(registers.SP.encoded, registers.SP.encoded, @intCast(self.plan.frameBytes()), .x));
217 }
218 for (function.getArguments(), 0..) |arg, index| {
219 try self.normalize(argumentRegister(@intCast(index)), try scalarIntegerKind(func, arg.type));
220 }
221 for (function.getResultTypes()) |typ| _ = try scalarIntegerKind(func, typ);
222
223 try self.emitSchedule(function);
224 }
225
226 const ControlFrame = struct {
227 op: *ir.Operation,
228 header: usize = 0,
229 branch: usize = 0,
230 merge: usize = 0,
231 condition: ?*ir.Operation = null,
232 };
233
234 /// The placement schedule owns nesting. Emission has no recursive calls.
235 fn emitSchedule(self: *Emitter, function: FuncDialect.FuncOp) BackendError!void {
236 var frames: [control.max_depth]ControlFrame = undefined;
237 for (self.plan.events[0..self.plan.event_count]) |event| {
238 const op = event.op;
239 if (event.kind == .next_region) {
240 const frame = &frames[event.depth];
241 switch (control.kind(op).?) {
242 .conditional => {
243 frame.merge = try self.branch();
244 try self.patchBranch(op, frame.branch, self.code.items.len, .eq);
245 },
246 .repeated => {
247 const condition = frame.condition.?;
248 const value = try self.location(condition, condition.operands.items[0].value, .x16);
249 try self.emitInstruction(encoding.AddSubImmediate.cmpImm(value.encode(), 0, .x));
250 frame.branch = try self.branch();
251 try self.transfer(condition, condition.getOperandValues()[1..], .{ .arguments = op.regions.items[1].getEntryBlock().?.arguments.items });
252 },
253 .counted => unreachable,
254 }
255 continue;
256 }
257 if (event.kind == .end) {
258 const frame = &frames[event.depth];
259 switch (control.kind(op).?) {
260 .conditional => try self.patchBranch(op, if (op.regions.items.len == 1) frame.branch else frame.merge, self.code.items.len, if (op.regions.items.len == 1) .eq else null),
261 .counted => {
262 try self.jump(op, frame.header);
263 try self.patchBranch(op, frame.branch, self.code.items.len, .ge);
264 const block = op.regions.items[0].getEntryBlock().?;
265 try self.transfer(op, block.arguments.items[1..], .{ .results = op.results.items });
266 },
267 .repeated => {
268 try self.jump(op, frame.header);
269 try self.patchBranch(op, frame.branch, self.code.items.len, .eq);
270 const condition = frame.condition.?;
271 try self.transfer(condition, condition.getOperandValues()[1..], .{ .results = op.results.items });
272 },
273 }
274 continue;
275 }
276 if (control.kind(op)) |construct| {
277 const frame = &frames[event.depth];
278 frame.* = .{ .op = op };
279 switch (construct) {
280 .conditional => {
281 const value = try self.location(op, op.operands.items[0].value, .x16);
282 try self.emitInstruction(encoding.AddSubImmediate.cmpImm(value.encode(), 0, .x));
283 frame.branch = try self.branch();
284 },
285 .counted => {
286 const block = op.regions.items[0].getEntryBlock().?;
287 var sources: [control.max_transfers]*ir.Value = undefined;
288 sources[0] = op.operands.items[0].value;
289 for (op.operands.items[3..], 1..) |operand, index| sources[index] = operand.value;
290 try self.transfer(op, sources[0..block.arguments.items.len], .{ .arguments = block.arguments.items });
291 frame.header = self.code.items.len;
292 const iv = try self.location(op, block.arguments.items[0], .x16);
293 const upper = try self.location(op, op.operands.items[1].value, .x17);
294 try self.emitInstruction(encoding.AddSubShiftedRegister.cmpReg(iv.encode(), upper.encode(), .x));
295 frame.branch = try self.branch();
296 },
297 .repeated => {
298 try self.transfer(op, op.getOperandValues(), .{ .arguments = op.regions.items[0].getEntryBlock().?.arguments.items });
299 frame.header = self.code.items.len;
300 },
301 }
302 continue;
303 }
304 if (control.is(op, dialects.ScfDialect.YieldOp.operation_name)) {
305 if (event.depth == 0) return refuse(op, "aarch64 yield requires a structured region", BackendError.UnsupportedOperation);
306 const parent = frames[event.depth - 1].op;
307 if (op.next_op != null) return refuse(op, "aarch64 yield must terminate its region", BackendError.UnsupportedOperation);
308 switch (control.kind(parent).?) {
309 .conditional => try self.transfer(op, op.getOperandValues(), .{ .results = parent.results.items }),
310 .counted => {
311 const block = parent.regions.items[0].getEntryBlock().?;
312 try self.transfer(op, op.getOperandValues(), .{ .arguments = block.arguments.items[1..] });
313 const iv = block.arguments.items[0];
314 const value = try self.location(op, iv, .x16);
315 const step = try self.location(op, parent.operands.items[2].value, .x17);
316 const target = self.destination(iv);
317 try self.emitInstruction(encoding.AddSubShiftedRegister.addReg(target.encode(), value.encode(), step.encode(), .x));
318 try self.store(iv, target);
319 },
320 .repeated => {
321 const before = parent.regions.items[0].getEntryBlock().?;
322 if (op != control.last(parent.regions.items[1].getEntryBlock().?)) return refuse(op, "aarch64 while yield must terminate the after region", BackendError.UnsupportedOperation);
323 try self.transfer(op, op.getOperandValues(), .{ .arguments = before.arguments.items });
324 },
325 }
326 continue;
327 }
328 if (control.is(op, dialects.ScfDialect.ConditionOp.operation_name)) {
329 if (event.depth == 0) return refuse(op, "aarch64 condition requires while before region", BackendError.UnsupportedOperation);
330 const frame = &frames[event.depth - 1];
331 if (control.kind(frame.op).? != .repeated or op != control.last(frame.op.regions.items[0].getEntryBlock().?)) return refuse(op, "aarch64 condition must terminate while before region", BackendError.UnsupportedOperation);
332 frame.condition = op;
333 continue;
334 }
335 if (control.is(op, FuncDialect.ReturnOp.operation_name)) {
336 if (event.depth != 0 or op.next_op != null) return refuse(op, "aarch64 return must end the function block", BackendError.UnsupportedOperation);
337 std.debug.assert(op.operands.items.len == function.getNumResults());
338 if (op.operands.items.len == 1) {
339 const source = try self.location(op, op.operands.items[0].value, .x8);
340 try self.move(.x0, source);
341 }
342 if (self.plan.frameBytes() != 0) try self.emitInstruction(encoding.AddSubImmediate.add(registers.SP.encoded, registers.SP.encoded, @intCast(self.plan.frameBytes()), .x));
343 try self.emitInstruction(encoding.UnconditionalBranchReg.retLr());
344 return;
345 }
346 try self.emitOperation(op);
347 }
348 return refuse(function.op, "aarch64 function requires func.return", BackendError.UnsupportedOperation);
349 }
350
351 const Destinations = union(enum) {
352 arguments: []const *ir.Value,
353 results: []ir.Value,
354
355 fn len(self: Destinations) usize {
356 return switch (self) {
357 .arguments => |values| values.len,
358 .results => |values| values.len,
359 };
360 }
361
362 fn at(self: Destinations, index: usize) *ir.Value {
363 return switch (self) {
364 .arguments => |values| values[index],
365 .results => |values| &values[index],
366 };
367 }
368 };
369
370 /// Resolve parallel assignments before overwriting any source home.
371 /// Each pass consumes a move or breaks a cycle, taking at most 2*N passes.
372 fn transfer(self: *Emitter, op: *ir.Operation, sources: []const *ir.Value, destinations: Destinations) BackendError!void {
373 std.debug.assert(sources.len == destinations.len());
374 std.debug.assert(sources.len <= control.max_transfers);
375 const Move = struct { source: placement.Location, target: placement.Location, pending: bool };
376 var moves: [control.max_transfers]Move = undefined;
377 var remaining: usize = 0;
378 for (sources, 0..) |source, index| {
379 const target = self.plan.get(destinations.at(index)).?;
380 const home = try self.valueHome(op, source);
381 const pending = target != .dead and !sameHome(home, target);
382 moves[index] = .{ .source = home, .target = target, .pending = pending };
383 remaining += @intFromBool(pending);
384 }
385 for (0..2 * control.max_transfers) |_| {
386 if (remaining == 0) return;
387 var progressed = false;
388 for (moves[0..sources.len]) |*move_item| {
389 if (!move_item.pending) continue;
390 var blocked = false;
391 for (moves[0..sources.len]) |other| {
392 if (other.pending and sameHome(move_item.target, other.source)) blocked = true;
393 }
394 if (blocked) continue;
395 try self.copyHome(move_item.source, move_item.target, .x16);
396 move_item.pending = false;
397 remaining -= 1;
398 progressed = true;
399 }
400 if (progressed) continue;
401 for (moves[0..sources.len]) |move_item| {
402 if (!move_item.pending) continue;
403 try self.copyHome(move_item.source, .{ .register = .x8 }, .x16);
404 for (moves[0..sources.len]) |*other| {
405 if (other.pending and sameHome(other.source, move_item.source)) other.source = .{ .register = .x8 };
406 }
407 break;
408 }
409 }
410 unreachable;
411 }
412
413 fn sameHome(lhs: placement.Location, rhs: placement.Location) bool {
414 return switch (lhs) {
415 .register => |reg| rhs == .register and rhs.register == reg,
416 .stack => |slot| rhs == .stack and rhs.stack == slot,
417 .dead => rhs == .dead,
418 };
419 }
420
421 fn valueHome(self: *Emitter, op: *ir.Operation, value: *ir.Value) BackendError!placement.Location {
422 if (self.plan.get(value)) |result| return result;
423 if (value.getOwnerBlock() == @as(*anyopaque, self.plan.entry_block.?) and value.kind.block_argument.arg_number < max_arguments) return .{ .register = argumentRegister(@intCast(value.kind.block_argument.arg_number)) };
424 return refuse(op, "aarch64 operand has no placement", BackendError.CodeGenFailed);
425 }
426
427 fn copyHome(self: *Emitter, source: placement.Location, target: placement.Location, scratch: GPR) BackendError!void {
428 const register = switch (source) {
429 .register => |reg| reg,
430 .stack => |slot| load: {
431 try self.emitInstruction(encoding.LoadStoreRegImm.ldr(scratch.encode(), registers.SP.encoded, @intCast(slot), .x));
432 break :load scratch;
433 },
434 .dead => unreachable,
435 };
436 switch (target) {
437 .register => |reg| try self.move(reg, register),
438 .stack => |slot| try self.emitInstruction(encoding.LoadStoreRegImm.str(register.encode(), registers.SP.encoded, @intCast(slot), .x)),
439 .dead => unreachable,
440 }
441 }
442
443 fn branch(self: *Emitter) BackendError!usize {
444 const offset = self.code.items.len;
445 try self.emitInstruction(encoding.UnconditionalBranchImm.b(0));
446 return offset;
447 }
448
449 fn jump(self: *Emitter, op: *ir.Operation, target: usize) BackendError!void {
450 const offset = try self.branch();
451 try self.patchBranch(op, offset, target, null);
452 }
453
454 fn patchBranch(self: *Emitter, op: *ir.Operation, offset: usize, target: usize, condition: ?encoding.Condition) BackendError!void {
455 std.debug.assert(offset % Instruction.size == 0);
456 std.debug.assert(target % Instruction.size == 0);
457 const distance = @divExact(@as(i64, @intCast(target)) - @as(i64, @intCast(offset)), Instruction.size);
458 const inst = if (condition) |cond|
459 encoding.ConditionalBranch.bCond(cond, std.math.cast(i19, distance) orelse return refuse(op, "aarch64 conditional branch exceeds signed 19-bit instruction displacement", BackendError.UnsupportedOperation))
460 else
461 encoding.UnconditionalBranchImm.b(std.math.cast(i26, distance) orelse return refuse(op, "aarch64 branch exceeds signed 26-bit instruction displacement", BackendError.UnsupportedOperation));
462 inst.write(self.code.items[offset..][0..Instruction.size]);
463 }
464
465 const ScalarOp = enum {
466 constant,
467 add,
468 sub,
469 mul,
470 div,
471 rem,
472 @"and",
473 @"or",
474 xor,
475 not,
476 shl,
477 shr,
478 ushr,
479 cmp,
480 select,
481 min,
482 max,
483 abs,
484 neg,
485 cast,
486 umulhi,
487 popcount,
488 };
489
490 /// Reads temporary operands before writing a potentially recycled home.
491 fn emitOperation(self: *Emitter, op: *ir.Operation) BackendError!void {
492 const admitted = @import("root.zig").admission.lookup(op.name.name);
493 if (admitted == null or admitted.?.status != .admitted) {
494 return refuse(op, "aarch64 emission defers this operation", BackendError.UnsupportedOperation);
495 }
496 const operation = std.meta.stringToEnum(ScalarOp, op.name.name["arith.".len..]) orelse
497 return refuse(op, "aarch64 emission defers this operation", BackendError.UnsupportedOperation);
498 std.debug.assert(op.results.items.len == 1);
499 const value = op.getResult(0).?;
500 const kind = try scalarIntegerKind(op, value.type);
501 for (op.operands.items) |operand| _ = try scalarIntegerKind(op, operand.value.type);
502 if (operation == .constant) {
503 const integer = (ArithDialect.ConstantOp{ .op = op }).getIntValue() orelse boolean: {
504 const attr = op.getAttrAs(ir.Attribute.BoolAttr, "value") orelse
505 return refuse(op, "aarch64 constant requires an integer or boolean value", BackendError.UnsupportedOperation);
506 break :boolean @as(i64, @intFromBool(attr.getValue()));
507 };
508 if (value.hasNoUses()) return;
509 const target = self.destination(value);
510 try self.emitMoveImmediate(target, @bitCast(integer));
511 try self.normalize(target, kind);
512 try self.store(value, target);
513 return;
514 }
515 try self.legalScalar(op, operation, kind);
516 const temporaries = [_]GPR{ .x16, .x17, .x8 };
517 std.debug.assert(op.operands.items.len <= temporaries.len);
518 for (op.operands.items, 0..) |operand, index| {
519 const source = try self.location(op, operand.value, temporaries[index]);
520 try self.move(temporaries[index], source);
521 }
522 try self.emitRequirements(op, operation, kind);
523 if (value.hasNoUses()) return;
524 const target = self.destination(value);
525 const bits = dialects.arith.scalarBitWidth(kind);
526 switch (operation) {
527 .constant => unreachable,
528 .add => try self.emitInstruction(encoding.AddSubShiftedRegister.addReg(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)),
529 .sub => try self.emitInstruction(encoding.AddSubShiftedRegister.subReg(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)),
530 .mul => try self.emitInstruction(encoding.DataProcessing3Source.mul(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)),
531 .div, .rem => {
532 const quotient = if (operation == .rem) GPR.x8 else target;
533 const inst = if (unsignedKind(kind))
534 encoding.DataProcessing2Source.udiv(quotient.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)
535 else
536 encoding.DataProcessing2Source.sdiv(quotient.encode(), GPR.x16.encode(), GPR.x17.encode(), .x);
537 try self.emitInstruction(inst);
538 if (operation == .rem) try self.emitInstruction(encoding.DataProcessing3Source.msub(target.encode(), quotient.encode(), GPR.x17.encode(), GPR.x16.encode(), .x));
539 },
540 .@"and" => try self.emitInstruction(encoding.LogicalShiftedRegister.andReg(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)),
541 .@"or" => try self.emitInstruction(encoding.LogicalShiftedRegister.orrReg(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)),
542 .xor => try self.emitInstruction(encoding.LogicalShiftedRegister.eorReg(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x)),
543 .not => {
544 try self.emitMoveImmediate(.x17, std.math.maxInt(u64));
545 try self.emitInstruction(encoding.LogicalShiftedRegister.eorReg(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x));
546 },
547 .neg => try self.emitInstruction(encoding.AddSubShiftedRegister.subReg(target.encode(), registers.ZR.encoded, GPR.x16.encode(), .x)),
548 .abs => {
549 if (unsignedKind(kind)) {
550 try self.move(target, .x16);
551 } else {
552 try self.emitInstruction(encoding.AddSubImmediate.cmpImm(GPR.x16.encode(), 0, .x));
553 try self.emitInstruction(encoding.AddSubShiftedRegister.subReg(GPR.x17.encode(), registers.ZR.encoded, GPR.x16.encode(), .x));
554 try self.emitInstruction(encoding.ConditionalSelect.csel(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .ge, .x));
555 }
556 },
557 .shl, .shr, .ushr => {
558 if (operation != .shl) try self.extend(.x16, bits, operation == .shr);
559 var inst = encoding.DataProcessing2Source.udiv(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x);
560 inst.data_processing_reg.data_proc_2src.opcode = switch (operation) {
561 .shl => .lslv,
562 .shr => .asrv,
563 .ushr => .lsrv,
564 else => unreachable,
565 };
566 try self.emitInstruction(inst);
567 },
568 .min, .max => {
569 try self.emitInstruction(encoding.AddSubShiftedRegister.cmpReg(GPR.x16.encode(), GPR.x17.encode(), .x));
570 const condition: encoding.Condition = if (unsignedKind(kind))
571 (if (operation == .min) .ls else .hs)
572 else
573 (if (operation == .min) .le else .ge);
574 try self.emitInstruction(encoding.ConditionalSelect.csel(target.encode(), GPR.x16.encode(), GPR.x17.encode(), condition, .x));
575 },
576 .select => {
577 try self.emitInstruction(encoding.AddSubImmediate.cmpImm(GPR.x16.encode(), 0, .x));
578 try self.emitInstruction(encoding.ConditionalSelect.csel(target.encode(), GPR.x17.encode(), GPR.x8.encode(), .ne, .x));
579 },
580 .cmp => {
581 const predicate = (ArithDialect.CmpOp{ .op = op }).getPredicate().?;
582 const operand_kind = try scalarIntegerKind(op, op.operands.items[0].value.type);
583 const operand_bits = dialects.arith.scalarBitWidth(operand_kind);
584 const signed = switch (predicate) {
585 .lt, .le, .gt, .ge, .slt, .sle, .sgt, .sge => true,
586 else => false,
587 };
588 try self.extend(.x16, operand_bits, signed);
589 try self.extend(.x17, operand_bits, signed);
590 try self.emitInstruction(encoding.AddSubShiftedRegister.cmpReg(GPR.x16.encode(), GPR.x17.encode(), .x));
591 const condition: encoding.Condition = switch (predicate) {
592 .eq => .eq,
593 .ne => .ne,
594 .lt, .slt => .lt,
595 .le, .sle => .le,
596 .gt, .sgt => .gt,
597 .ge, .sge => .ge,
598 .ult => .lo,
599 .ule => .ls,
600 .ugt => .hi,
601 .uge => .hs,
602 };
603 try self.emitInstruction(encoding.ConditionalSelect.cset(target.encode(), condition, .x));
604 },
605 .cast => try self.move(target, .x16),
606 .umulhi => {
607 try self.extend(.x16, bits, false);
608 try self.extend(.x17, bits, false);
609 if (bits == 64) {
610 try self.emitInstruction(.{ .raw = 0x9bc07c00 | (@as(u32, @backingInt(GPR.x17)) << 16) | (@as(u32, @backingInt(GPR.x16)) << 5) | @as(u32, @backingInt(target)) });
611 } else {
612 try self.emitInstruction(encoding.DataProcessing3Source.mul(target.encode(), GPR.x16.encode(), GPR.x17.encode(), .x));
613 try self.logicalRight(target, target, @intCast(bits));
614 }
615 },
616 .popcount => try self.popcount(target, bits),
617 }
618 try self.normalize(target, kind);
619 try self.store(value, target);
620 }
621
622 fn move(self: *Emitter, target: GPR, source: GPR) BackendError!void {
623 if (target != source) try self.emitInstruction(encoding.LogicalShiftedRegister.movReg(target.encode(), source.encode(), .x));
624 }
625
626 fn isDivision(operation: ScalarOp) bool {
627 return operation == .div or operation == .rem;
628 }
629
630 fn isShift(operation: ScalarOp) bool {
631 return operation == .shl or operation == .shr or operation == .ushr;
632 }
633
634 fn unsignedKind(kind: dialects.arith.ScalarKind) bool {
635 return dialects.arith.scalarKindIsUnsignedInteger(kind) or kind == .index;
636 }
637
638 /// Admission follows the dialect's complete scalar declaration.
639 fn legalScalar(_: *Emitter, op: *ir.Operation, operation: ScalarOp, kind: dialects.arith.ScalarKind) BackendError!void {
640 if (kind == .bool) switch (operation) {
641 .@"and", .@"or", .xor, .not, .cmp, .select, .cast => {},
642 else => return refuse(op, "aarch64 arithmetic requires an integer operand", BackendError.UnsupportedOperation),
643 };
644 if (operation == .cast) {
645 const source = try scalarIntegerKind(op, op.operands.items[0].value.type);
646 if ((source == .bool or kind == .bool) and source != kind) {
647 return refuse(op, "aarch64 cast does not admit integer boolean conversions", BackendError.UnsupportedOperation);
648 }
649 }
650 if (operation == .cmp) {
651 const predicate = (ArithDialect.CmpOp{ .op = op }).getPredicate() orelse
652 return refuse(op, "aarch64 comparison requires a predicate", BackendError.UnsupportedOperation);
653 const source = try scalarIntegerKind(op, op.operands.items[0].value.type);
654 if (source == .bool and predicate != .eq and predicate != .ne) {
655 return refuse(op, "aarch64 boolean comparison requires eq or ne", BackendError.UnsupportedOperation);
656 }
657 }
658 }
659
660 /// UBFM Xd, Xn, #shift, #63 is a logical right shift.
661 fn logicalRight(self: *Emitter, target: GPR, source: GPR, shift: u6) BackendError!void {
662 try self.emitInstruction(.{ .raw = 0xd340fc00 | (@as(u32, shift) << 16) | (@as(u32, @backingInt(source)) << 5) | @as(u32, @backingInt(target)) });
663 }
664
665 /// Fixed SWAR reduction uses only the three caller-saved temporaries.
666 fn popcount(self: *Emitter, target: GPR, bits: u8) BackendError!void {
667 try self.extend(.x16, bits, false);
668 try self.logicalRight(.x8, .x16, 1);
669 try self.emitMoveImmediate(.x17, 0x5555555555555555);
670 try self.emitInstruction(encoding.LogicalShiftedRegister.andReg(GPR.x8.encode(), GPR.x8.encode(), GPR.x17.encode(), .x));
671 try self.emitInstruction(encoding.AddSubShiftedRegister.subReg(GPR.x16.encode(), GPR.x16.encode(), GPR.x8.encode(), .x));
672 try self.logicalRight(.x8, .x16, 2);
673 try self.emitMoveImmediate(.x17, 0x3333333333333333);
674 try self.emitInstruction(encoding.LogicalShiftedRegister.andReg(GPR.x8.encode(), GPR.x8.encode(), GPR.x17.encode(), .x));
675 try self.emitInstruction(encoding.LogicalShiftedRegister.andReg(GPR.x16.encode(), GPR.x16.encode(), GPR.x17.encode(), .x));
676 try self.emitInstruction(encoding.AddSubShiftedRegister.addReg(GPR.x16.encode(), GPR.x16.encode(), GPR.x8.encode(), .x));
677 try self.logicalRight(.x8, .x16, 4);
678 try self.emitInstruction(encoding.AddSubShiftedRegister.addReg(GPR.x16.encode(), GPR.x16.encode(), GPR.x8.encode(), .x));
679 try self.emitMoveImmediate(.x17, 0x0f0f0f0f0f0f0f0f);
680 try self.emitInstruction(encoding.LogicalShiftedRegister.andReg(GPR.x16.encode(), GPR.x16.encode(), GPR.x17.encode(), .x));
681 inline for (.{ 8, 16, 32 }) |shift| {
682 try self.logicalRight(.x8, .x16, shift);
683 try self.emitInstruction(encoding.AddSubShiftedRegister.addReg(GPR.x16.encode(), GPR.x16.encode(), GPR.x8.encode(), .x));
684 }
685 try self.extend(.x16, 7, false);
686 try self.move(target, .x16);
687 }
688
689 /// Seven facts are the scalar dialect's six entries plus its one result.
690 const max_scalar_facts = 7;
691
692 fn emitRequirements(self: *Emitter, op: *ir.Operation, operation: ScalarOp, kind: dialects.arith.ScalarKind) BackendError!void {
693 const facts = ir.interfaces.effects;
694 var storage: [max_scalar_facts]facts.Fact = undefined;
695 const declaration = facts.collectInto(op, &storage) catch
696 return refuse(op, "aarch64 scalar effect declaration cannot be collected", BackendError.UnsupportedOperation);
697 if (!declaration.complete) return refuse(op, "aarch64 scalar effect declaration is incomplete", BackendError.UnsupportedOperation);
698 const bits = dialects.arith.scalarBitWidth(kind);
699 for (declaration.records) |record| switch (record) {
700 .requirement => |requirement| switch (requirement.kind) {
701 .nonzero => {
702 if (literalInteger(op.operands.items[1].value)) |rhs| {
703 if (dialects.arith.scalar.maskToBits(rhs, bits) == 0) return refuse(op, "aarch64 DivisionByZero is statically certain", BackendError.UnsupportedOperation);
704 }
705 try self.emitInstruction(encoding.AddSubImmediate.cmpImm(GPR.x17.encode(), 0, .x));
706 try self.requireCondition(.ne);
707 },
708 .quotient_representable => {
709 std.debug.assert(isDivision(operation));
710 const minimum: i64 = @intCast(dialects.arith.scalar.intLimits(bits).min);
711 if (literalInteger(op.operands.items[0].value)) |lhs| {
712 if (literalInteger(op.operands.items[1].value)) |rhs| {
713 if (dialects.arith.scalar.truncate(lhs, bits) == minimum and dialects.arith.scalar.truncate(rhs, bits) == -1) {
714 return refuse(op, "aarch64 SignedDivisionOverflow is statically certain", BackendError.UnsupportedOperation);
715 }
716 }
717 }
718 try self.emitMoveImmediate(.x8, @bitCast(minimum));
719 try self.emitInstruction(encoding.AddSubShiftedRegister.cmpReg(GPR.x16.encode(), GPR.x8.encode(), .x));
720 const skip = self.code.items.len;
721 try self.emitInstruction(encoding.ConditionalBranch.bCond(.ne, 0));
722 try self.emitMoveImmediate(.x8, std.math.maxInt(u64));
723 try self.emitInstruction(encoding.AddSubShiftedRegister.cmpReg(GPR.x17.encode(), GPR.x8.encode(), .x));
724 try self.requireCondition(.ne);
725 const distance: i19 = @intCast((self.code.items.len - skip) / Instruction.size);
726 encoding.ConditionalBranch.bCond(.ne, distance).write(self.code.items[skip..][0..Instruction.size]);
727 },
728 .in_bounds => {
729 std.debug.assert(isShift(operation));
730 if (literalInteger(op.operands.items[1].value)) |count| {
731 if (dialects.arith.scalar.shiftCount(count, bits) == null) return refuse(op, "aarch64 InvalidShiftAmount is statically certain", BackendError.UnsupportedOperation);
732 }
733 try self.emitInstruction(encoding.AddSubImmediate.cmpImm(GPR.x17.encode(), bits, .x));
734 try self.requireCondition(.lo);
735 },
736 else => return refuse(op, "aarch64 scalar requirement is unsupported", BackendError.UnsupportedOperation),
737 },
738 else => {},
739 };
740 }
741
742 fn literalInteger(value: *ir.Value) ?i64 {
743 const raw = value.getDefiningOp() orelse return null;
744 const op: *ir.Operation = @ptrCast(@alignCast(raw));
745 if (!std.mem.eql(u8, op.name.name, ArithDialect.ConstantOp.operation_name)) return null;
746 return (ArithDialect.ConstantOp{ .op = op }).getIntValue();
747 }
748
749 /// AArch64 BRK #0 raises SIGTRAP on Linux. The successful edge skips it.
750 fn requireCondition(self: *Emitter, condition: encoding.Condition) BackendError!void {
751 try self.emitInstruction(encoding.ConditionalBranch.bCond(condition, 2));
752 try self.emitInstruction(.{ .raw = 0xd4200000 });
753 }
754
755 fn location(self: *Emitter, op: *ir.Operation, value: *ir.Value, temporary: GPR) BackendError!GPR {
756 const home = try self.valueHome(op, value);
757 return switch (home) {
758 .register => |register| register,
759 .stack => |slot| blk: {
760 std.debug.assert(slot < self.plan.frame_slots);
761 try self.emitInstruction(encoding.LoadStoreRegImm.ldr(temporary.encode(), registers.SP.encoded, @intCast(slot), .x));
762 break :blk temporary;
763 },
764 .dead => unreachable,
765 };
766 }
767
768 fn destination(self: *const Emitter, value: *ir.Value) GPR {
769 return switch (self.plan.get(value).?) {
770 .register => |register| register,
771 .stack => .x8,
772 .dead => unreachable,
773 };
774 }
775
776 fn store(self: *Emitter, value: *ir.Value, register: GPR) BackendError!void {
777 switch (self.plan.get(value).?) {
778 .stack => |slot| {
779 std.debug.assert(slot < self.plan.frame_slots);
780 try self.emitInstruction(encoding.LoadStoreRegImm.str(register.encode(), registers.SP.encoded, @intCast(slot), .x));
781 },
782 .register => |home| std.debug.assert(home == register),
783 .dead => unreachable,
784 }
785 }
786
787 /// SBFM/UBFM Xd, Xn, #0, #(width-1) extends the low declared bits.
788 fn normalize(self: *Emitter, register: GPR, kind: dialects.arith.ScalarKind) BackendError!void {
789 const scalar = dialects.arith.scalarDescriptor(kind);
790 std.debug.assert(scalar.class != .float);
791 std.debug.assert(scalar.bit_width > 0);
792 std.debug.assert(scalar.bit_width <= 64);
793 if (scalar.bit_width == 64) return;
794
795 try self.extend(register, scalar.bit_width, scalar.class == .signed_integer);
796 }
797
798 fn extend(self: *Emitter, register: GPR, bits: u8, signed: bool) BackendError!void {
799 std.debug.assert(bits > 0);
800 std.debug.assert(bits <= 64);
801 if (bits == 64) return;
802 const opcode: u32 = if (signed) 0x93400000 else 0xd3400000;
803 const encoded: u32 = @backingInt(register.encode());
804 try self.emitInstruction(.{ .raw = opcode | (@as(u32, bits - 1) << 10) | (encoded << 5) | encoded });
805 }
806
807 fn emitMoveImmediate(self: *Emitter, target: GPR, value: u64) BackendError!void {
808 if (value == 0) {
809 try self.emitInstruction(encoding.MoveWide.movz(target.encode(), 0, 0, .x));
810 return;
811 }
812
813 var first_chunk: ?u2 = null;
814 for (0..4) |index| {
815 const shift: u6 = @intCast(index * 16);
816 const chunk: u16 = @truncate(value >> shift);
817 if (chunk == 0) continue;
818 const hw: u2 = @intCast(index);
819 first_chunk = hw;
820 try self.emitInstruction(encoding.MoveWide.movz(target.encode(), chunk, hw, .x));
821 break;
822 }
823
824 const first = first_chunk orelse unreachable;
825 for (0..4) |index| {
826 const hw: u2 = @intCast(index);
827 if (hw == first) continue;
828 const shift: u6 = @intCast(index * 16);
829 const chunk: u16 = @truncate(value >> shift);
830 if (chunk == 0) continue;
831 try self.emitInstruction(encoding.MoveWide.movk(target.encode(), chunk, hw, .x));
832 }
833 }
834
835 fn emitInstruction(self: *Emitter, inst: Instruction) BackendError!void {
836 const old_len = self.code.items.len;
837 self.code.resize(self.allocator, old_len + Instruction.size) catch return BackendError.OutOfMemory;
838 inst.write(self.code.items[old_len..][0..Instruction.size]);
839 }
840 };
841
842 /// Publishes the refused operation's exact name and location in its diagnostic.
843 fn refuse(op: *ir.Operation, message: []const u8, err: BackendError) BackendError {
844 var diagnostic = op.emitError(message);
845 defer diagnostic.deinit();
846 _ = diagnostic.emit() catch {};
847 return err;
848 }
849
850 fn scalarIntegerKind(op: *ir.Operation, typ: ir.Type) BackendError!dialects.arith.ScalarKind {
851 const name = typ.getDialectTypeName() orelse return refuse(op, "aarch64 emission requires a scalar integer type", BackendError.UnsupportedOperation);
852 const kind = dialects.arith.scalarKindFromTypeName(name) orelse return refuse(op, "aarch64 emission requires a scalar integer type", BackendError.UnsupportedOperation);
853 return switch (kind) {
854 .i8, .i16, .i32, .i64, .u8, .u16, .u32, .u64, .index, .bool => kind,
855 else => refuse(op, "aarch64 emission requires a scalar integer type", BackendError.UnsupportedOperation),
856 };
857 }
858
859 fn argumentRegister(index: u3) GPR {
860 return switch (index) {
861 0 => .x0,
862 1 => .x1,
863 2 => .x2,
864 3 => .x3,
865 4 => .x4,
866 5 => .x5,
867 6 => .x6,
868 7 => .x7,
869 };
870 }
871
872 fn executeMachineCode(code: []const u8, args: []const i64, has_result: bool) BackendError!ExecuteResult {
873 std.debug.assert(args.len <= max_arguments);
874 std.debug.assert(code.len > 0);
875
876 var memory = sys.memory.mapAnonymous(code.len, .{ .read = true, .write = true }) catch return BackendError.ExecutionFailed;
877 defer sys.memory.unmap(memory);
878
879 @memcpy(memory[0..code.len], code);
880 sys.memory.protect(memory, .{ .read = true, .execute = true }) catch return BackendError.ExecutionFailed;
881
882 if (has_result) return .{ .int = invokeMachineCode(i64, memory.ptr, args) };
883 invokeMachineCode(void, memory.ptr, args);
884 return .void_;
885 }
886
887 fn invokeMachineCode(comptime Result: type, address: [*]u8, args: []const i64) Result {
888 std.debug.assert(args.len <= max_arguments);
889 return switch (args.len) {
890 0 => blk: {
891 const func: *const fn () callconv(.c) Result = @ptrCast(@alignCast(address));
892 break :blk func();
893 },
894 1 => blk: {
895 const func: *const fn (i64) callconv(.c) Result = @ptrCast(@alignCast(address));
896 break :blk func(args[0]);
897 },
898 2 => blk: {
899 const func: *const fn (i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
900 break :blk func(args[0], args[1]);
901 },
902 3 => blk: {
903 const func: *const fn (i64, i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
904 break :blk func(args[0], args[1], args[2]);
905 },
906 4 => blk: {
907 const func: *const fn (i64, i64, i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
908 break :blk func(args[0], args[1], args[2], args[3]);
909 },
910 5 => blk: {
911 const func: *const fn (i64, i64, i64, i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
912 break :blk func(args[0], args[1], args[2], args[3], args[4]);
913 },
914 6 => blk: {
915 const func: *const fn (i64, i64, i64, i64, i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
916 break :blk func(args[0], args[1], args[2], args[3], args[4], args[5]);
917 },
918 7 => blk: {
919 const func: *const fn (i64, i64, i64, i64, i64, i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
920 break :blk func(args[0], args[1], args[2], args[3], args[4], args[5], args[6]);
921 },
922 8 => blk: {
923 const func: *const fn (i64, i64, i64, i64, i64, i64, i64, i64) callconv(.c) Result = @ptrCast(@alignCast(address));
924 break :blk func(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
925 },
926 else => unreachable,
927 };
928 }
929
930 fn makeConstantReturnModule(ctx: *ir.Context, value: i64) !*ir.Operation {
931 const loc = ir.Location.getUnknown();
932 const i64_type = try ArithDialect.getScalarType(ctx, .i64);
933
934 const module = try BuiltinDialect.ModuleOp.create(ctx, loc);
935 var func = try FuncDialect.FuncOp.create(ctx, loc, "ret_const", &.{}, &.{i64_type});
936 try module.getBodyBlock().addOperation(func.op);
937
938 const entry = func.getEntryBlock();
939 var c = try ArithDialect.ConstantOp.createInt(ctx, loc, i64_type, value);
940 try entry.addOperation(c.op);
941 const ret = try FuncDialect.ReturnOp.create(ctx, loc, &.{c.getResult()});
942 try entry.addOperation(ret.op);
943
944 return module.op;
945 }
946
947 fn makeAddArgsModule(ctx: *ir.Context) !*ir.Operation {
948 const loc = ir.Location.getUnknown();
949 const i64_type = try ArithDialect.getScalarType(ctx, .i64);
950
951 const module = try BuiltinDialect.ModuleOp.create(ctx, loc);
952 var func = try FuncDialect.FuncOp.create(ctx, loc, "add_args", &.{ i64_type, i64_type }, &.{i64_type});
953 try module.getBodyBlock().addOperation(func.op);
954
955 const entry = func.getEntryBlock();
956 var add = try ArithDialect.AddOp.create(ctx, loc, func.getArgument(0), func.getArgument(1));
957 try entry.addOperation(add.op);
958 const ret = try FuncDialect.ReturnOp.create(ctx, loc, &.{add.getResult()});
959 try entry.addOperation(ret.op);
960
961 return module.op;
962 }
963
964 test "aarch64 backend handle exposes artifacts and debug" {
965 var arena = alloc_arena.Arena.init(std.testing.allocator);
966 defer arena.deinit();
967 const allocator = arena.allocator();
968
969 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
970 defer ctx.deinit(allocator);
971 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
972
973 var handle = try initHandle(allocator, &ctx);
974 defer handle.deinit();
975
976 try std.testing.expect(interface.BackendTarget.aarch64.eql(handle.target));
977 try std.testing.expect(handle.capabilities.artifact.machine_code);
978 try std.testing.expect(handle.capabilities.cpu.disassemble);
979 }
980
981 test "aarch64 backend emits golden scalar constant return" {
982 var arena = alloc_arena.Arena.init(std.testing.allocator);
983 defer arena.deinit();
984 const allocator = arena.allocator();
985
986 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
987 defer ctx.deinit(allocator);
988 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
989
990 const module = try makeConstantReturnModule(&ctx, 42);
991 var backend = try Backend.init(allocator, &ctx);
992 defer backend.deinit();
993
994 const code = try backend.compileFunctionToMachineCode(module, "ret_const");
995 defer allocator.free(code);
996
997 const expected = [_]u8{
998 0x49, 0x05, 0x80, 0xd2,
999 0xe0, 0x03, 0x09, 0xaa,
1000 0xc0, 0x03, 0x5f, 0xd6,
1001 };
1002 try std.testing.expectEqualSlices(u8, &expected, code);
1003 }
1004
1005 test "aarch64 backend emits add of two AAPCS64 integer arguments" {
1006 var arena = alloc_arena.Arena.init(std.testing.allocator);
1007 defer arena.deinit();
1008 const allocator = arena.allocator();
1009
1010 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1011 defer ctx.deinit(allocator);
1012 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1013
1014 const module = try makeAddArgsModule(&ctx);
1015 var backend = try Backend.init(allocator, &ctx);
1016 defer backend.deinit();
1017
1018 const code = try backend.compileFunctionToMachineCode(module, "add_args");
1019 defer allocator.free(code);
1020
1021 const expected = [_]u8{
1022 0xf0, 0x03, 0x00, 0xaa,
1023 0xf1, 0x03, 0x01, 0xaa,
1024 0x09, 0x02, 0x11, 0x8b,
1025 0xe0, 0x03, 0x09, 0xaa,
1026 0xc0, 0x03, 0x5f, 0xd6,
1027 };
1028 try std.testing.expectEqualSlices(u8, &expected, code);
1029 }
1030
1031 test "aarch64 backend disassembles the scalar subset" {
1032 var arena = alloc_arena.Arena.init(std.testing.allocator);
1033 defer arena.deinit();
1034 const allocator = arena.allocator();
1035
1036 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1037 defer ctx.deinit(allocator);
1038 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1039
1040 const module = try makeConstantReturnModule(&ctx, 42);
1041 var backend = try Backend.init(allocator, &ctx);
1042 defer backend.deinit();
1043
1044 const code = try backend.compileFunctionToMachineCode(module, "ret_const");
1045 defer allocator.free(code);
1046
1047 var buf: [128]u8 = undefined;
1048 var writer = std.Io.Writer.fixed(&buf);
1049 try backend.disassemble(code, &writer);
1050
1051 try std.testing.expectEqualStrings(
1052 "movz x9, #0x2a\nmov x0, x9\nret",
1053 writer.buffered(),
1054 );
1055 }
1056
1057 test "aarch64 backend records machine-code artifact metadata" {
1058 var arena = alloc_arena.Arena.init(std.testing.allocator);
1059 defer arena.deinit();
1060 const allocator = arena.allocator();
1061
1062 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1063 defer ctx.deinit(allocator);
1064 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1065
1066 const module = try makeConstantReturnModule(&ctx, 42);
1067 var backend = try Backend.init(allocator, &ctx);
1068 defer backend.deinit();
1069
1070 var out = try backend.compileFunctionToArtifact(module, "ret_const");
1071 defer out.deinit();
1072
1073 try std.testing.expectEqual(artifact.ArtifactKind.machine_code, out.metadata.kind);
1074 try std.testing.expectEqual(artifact.Architecture.aarch64, out.metadata.target.architecture);
1075 try std.testing.expectEqual(@as(u16, 64), out.metadata.abi.pointer_width_bits.?);
1076 try std.testing.expectEqual(artifact.BufferFormat.machine_code, out.payload.buffers.items[0].format);
1077 }
1078
1079 test "aarch64 native execution runs only on AArch64 hosts" {
1080 if (!supports_native_execution) return error.SkipZigTest;
1081
1082 var arena = alloc_arena.Arena.init(std.testing.allocator);
1083 defer arena.deinit();
1084 const allocator = arena.allocator();
1085
1086 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1087 defer ctx.deinit(allocator);
1088 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1089
1090 const module = try makeAddArgsModule(&ctx);
1091 var backend = try Backend.init(allocator, &ctx);
1092 defer backend.deinit();
1093
1094 const result = try compileAndRun(&backend, module, "add_args", &.{ 40, 2 });
1095 try std.testing.expectEqual(@as(i64, 42), result.asInt().?);
1096 }
1097
1098 /// Keep execution witnesses behind this seam for the subsequent runtime contract unit.
1099 fn compileAndRun(
1100 backend: *Backend,
1101 module: *ir.Operation,
1102 entry: []const u8,
1103 args: []const i64,
1104 ) !ExecuteResult {
1105 try backend.compile(module);
1106 return backend.executeJit(entry, args);
1107 }
1108
1109 const Witness = struct {
1110 arena: alloc_arena.Arena,
1111 ctx: ir.Context,
1112 backend: Backend,
1113 module: *ir.Operation,
1114 function: FuncDialect.FuncOp,
1115 diagnostic_operation: []const u8 = "",
1116 diagnostic_message: []const u8 = "",
1117
1118 fn init(self: *Witness, kind: dialects.arith.ScalarKind, arguments: usize, results: usize) !void {
1119 return self.initSignature(kind, kind, arguments, results);
1120 }
1121
1122 fn initSignature(self: *Witness, argument_kind: dialects.arith.ScalarKind, kind: dialects.arith.ScalarKind, arguments: usize, results: usize) !void {
1123 self.arena = alloc_arena.Arena.init(std.testing.allocator);
1124 const allocator = self.arena.allocator();
1125 self.ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1126 try dialects.registerAllDialects(&self.ctx);
1127 self.backend = try Backend.init(allocator, &self.ctx);
1128 self.diagnostic_operation = "";
1129 self.diagnostic_message = "";
1130 _ = try self.ctx.registerDiagnosticHandler(.{ .context = self, .handle = recordDiagnostic });
1131 const typ = try ArithDialect.getScalarType(&self.ctx, kind);
1132 var types: [max_arguments + 1]ir.Type = @splat(typ);
1133 var argument_types: [max_arguments + 1]ir.Type = @splat(try ArithDialect.getScalarType(&self.ctx, argument_kind));
1134 std.debug.assert(arguments <= types.len);
1135 std.debug.assert(results <= types.len);
1136 const module = try BuiltinDialect.ModuleOp.create(&self.ctx, .unknown);
1137 self.module = module.op;
1138 self.function = try FuncDialect.FuncOp.create(&self.ctx, .unknown, "witness", argument_types[0..arguments], types[0..results]);
1139 try module.getBodyBlock().addOperation(self.function.op);
1140 }
1141
1142 fn deinit(self: *Witness) void {
1143 self.backend.deinit();
1144 self.ctx.deinit(self.arena.allocator());
1145 self.arena.deinit();
1146 }
1147
1148 fn recordDiagnostic(context: ?*anyopaque, diagnostic: *const diagnostics.Diagnostic) !diagnostics.HandlerResult {
1149 const self: *Witness = @ptrCast(@alignCast(context.?));
1150 self.diagnostic_operation = diagnostic.operationName() orelse "";
1151 self.diagnostic_message = diagnostic.message;
1152 return .consumed;
1153 }
1154
1155 fn constant(self: *Witness, value: i64) !*ir.Value {
1156 const typ = self.function.getResultTypes()[0];
1157 var op = try ArithDialect.ConstantOp.createInt(&self.ctx, .unknown, typ, value);
1158 try self.function.getEntryBlock().addOperation(op.op);
1159 return op.getResult();
1160 }
1161
1162 fn binary(self: *Witness, comptime add: bool, lhs: *ir.Value, rhs: *ir.Value) !*ir.Value {
1163 var op = if (add)
1164 try ArithDialect.AddOp.create(&self.ctx, .unknown, lhs, rhs)
1165 else
1166 try ArithDialect.SubOp.create(&self.ctx, .unknown, lhs, rhs);
1167 try self.function.getEntryBlock().addOperation(op.op);
1168 return op.getResult();
1169 }
1170
1171 fn ret(self: *Witness, values: []const *ir.Value) !void {
1172 const op = try FuncDialect.ReturnOp.create(&self.ctx, .unknown, values);
1173 try self.function.getEntryBlock().addOperation(op.op);
1174 }
1175
1176 fn run(self: *Witness, args: []const i64) !ExecuteResult {
1177 return compileAndRun(&self.backend, self.module, "witness", args);
1178 }
1179 };
1180
1181 test "aarch64 native reordered operands preserve every argument register" {
1182 if (!supports_native_execution) return error.SkipZigTest;
1183 const args = [_]i64{ 3, 17, -9, 41, -23, 68, 107, -201 };
1184 for (0..max_arguments) |lhs| {
1185 for (0..max_arguments) |rhs| {
1186 var witness: Witness = undefined;
1187 try witness.init(.i64, max_arguments, 1);
1188 defer witness.deinit();
1189 const result = try witness.binary(false, witness.function.getArgument(lhs), witness.function.getArgument(rhs));
1190 try witness.ret(&.{result});
1191 try std.testing.expectEqual(args[lhs] - args[rhs], (try witness.run(&args)).asInt().?);
1192 }
1193 }
1194 }
1195
1196 test "aarch64 native refuses an effect outside the return dependency tree by name" {
1197 if (!supports_native_execution) return error.SkipZigTest;
1198 var witness: Witness = undefined;
1199 try witness.init(.i64, 0, 1);
1200 defer witness.deinit();
1201 const number = try witness.constant(172);
1202 const effect = try FuncDialect.SyscallOp.create(&witness.ctx, .unknown, number, &.{}, number.type);
1203 try witness.function.getEntryBlock().addOperation(effect.op);
1204 try witness.ret(&.{try witness.constant(42)});
1205 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{}));
1206 try std.testing.expectEqualStrings("func.syscall", witness.diagnostic_operation);
1207 }
1208
1209 test "aarch64 native narrow signed and unsigned integers normalize at each definition" {
1210 if (!supports_native_execution) return error.SkipZigTest;
1211 const Case = struct { kind: dialects.arith.ScalarKind, maximum: i64, overflow: i64, underflow: i64 };
1212 const cases = [_]Case{
1213 .{ .kind = .i8, .maximum = 127, .overflow = -128, .underflow = -1 },
1214 .{ .kind = .i16, .maximum = 32767, .overflow = -32768, .underflow = -1 },
1215 .{ .kind = .i32, .maximum = 2147483647, .overflow = -2147483648, .underflow = -1 },
1216 .{ .kind = .i64, .maximum = std.math.maxInt(i64), .overflow = std.math.minInt(i64), .underflow = -1 },
1217 .{ .kind = .u8, .maximum = 255, .overflow = 0, .underflow = 255 },
1218 .{ .kind = .u16, .maximum = 65535, .overflow = 0, .underflow = 65535 },
1219 .{ .kind = .u32, .maximum = 4294967295, .overflow = 0, .underflow = 4294967295 },
1220 .{ .kind = .u64, .maximum = -1, .overflow = 0, .underflow = -1 },
1221 .{ .kind = .index, .maximum = -1, .overflow = 0, .underflow = -1 },
1222 };
1223 for (cases) |case| {
1224 inline for (.{ true, false }) |add| {
1225 var witness: Witness = undefined;
1226 try witness.init(case.kind, 2, 1);
1227 defer witness.deinit();
1228 const result = try witness.binary(add, witness.function.getArgument(0), witness.function.getArgument(1));
1229 try witness.ret(&.{result});
1230 const args = [_]i64{ if (add) case.maximum else 0, 1 };
1231 try std.testing.expectEqual(if (add) case.overflow else case.underflow, (try witness.run(&args)).asInt().?);
1232 }
1233 var identity: Witness = undefined;
1234 try identity.init(case.kind, 1, 1);
1235 defer identity.deinit();
1236 try identity.ret(&.{identity.function.getArgument(0)});
1237 try std.testing.expectEqual(case.underflow, (try identity.run(&.{-1})).asInt().?);
1238 var constant: Witness = undefined;
1239 try constant.init(case.kind, 0, 1);
1240 defer constant.deinit();
1241 try constant.ret(&.{try constant.constant(-1)});
1242 try std.testing.expectEqual(case.underflow, (try constant.run(&.{})).asInt().?);
1243 }
1244 }
1245
1246 test "aarch64 native boolean arguments and constants and void results" {
1247 if (!supports_native_execution) return error.SkipZigTest;
1248 for ([_]bool{ false, true }) |value| {
1249 var identity: Witness = undefined;
1250 try identity.init(.bool, 1, 1);
1251 defer identity.deinit();
1252 try identity.ret(&.{identity.function.getArgument(0)});
1253 try std.testing.expectEqual(@as(i64, @intFromBool(value)), (try identity.run(&.{@intFromBool(value)})).asInt().?);
1254 var constant: Witness = undefined;
1255 try constant.init(.bool, 0, 1);
1256 defer constant.deinit();
1257 var op = try ArithDialect.ConstantOp.createBool(&constant.ctx, .unknown, value);
1258 try constant.function.getEntryBlock().addOperation(op.op);
1259 try constant.ret(&.{op.getResult()});
1260 try std.testing.expectEqual(@as(i64, @intFromBool(value)), (try constant.run(&.{})).asInt().?);
1261 }
1262 for ([_]usize{ 0, max_arguments }) |count| {
1263 var witness: Witness = undefined;
1264 try witness.init(.i64, count, 0);
1265 defer witness.deinit();
1266 try witness.ret(&.{});
1267 const args: [max_arguments]i64 = @splat(42);
1268 try std.testing.expectEqual(ExecuteResult.void_, try witness.run(args[0..count]));
1269 }
1270 }
1271
1272 test "aarch64 native wrong arity is refused before code generation or entry" {
1273 if (!supports_native_execution) return error.SkipZigTest;
1274 var witness: Witness = undefined;
1275 try witness.init(.i64, 2, 1);
1276 defer witness.deinit();
1277
1278 const effect = try FuncDialect.SyscallOp.create(&witness.ctx, .unknown, witness.function.getArgument(0), &.{}, witness.function.getArgument(0).type);
1279 try witness.function.getEntryBlock().addOperation(effect.op);
1280 try witness.ret(&.{witness.function.getArgument(1)});
1281 const args: [max_arguments + 1]i64 = @splat(172);
1282 for ([_]usize{ 0, 1, 3, max_arguments + 1 }) |count| {
1283 try std.testing.expectError(BackendError.ExecutionFailed, witness.run(args[0..count]));
1284 try std.testing.expectEqualStrings("func.func", witness.diagnostic_operation);
1285 try std.testing.expect(std.mem.indexOf(u8, witness.diagnostic_message, "argument count") != null);
1286 }
1287 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(args[0..2]));
1288 try std.testing.expectEqualStrings("func.syscall", witness.diagnostic_operation);
1289 }
1290
1291 test "aarch64 native argument result and register pressure boundaries" {
1292 if (!supports_native_execution) return error.SkipZigTest;
1293 var arguments: Witness = undefined;
1294 try arguments.init(.i64, max_arguments + 1, 1);
1295 defer arguments.deinit();
1296 try arguments.ret(&.{arguments.function.getArgument(0)});
1297 const args: [max_arguments + 1]i64 = @splat(0);
1298 try std.testing.expectError(BackendError.UnsupportedOperation, arguments.run(&args));
1299 try std.testing.expect(std.mem.indexOf(u8, arguments.diagnostic_message, "8 arguments") != null);
1300 var results: Witness = undefined;
1301 try results.init(.i64, 1, max_results + 1);
1302 defer results.deinit();
1303 const arg = results.function.getArgument(0);
1304 try results.ret(&.{ arg, arg });
1305 try std.testing.expectError(BackendError.UnsupportedOperation, results.run(&.{0}));
1306 try std.testing.expect(std.mem.indexOf(u8, results.diagnostic_message, "1 result") != null);
1307
1308 for ([_]usize{ placement.max_registers, placement.max_registers + 1 }) |count| {
1309 var pressure: Witness = undefined;
1310 try pressure.init(.i64, 0, 1);
1311 defer pressure.deinit();
1312 var values: [placement.max_registers + 1]*ir.Value = undefined;
1313 for (values[0..count], 0..) |*value, index| value.* = try pressure.constant(@intCast(index + 1));
1314 var sum = values[0];
1315 for (values[1..count]) |value| sum = try pressure.binary(true, sum, value);
1316
1317 const doubled = try pressure.binary(true, sum, sum);
1318 const restored = try pressure.binary(false, doubled, sum);
1319 try pressure.ret(&.{restored});
1320 try std.testing.expectEqual(@as(i64, @intCast(count * (count + 1) / 2)), (try pressure.run(&.{})).asInt().?);
1321 }
1322 }
1323
1324 /// Compare typed integer bits. Evaluator attributes may carry signed raw bits
1325 /// for an unsigned operation, so the declared result type owns extension.
1326 fn expectEvaluator(witness: *Witness, args: []const i64) !void {
1327 const Evaluator = @import("../../root.zig").eval.Evaluator;
1328 var evaluator = Evaluator.init(witness.arena.allocator(), &witness.ctx);
1329 defer evaluator.deinit();
1330 try evaluator.setRootOperation(witness.module);
1331 var attributes: [max_arguments]ir.Attribute = undefined;
1332 for (args, 0..) |arg, index| {
1333 const argument_kind = try scalarIntegerKind(witness.function.op, witness.function.getArgument(index).type);
1334 attributes[index] = if (argument_kind == .bool)
1335 try witness.ctx.getBoolAttr(arg != 0)
1336 else
1337 try witness.ctx.getI64Attr(canonicalInteger(arg, argument_kind));
1338 }
1339 const attribute = try evaluator.evaluateFunctionOp(witness.function.op, attributes[0..args.len]);
1340 const raw = if (attribute.cast(ir.Attribute.BoolAttr)) |boolean|
1341 @as(i64, @intFromBool(boolean.getValue()))
1342 else
1343 ArithDialect.getIntValue(attribute) orelse return error.EvalResultNotInteger;
1344 const kind = try scalarIntegerKind(witness.function.op, witness.function.getResultTypes()[0]);
1345 const descriptor = dialects.arith.scalarDescriptor(kind);
1346 const bits = descriptor.bit_width;
1347 const scalar = dialects.arith.scalar;
1348 const expected = if (descriptor.class == .signed_integer)
1349 scalar.truncate(raw, bits)
1350 else
1351 scalar.unsignedResult(@bitCast(raw), bits);
1352 try std.testing.expectEqual(expected, (try witness.run(args)).asInt().?);
1353 }
1354
1355 test "aarch64 native frame slots and live values at capacity and one past" {
1356 if (!supports_native_execution) return error.SkipZigTest;
1357 const full = placement.max_live_values;
1358 for ([_]usize{ placement.max_registers, placement.max_registers + 1, full, full + 1 }) |count| {
1359 var witness: Witness = undefined;
1360 try witness.init(.i64, 0, 1);
1361 defer witness.deinit();
1362 var values: [full + 1]*ir.Value = undefined;
1363 for (values[0..count], 0..) |*value, index| value.* = try witness.constant(@intCast(index + 1));
1364 var sum = values[0];
1365 for (values[1..count]) |value| sum = try witness.binary(true, sum, value);
1366 try witness.ret(&.{sum});
1367 if (count <= full) {
1368 var plan: placement.Plan = .{};
1369 try plan.build(witness.function.getEntryBlock());
1370 try std.testing.expectEqual(count - placement.max_registers, plan.frame_slots);
1371 try std.testing.expectEqual(@as(usize, 0), plan.frameBytes() % 16);
1372 try expectEvaluator(&witness, &.{});
1373 } else {
1374 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{}));
1375 try std.testing.expectEqualStrings("arith.constant", witness.diagnostic_operation);
1376 try std.testing.expect(std.mem.indexOf(u8, witness.diagnostic_message, "64 frame slots") != null);
1377 }
1378 }
1379 }
1380
1381 test "aarch64 native tracked values at capacity and one past" {
1382 if (!supports_native_execution) return error.SkipZigTest;
1383 for ([_]usize{ placement.max_values, placement.max_values + 1 }) |count| {
1384 var witness: Witness = undefined;
1385 try witness.init(.i64, 1, 1);
1386 defer witness.deinit();
1387 var value = witness.function.getArgument(0);
1388 for (0..count) |_| value = try witness.binary(true, value, witness.function.getArgument(0));
1389 try witness.ret(&.{value});
1390 if (count == placement.max_values) {
1391 try expectEvaluator(&witness, &.{17});
1392 } else {
1393 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{17}));
1394 try std.testing.expectEqualStrings("arith.add", witness.diagnostic_operation);
1395 try std.testing.expect(std.mem.indexOf(u8, witness.diagnostic_message, "256 tracked values") != null);
1396 }
1397 }
1398 }
1399
1400 test "aarch64 placement operation bound at capacity and one past" {
1401 var witness: Witness = undefined;
1402 try witness.init(.i64, 0, 0);
1403 defer witness.deinit();
1404 for (0..placement.max_operations) |_| try witness.ret(&.{});
1405 var plan: placement.Plan = .{};
1406 try plan.build(witness.function.getEntryBlock());
1407 try witness.ret(&.{});
1408 try std.testing.expectError(error.OperationCapacity, plan.build(witness.function.getEntryBlock()));
1409 }
1410
1411 test "aarch64 native expired spill slots are reused across pressure groups" {
1412 if (!supports_native_execution) return error.SkipZigTest;
1413 var witness: Witness = undefined;
1414 try witness.init(.i64, 0, 1);
1415 defer witness.deinit();
1416 const count = placement.max_registers + 3;
1417 var values: [count]*ir.Value = undefined;
1418 var result = try witness.constant(0);
1419 for (0..8) |group| {
1420 for (&values, 0..) |*value, index| value.* = try witness.constant(@intCast(group * count + index));
1421 for (values) |value| result = try witness.binary(true, result, value);
1422 }
1423 try witness.ret(&.{result});
1424 var plan: placement.Plan = .{};
1425 try plan.build(witness.function.getEntryBlock());
1426 try std.testing.expectEqual(@as(usize, 4), plan.frame_slots);
1427 try expectEvaluator(&witness, &.{});
1428 }
1429
1430 test "aarch64 native admitted scalar operations match evaluator at extremes and seeded inputs" {
1431 if (!supports_native_execution) return error.SkipZigTest;
1432 const kinds = [_]dialects.arith.ScalarKind{ .i8, .i16, .i32, .i64, .u8, .u16, .u32, .u64, .index };
1433 var seed: u64 = 0x6f37_92ba_1d08_5ce4;
1434 for (kinds) |kind| {
1435 const width = dialects.arith.scalarBitWidth(kind);
1436 const sign_bit: u64 = @as(u64, 1) << @intCast(width - 1);
1437 const edges = [_]i64{ 0, 1, -1, @bitCast(sign_bit), @bitCast(sign_bit - 1) };
1438 for (0..edges.len + 8) |sample| {
1439 seed ^= seed << 13;
1440 seed ^= seed >> 7;
1441 seed ^= seed << 17;
1442 const raw: i64 = if (sample < edges.len) edges[sample] else @bitCast(seed);
1443 const other: i64 = if (sample < edges.len) edges[edges.len - sample - 1] else @bitCast(seed ^ 0xa6ac_e510_d07b_5823);
1444 var constant: Witness = undefined;
1445 try constant.init(kind, 0, 1);
1446 defer constant.deinit();
1447 try constant.ret(&.{try constant.constant(raw)});
1448 try expectEvaluator(&constant, &.{});
1449 inline for (.{ true, false }) |add| {
1450 var witness: Witness = undefined;
1451 try witness.init(kind, 2, 1);
1452 defer witness.deinit();
1453 const result = try witness.binary(add, witness.function.getArgument(0), witness.function.getArgument(1));
1454 try witness.ret(&.{result});
1455 try expectEvaluator(&witness, &.{ raw, other });
1456 }
1457 }
1458 }
1459 for ([_]bool{ false, true }) |value| {
1460 var witness: Witness = undefined;
1461 try witness.init(.bool, 0, 1);
1462 defer witness.deinit();
1463 const constant = try ArithDialect.ConstantOp.createBool(&witness.ctx, .unknown, value);
1464 try witness.function.getEntryBlock().addOperation(constant.op);
1465 try witness.ret(&.{constant.getResult()});
1466 try expectEvaluator(&witness, &.{});
1467 }
1468 }
1469
1470 const integer_kinds = [_]dialects.arith.ScalarKind{ .i8, .i16, .i32, .i64, .u8, .u16, .u32, .u64, .index };
1471
1472 fn canonicalInteger(raw: i64, kind: dialects.arith.ScalarKind) i64 {
1473 const scalar = dialects.arith.scalar;
1474 const descriptor = dialects.arith.scalarDescriptor(kind);
1475 return if (descriptor.class == .signed_integer)
1476 scalar.truncate(raw, descriptor.bit_width)
1477 else
1478 scalar.unsignedResult(@bitCast(raw), descriptor.bit_width);
1479 }
1480
1481 fn nextSeed(seed: *u64) i64 {
1482 seed.* ^= seed.* << 13;
1483 seed.* ^= seed.* >> 7;
1484 seed.* ^= seed.* << 17;
1485 return @bitCast(seed.*);
1486 }
1487
1488 fn integerSamples(kind: dialects.arith.ScalarKind, seed: *u64) [13]i64 {
1489 const width = dialects.arith.scalarBitWidth(kind);
1490 const sign_bit: u64 = @as(u64, 1) << @intCast(width - 1);
1491 var samples: [13]i64 = undefined;
1492 const edges = [_]i64{ 0, 1, -1, @bitCast(sign_bit), @bitCast(sign_bit - 1) };
1493 for (&samples, 0..) |*sample, index| {
1494 sample.* = canonicalInteger(if (index < edges.len) edges[index] else nextSeed(seed), kind);
1495 }
1496 return samples;
1497 }
1498
1499 test "aarch64 native scalar binary and unary operations differentially cover every integer width" {
1500 if (!supports_native_execution) return error.SkipZigTest;
1501 var seed: u64 = 0xd761_2c93_a5e8_04bf;
1502 for (integer_kinds) |kind| {
1503 const samples = integerSamples(kind, &seed);
1504 inline for (.{ ArithDialect.MulOp, ArithDialect.DivOp, ArithDialect.RemOp, ArithDialect.AndOp, ArithDialect.OrOp, ArithDialect.XorOp, ArithDialect.MinOp, ArithDialect.MaxOp, ArithDialect.UmulhiOp, ArithDialect.ShlOp, ArithDialect.ShrOp, ArithDialect.UshrOp }) |Op| {
1505 var witness: Witness = undefined;
1506 try witness.init(kind, 2, 1);
1507 defer witness.deinit();
1508 const op = try Op.create(&witness.ctx, .unknown, witness.function.getArgument(0), witness.function.getArgument(1));
1509 try witness.function.getEntryBlock().addOperation(op.op);
1510 try witness.ret(&.{op.getResult()});
1511 for (samples, 0..) |lhs, index| {
1512 var rhs = samples[(index + 3) % samples.len];
1513 if (Op == ArithDialect.DivOp or Op == ArithDialect.RemOp) {
1514 const bits = dialects.arith.scalarBitWidth(kind);
1515 const minimum: i64 = @intCast(dialects.arith.scalar.intLimits(bits).min);
1516 if (rhs == 0 or (lhs == minimum and rhs == -1)) rhs = 1;
1517 }
1518 if (Op == ArithDialect.ShlOp or Op == ArithDialect.ShrOp or Op == ArithDialect.UshrOp) {
1519 rhs = @intCast(index % dialects.arith.scalarBitWidth(kind));
1520 if (index == samples.len - 1) rhs = dialects.arith.scalarBitWidth(kind) - 1;
1521 }
1522 try expectEvaluator(&witness, &.{ lhs, rhs });
1523 }
1524 }
1525 inline for (.{ ArithDialect.NegOp, ArithDialect.NotOp, ArithDialect.AbsOp, ArithDialect.PopCountOp }) |Op| {
1526 var witness: Witness = undefined;
1527 try witness.init(kind, 1, 1);
1528 defer witness.deinit();
1529 const op = try Op.create(&witness.ctx, .unknown, witness.function.getArgument(0));
1530 try witness.function.getEntryBlock().addOperation(op.op);
1531 try witness.ret(&.{op.getResult()});
1532 for (samples) |value| {
1533 if (Op == ArithDialect.AbsOp and Emitter.unsignedKind(kind) and value < 0) {
1534 try expectTypedAbsolute(&witness, &.{value}, value, kind);
1535 } else {
1536 try expectEvaluator(&witness, &.{value});
1537 }
1538 }
1539 }
1540 }
1541 }
1542
1543 test "aarch64 native all integer comparison predicates match evaluator" {
1544 if (!supports_native_execution) return error.SkipZigTest;
1545 var seed: u64 = 0xab16_c78f_2109_d4e3;
1546 for (integer_kinds) |kind| {
1547 const samples = integerSamples(kind, &seed);
1548 inline for (std.meta.tags(dialects.arith.CmpPredicate)) |predicate| {
1549 var witness: Witness = undefined;
1550 try witness.initSignature(kind, .bool, 2, 1);
1551 defer witness.deinit();
1552 const cmp = try ArithDialect.CmpOp.create(&witness.ctx, .unknown, predicate, witness.function.getArgument(0), witness.function.getArgument(1));
1553 try witness.function.getEntryBlock().addOperation(cmp.op);
1554 try witness.ret(&.{cmp.getResult()});
1555 for (samples, 0..) |value, index| {
1556 try expectEvaluator(&witness, &.{ value, samples[(index + 3) % samples.len] });
1557 try expectEvaluator(&witness, &.{ value, value });
1558 }
1559 }
1560 }
1561 }
1562
1563 test "aarch64 native integer casts cover every source destination width pair" {
1564 if (!supports_native_execution) return error.SkipZigTest;
1565 var seed: u64 = 0x653e_920b_f8d4_107c;
1566 for (integer_kinds) |source| {
1567 const samples = integerSamples(source, &seed);
1568 for (integer_kinds) |destination| {
1569 var witness: Witness = undefined;
1570 try witness.initSignature(source, destination, 1, 1);
1571 defer witness.deinit();
1572 const cast = try ArithDialect.CastOp.create(&witness.ctx, .unknown, witness.function.getArgument(0), witness.function.getResultTypes()[0]);
1573 try witness.function.getEntryBlock().addOperation(cast.op);
1574 try witness.ret(&.{cast.getResult()});
1575 for (samples) |value| try expectEvaluator(&witness, &.{value});
1576 }
1577 }
1578 }
1579
1580 test "aarch64 native select and boolean scalar operations match evaluator" {
1581 if (!supports_native_execution) return error.SkipZigTest;
1582 var seed: u64 = 0x8da2_6094_b713_f5ec;
1583 for (integer_kinds) |kind| {
1584 const samples = integerSamples(kind, &seed);
1585 var witness: Witness = undefined;
1586 try witness.init(kind, 2, 1);
1587 defer witness.deinit();
1588 const lhs = witness.function.getArgument(0);
1589 const rhs = witness.function.getArgument(1);
1590 const cmp = try ArithDialect.CmpOp.create(&witness.ctx, .unknown, .eq, lhs, rhs);
1591 try witness.function.getEntryBlock().addOperation(cmp.op);
1592 const select = try ArithDialect.SelectOp.create(&witness.ctx, .unknown, cmp.getResult(), lhs, rhs);
1593 try witness.function.getEntryBlock().addOperation(select.op);
1594 try witness.ret(&.{select.getResult()});
1595 for (samples, 0..) |value, index| {
1596 try expectEvaluator(&witness, &.{ value, samples[(index + 1) % samples.len] });
1597 try expectEvaluator(&witness, &.{ value, value });
1598 }
1599 }
1600 inline for (.{ ArithDialect.AndOp, ArithDialect.OrOp, ArithDialect.XorOp }) |Op| {
1601 var witness: Witness = undefined;
1602 try witness.init(.bool, 2, 1);
1603 defer witness.deinit();
1604 const op = try Op.create(&witness.ctx, .unknown, witness.function.getArgument(0), witness.function.getArgument(1));
1605 try witness.function.getEntryBlock().addOperation(op.op);
1606 try witness.ret(&.{op.getResult()});
1607 for (0..2) |a| for (0..2) |b| try expectEvaluator(&witness, &.{ @intCast(a), @intCast(b) });
1608 }
1609 var witness: Witness = undefined;
1610 try witness.init(.bool, 1, 1);
1611 defer witness.deinit();
1612 const op = try ArithDialect.NotOp.create(&witness.ctx, .unknown, witness.function.getArgument(0));
1613 try witness.function.getEntryBlock().addOperation(op.op);
1614 try witness.ret(&.{op.getResult()});
1615 try expectEvaluator(&witness, &.{0});
1616 try expectEvaluator(&witness, &.{1});
1617 }
1618
1619 fn expectTrap(witness: *Witness, args: []const i64) !void {
1620 var evaluator = @import("../../root.zig").eval.Evaluator.init(witness.arena.allocator(), &witness.ctx);
1621 defer evaluator.deinit();
1622 try evaluator.setRootOperation(witness.module);
1623 var attributes: [max_arguments]ir.Attribute = undefined;
1624 for (args, 0..) |arg, index| attributes[index] = try witness.ctx.getI64Attr(arg);
1625 try std.testing.expectError(error.InvalidOperand, evaluator.evaluateFunctionOp(witness.function.op, attributes[0..args.len]));
1626 switch (try sys.process.fork()) {
1627 .child => {
1628 _ = witness.run(args) catch sys.process.exit(101);
1629 sys.process.exit(102);
1630 },
1631 .parent => |pid| {
1632 const termination = try sys.process.waitDirect(pid);
1633 switch (termination) {
1634 .signal => |signal| try std.testing.expect(signal == .TRAP),
1635 else => return error.ExpectedSigtrap,
1636 }
1637 },
1638 }
1639 }
1640
1641 test "aarch64 native invalid arithmetic traps even when its result is unused" {
1642 if (!supports_native_execution) return error.SkipZigTest;
1643 if (@import("builtin").os.tag != .linux) return error.SkipZigTest;
1644 for (integer_kinds) |kind| {
1645 const bits = dialects.arith.scalarBitWidth(kind);
1646 inline for (.{ ArithDialect.DivOp, ArithDialect.RemOp, ArithDialect.ShlOp, ArithDialect.ShrOp, ArithDialect.UshrOp }) |Op| {
1647 var witness: Witness = undefined;
1648 try witness.init(kind, 2, 1);
1649 defer witness.deinit();
1650 const op = try Op.create(&witness.ctx, .unknown, witness.function.getArgument(0), witness.function.getArgument(1));
1651 try witness.function.getEntryBlock().addOperation(op.op);
1652 try witness.ret(&.{witness.function.getArgument(0)});
1653 if (Op == ArithDialect.DivOp or Op == ArithDialect.RemOp) {
1654 try expectTrap(&witness, &.{ 1, 0 });
1655 if (dialects.arith.scalarKindIsSignedInteger(kind) and kind != .index) {
1656 const minimum: i64 = @intCast(dialects.arith.scalar.intLimits(bits).min);
1657 try expectTrap(&witness, &.{ minimum, -1 });
1658 }
1659 } else {
1660 try expectTrap(&witness, &.{ 1, bits });
1661 try expectTrap(&witness, &.{ 1, -1 });
1662 }
1663 }
1664 }
1665 }
1666
1667 test "aarch64 native statically certain arithmetic failures refuse by operation name" {
1668 if (!supports_native_execution) return error.SkipZigTest;
1669 inline for (.{ ArithDialect.DivOp, ArithDialect.RemOp, ArithDialect.ShlOp, ArithDialect.ShrOp, ArithDialect.UshrOp }) |Op| {
1670 var witness: Witness = undefined;
1671 try witness.init(.i64, 0, 1);
1672 defer witness.deinit();
1673 const lhs = try witness.constant(1);
1674 const rhs = try witness.constant(if (Op == ArithDialect.DivOp or Op == ArithDialect.RemOp) 0 else 64);
1675 const op = try Op.create(&witness.ctx, .unknown, lhs, rhs);
1676 try witness.function.getEntryBlock().addOperation(op.op);
1677 try witness.ret(&.{lhs});
1678 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{}));
1679 try std.testing.expectEqualStrings(Op.operation_name, witness.diagnostic_operation);
1680 }
1681 }
1682
1683 test "aarch64 native scalar temporaries preserve spilled operands and results" {
1684 if (!supports_native_execution) return error.SkipZigTest;
1685 inline for (.{ ArithDialect.MulOp, ArithDialect.DivOp, ArithDialect.RemOp, ArithDialect.AndOp, ArithDialect.OrOp, ArithDialect.XorOp, ArithDialect.MinOp, ArithDialect.MaxOp, ArithDialect.UmulhiOp, ArithDialect.ShlOp, ArithDialect.ShrOp, ArithDialect.UshrOp }) |Op| {
1686 var witness: Witness = undefined;
1687 try witness.init(.i64, 0, 1);
1688 defer witness.deinit();
1689 const count = placement.max_registers + 4;
1690 var values: [count]*ir.Value = undefined;
1691 for (&values, 0..) |*value, index| value.* = try witness.constant(@intCast(index + 1));
1692 const op = try Op.create(&witness.ctx, .unknown, values[count - 1], values[count - 2]);
1693 try witness.function.getEntryBlock().addOperation(op.op);
1694 var result = op.getResult();
1695 for (values) |value| result = try witness.binary(true, result, value);
1696 try witness.ret(&.{result});
1697 try expectEvaluator(&witness, &.{});
1698 }
1699 inline for (.{ ArithDialect.AbsOp, ArithDialect.NegOp, ArithDialect.NotOp, ArithDialect.PopCountOp }) |Op| {
1700 var witness: Witness = undefined;
1701 try witness.init(.i64, 0, 1);
1702 defer witness.deinit();
1703 const count = placement.max_registers + 4;
1704 var values: [count]*ir.Value = undefined;
1705 for (&values, 0..) |*value, index| value.* = try witness.constant(-@as(i64, @intCast(index + 1)));
1706 const op = try Op.create(&witness.ctx, .unknown, values[count - 1]);
1707 try witness.function.getEntryBlock().addOperation(op.op);
1708 var result = op.getResult();
1709 for (values) |value| result = try witness.binary(true, result, value);
1710 try witness.ret(&.{result});
1711 try expectEvaluator(&witness, &.{});
1712 }
1713 }
1714
1715 test "aarch64 native literal signed division overflow refuses by name at every width" {
1716 if (!supports_native_execution) return error.SkipZigTest;
1717 for ([_]dialects.arith.ScalarKind{ .i8, .i16, .i32, .i64 }) |kind| {
1718 inline for (.{ ArithDialect.DivOp, ArithDialect.RemOp }) |Op| {
1719 var witness: Witness = undefined;
1720 try witness.init(kind, 0, 1);
1721 defer witness.deinit();
1722 const bits = dialects.arith.scalarBitWidth(kind);
1723 const lhs = try witness.constant(@intCast(dialects.arith.scalar.intLimits(bits).min));
1724 const rhs = try witness.constant(-1);
1725 const op = try Op.create(&witness.ctx, .unknown, lhs, rhs);
1726 try witness.function.getEntryBlock().addOperation(op.op);
1727 try witness.ret(&.{op.getResult()});
1728 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{}));
1729 try std.testing.expectEqualStrings(Op.operation_name, witness.diagnostic_operation);
1730 try std.testing.expect(std.mem.indexOf(u8, witness.diagnostic_message, "SignedDivisionOverflow") != null);
1731 }
1732 }
1733 }
1734
1735 /// The evaluator observes raw Attribute signs for abs, while the typed contract
1736 /// treats unsigned abs as identity and signed abs as wrapping arithmetic.
1737 fn expectTypedAbsolute(witness: *Witness, args: []const i64, raw: i64, kind: dialects.arith.ScalarKind) !void {
1738 const input = canonicalInteger(raw, kind);
1739 const expected = if (Emitter.unsignedKind(kind)) input else dialects.arith.scalar.absWrap(input, dialects.arith.scalarBitWidth(kind));
1740 try std.testing.expectEqual(expected, (try witness.run(args)).asInt().?);
1741 }
1742
1743 test "aarch64 native composed absolute value follows normalized typed operands" {
1744 if (!supports_native_execution) return error.SkipZigTest;
1745 for (integer_kinds) |kind| {
1746 var witness: Witness = undefined;
1747 try witness.init(kind, 2, 1);
1748 defer witness.deinit();
1749 const difference = try witness.binary(false, witness.function.getArgument(0), witness.function.getArgument(1));
1750 const absolute = try ArithDialect.AbsOp.create(&witness.ctx, .unknown, difference);
1751 try witness.function.getEntryBlock().addOperation(absolute.op);
1752 try witness.ret(&.{absolute.getResult()});
1753 try expectTypedAbsolute(&witness, &.{ 0, 1 }, -1, kind);
1754 const minimum: i64 = @bitCast(@as(u64, 1) << @intCast(dialects.arith.scalarBitWidth(kind) - 1));
1755 try expectTypedAbsolute(&witness, &.{ minimum, 0 }, minimum, kind);
1756 var literal: Witness = undefined;
1757 try literal.init(kind, 0, 1);
1758 defer literal.deinit();
1759 const raw: i64 = if (Emitter.unsignedKind(kind)) -1 else @bitCast(@as(u64, std.math.maxInt(u64)) >> @intCast(64 - dialects.arith.scalarBitWidth(kind)));
1760 const input = try literal.constant(raw);
1761 const abs = try ArithDialect.AbsOp.create(&literal.ctx, .unknown, input);
1762 try literal.function.getEntryBlock().addOperation(abs.op);
1763 try literal.ret(&.{abs.getResult()});
1764 try expectTypedAbsolute(&literal, &.{}, raw, kind);
1765 }
1766 }
1767
1768 test "aarch64 backend refuses overflow arithmetic by name" {
1769 const allocator = std.testing.allocator;
1770 const Arith = dialects.ArithDialect;
1771 inline for (.{ Arith.AddoOp, Arith.SuboOp, Arith.MuloOp }) |Op| {
1772 for ([_]bool{ false, true }) |used| {
1773 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1774 defer ctx.deinit(allocator);
1775 try dialects.registerAllDialects(&ctx);
1776 const integer = try Arith.getScalarType(&ctx, .i64);
1777 const module = try BuiltinDialect.ModuleOp.create(&ctx, .unknown);
1778 const function = try FuncDialect.FuncOp.create(&ctx, .unknown, "overflow", &.{ integer, integer }, &.{integer});
1779 try module.getBodyBlock().addOperation(function.op);
1780 const checked = try Op.create(&ctx, .unknown, function.getArgument(0), function.getArgument(1));
1781 try function.getEntryBlock().addOperation(checked.op);
1782 const value = if (used) checked.getResult() else function.getArgument(0);
1783 const ret = try FuncDialect.ReturnOp.create(&ctx, .unknown, &.{value});
1784 try function.getEntryBlock().addOperation(ret.op);
1785 var backend = try Backend.init(allocator, &ctx);
1786 defer backend.deinit();
1787 try std.testing.expectError(error.UnsupportedOperation, backend.compileFunctionToArtifact(module.op, "overflow"));
1788 }
1789 }
1790 }
1791
1792 const ScfDialect = dialects.ScfDialect;
1793
1794 fn blockConstant(witness: *Witness, block: *ir.Block, kind: dialects.arith.ScalarKind, value: i64) !*ir.Value {
1795 const typ = try ArithDialect.getScalarType(&witness.ctx, kind);
1796 const op = if (kind == .bool)
1797 try ArithDialect.ConstantOp.createBool(&witness.ctx, .unknown, value != 0)
1798 else
1799 try ArithDialect.ConstantOp.createInt(&witness.ctx, .unknown, typ, value);
1800 try block.addOperation(op.op);
1801 return op.getResult();
1802 }
1803
1804 fn blockBinary(witness: *Witness, block: *ir.Block, comptime Op: type, lhs: *ir.Value, rhs: *ir.Value) !*ir.Value {
1805 const op = try Op.create(&witness.ctx, .unknown, lhs, rhs);
1806 try block.addOperation(op.op);
1807 return op.getResult();
1808 }
1809
1810 fn blockYield(witness: *Witness, block: *ir.Block, values: []const *ir.Value) !void {
1811 const op = try ScfDialect.YieldOp.create(&witness.ctx, .unknown, values);
1812 try block.addOperation(op.op);
1813 }
1814
1815 fn blockCondition(witness: *Witness, block: *ir.Block, cond: *ir.Value, values: []const *ir.Value) !void {
1816 const op = try ScfDialect.ConditionOp.create(&witness.ctx, .unknown, cond, values);
1817 try block.addOperation(op.op);
1818 }
1819
1820 fn blockCompare(witness: *Witness, block: *ir.Block, predicate: dialects.arith.CmpPredicate, lhs: *ir.Value, rhs: *ir.Value) !*ir.Value {
1821 const op = try ArithDialect.CmpOp.create(&witness.ctx, .unknown, predicate, lhs, rhs);
1822 try block.addOperation(op.op);
1823 return op.getResult();
1824 }
1825
1826 test "aarch64 native control if selects both branches with scalar results" {
1827 if (!supports_native_execution) return error.SkipZigTest;
1828 for (integer_kinds ++ [_]dialects.arith.ScalarKind{.bool}) |kind| {
1829 var witness: Witness = undefined;
1830 try witness.initSignature(.bool, kind, 1, 1);
1831 defer witness.deinit();
1832 const typ = witness.function.getResultTypes()[0];
1833 const branch = try ScfDialect.IfOp.create(&witness.ctx, .unknown, witness.function.getArgument(0), &.{typ});
1834 try witness.function.getEntryBlock().addOperation(branch.op);
1835 const yes = try blockConstant(&witness, branch.getThenBlock(), kind, 1);
1836 const no = try blockConstant(&witness, branch.getElseBlock().?, kind, 0);
1837 try blockYield(&witness, branch.getThenBlock(), &.{yes});
1838 try blockYield(&witness, branch.getElseBlock().?, &.{no});
1839 try witness.ret(&.{branch.getResult(0).?});
1840 try expectEvaluator(&witness, &.{0});
1841 try expectEvaluator(&witness, &.{1});
1842 }
1843 }
1844
1845 test "aarch64 native control if without results preserves conditional failures" {
1846 if (!supports_native_execution) return error.SkipZigTest;
1847 var witness: Witness = undefined;
1848 try witness.init(.i64, 2, 1);
1849 defer witness.deinit();
1850 const entry = witness.function.getEntryBlock();
1851 const zero = try witness.constant(0);
1852 const cond = try blockCompare(&witness, entry, .ne, witness.function.getArgument(0), zero);
1853 const branch = try ScfDialect.IfOp.createWithoutElse(&witness.ctx, .unknown, cond);
1854 try entry.addOperation(branch.op);
1855 _ = try blockBinary(&witness, branch.getThenBlock(), ArithDialect.DivOp, witness.function.getArgument(0), witness.function.getArgument(1));
1856 try blockYield(&witness, branch.getThenBlock(), &.{});
1857 try witness.ret(&.{zero});
1858 try expectEvaluator(&witness, &.{ 0, 0 });
1859 try expectEvaluator(&witness, &.{ 1, 2 });
1860 if (@import("builtin").os.tag == .linux) try expectTrap(&witness, &.{ 1, 0 });
1861 }
1862
1863 /// A before-region condition forwards a transformed value on both exit edges.
1864 fn makeWhile(witness: *Witness, parent: *ir.Block, initial: *ir.Value, upper: *ir.Value, step: *ir.Value) !ScfDialect.WhileOp {
1865 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{initial}, &.{initial.type});
1866 try parent.addOperation(loop.op);
1867 const before = loop.getBeforeBlock();
1868 const current = before.arguments.items[0];
1869 const cond = try blockCompare(witness, before, .slt, current, upper);
1870 try blockCondition(witness, before, cond, &.{current});
1871 const after = loop.getAfterBlock();
1872 const next = try blockBinary(witness, after, ArithDialect.AddOp, after.arguments.items[0], step);
1873 try blockYield(witness, after, &.{next});
1874 return loop;
1875 }
1876
1877 test "aarch64 native control while zero one many tests and nested loops match evaluator" {
1878 if (!supports_native_execution) return error.SkipZigTest;
1879 for ([_]bool{ true, false }) |nested| {
1880 var witness: Witness = undefined;
1881 try witness.init(.i64, 2, 1);
1882 defer witness.deinit();
1883 const entry = witness.function.getEntryBlock();
1884 const one = try witness.constant(1);
1885 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{witness.function.getArgument(0)}, witness.function.getResultTypes());
1886 try entry.addOperation(loop.op);
1887 const before = loop.getBeforeBlock();
1888 const current = before.arguments.items[0];
1889 const cond = try blockCompare(&witness, before, .slt, current, witness.function.getArgument(1));
1890 try blockCondition(&witness, before, cond, &.{current});
1891 const after = loop.getAfterBlock();
1892 const next = try blockBinary(&witness, after, ArithDialect.AddOp, after.arguments.items[0], one);
1893 const yielded = if (nested) value: {
1894 const inner = try makeWhile(&witness, after, after.arguments.items[0], next, one);
1895 break :value inner.op.getResult(0).?;
1896 } else next;
1897 try blockYield(&witness, after, &.{yielded});
1898 try witness.ret(&.{loop.op.getResult(0).?});
1899 for ([_]i64{ 0, 1, 19 }) |upper| try std.testing.expectEqual(upper, (try witness.run(&.{ 0, upper })).asInt().?);
1900 try std.testing.expectEqual(@as(i64, 8), (try witness.run(&.{ 8, 3 })).asInt().?);
1901 for ([_]i64{ 0, 1, 19 }) |upper| try expectEvaluator(&witness, &.{ 0, upper });
1902 try expectEvaluator(&witness, &.{ 8, 3 });
1903 }
1904 }
1905
1906 test "aarch64 native control condition forwards computed exit and entry values" {
1907 if (!supports_native_execution) return error.SkipZigTest;
1908 var witness: Witness = undefined;
1909 try witness.init(.i64, 1, 1);
1910 defer witness.deinit();
1911 const one = try witness.constant(1);
1912 const ten = try witness.constant(10);
1913 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{witness.function.getArgument(0)}, &.{one.type});
1914 try witness.function.getEntryBlock().addOperation(loop.op);
1915 const before = loop.getBeforeBlock();
1916 const next = try blockBinary(&witness, before, ArithDialect.AddOp, before.arguments.items[0], one);
1917 const cond = try blockCompare(&witness, before, .slt, next, ten);
1918 try blockCondition(&witness, before, cond, &.{next});
1919 try blockYield(&witness, loop.getAfterBlock(), loop.getAfterBlock().arguments.items);
1920 try witness.ret(&.{loop.op.getResult(0).?});
1921 for ([_]i64{ 0, 8, 9, 12 }) |initial| try expectEvaluator(&witness, &.{initial});
1922 }
1923
1924 fn makeCounted(witness: *Witness, parent: *ir.Block, upper: *ir.Value, count: usize) !ScfDialect.ForOp {
1925 const zero = try blockConstant(witness, parent, .index, 0);
1926 const one = try blockConstant(witness, parent, .index, 1);
1927 var values: [control.max_carried_values + 1]*ir.Value = undefined;
1928 var types: [control.max_carried_values + 1]ir.Type = undefined;
1929 for (values[0..count], types[0..count], 0..) |*value, *typ, index| {
1930 value.* = try blockConstant(witness, parent, .index, @intCast(index));
1931 typ.* = zero.type;
1932 }
1933 const loop = try ScfDialect.ForOp.create(&witness.ctx, .unknown, zero, upper, one, values[0..count], types[0..count]);
1934 try parent.addOperation(loop.op);
1935 const body = loop.getBodyBlock();
1936 for (values[0..count], 0..) |*value, index| value.* = try blockBinary(witness, body, ArithDialect.AddOp, body.arguments.items[index + 1], body.arguments.items[0]);
1937 try blockYield(witness, body, values[0..count]);
1938 return loop;
1939 }
1940
1941 test "aarch64 native control counted loop zero one many iterations match evaluator" {
1942 if (!supports_native_execution) return error.SkipZigTest;
1943 var witness: Witness = undefined;
1944 try witness.init(.index, 1, 1);
1945 defer witness.deinit();
1946 const loop = try makeCounted(&witness, witness.function.getEntryBlock(), witness.function.getArgument(0), 1);
1947 try witness.ret(&.{loop.op.getResult(0).?});
1948 for ([_]i64{ 0, 1, 17 }) |upper| try std.testing.expectEqual(@divTrunc(upper * (upper - 1), 2), (try witness.run(&.{upper})).asInt().?);
1949 for ([_]i64{ 0, 1, 17 }) |upper| try expectEvaluator(&witness, &.{upper});
1950 }
1951
1952 fn makeRotatingWhile(witness: *Witness, count: usize) !ScfDialect.WhileOp {
1953 const entry = witness.function.getEntryBlock();
1954 var values: [control.max_carried_values + 1]*ir.Value = undefined;
1955 var types: [control.max_carried_values + 1]ir.Type = undefined;
1956 for (values[0..count], types[0..count], 0..) |*value, *typ, index| {
1957 value.* = try witness.constant(@intCast(index));
1958 typ.* = value.*.type;
1959 }
1960 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, values[0..count], types[0..count]);
1961 try entry.addOperation(loop.op);
1962 const before = loop.getBeforeBlock();
1963 const cond = try blockCompare(witness, before, .slt, before.arguments.items[0], witness.function.getArgument(0));
1964 values[0] = before.arguments.items[0];
1965 for (1..count) |index| values[index] = before.arguments.items[1 + index % (count - 1)];
1966 try blockCondition(witness, before, cond, values[0..count]);
1967 const after = loop.getAfterBlock();
1968 const one = try blockConstant(witness, after, .i64, 1);
1969 values[0] = try blockBinary(witness, after, ArithDialect.AddOp, after.arguments.items[0], one);
1970 for (1..count) |index| values[index] = after.arguments.items[1 + index % (count - 1)];
1971 try blockYield(witness, after, values[0..count]);
1972 return loop;
1973 }
1974
1975 test "aarch64 native control carried bound parallel cycles and one past" {
1976 if (!supports_native_execution) return error.SkipZigTest;
1977 for ([_]usize{ control.max_carried_values + 1, control.max_carried_values }) |count| {
1978 var witness: Witness = undefined;
1979 try witness.init(.i64, 1, 1);
1980 defer witness.deinit();
1981 const loop = try makeRotatingWhile(&witness, count);
1982 var result = loop.op.getResult(0).?;
1983 for (1..count) |index| {
1984 const weight = try witness.constant(@intCast(index + 1));
1985 const product = try blockBinary(&witness, witness.function.getEntryBlock(), ArithDialect.MulOp, loop.op.getResult(index).?, weight);
1986 result = try witness.binary(true, result, product);
1987 }
1988 try witness.ret(&.{result});
1989 if (count > control.max_carried_values) {
1990 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{0}));
1991 try std.testing.expectEqualStrings("scf.while", witness.diagnostic_operation);
1992 try std.testing.expect(std.mem.indexOf(u8, witness.diagnostic_message, "16 carried") != null);
1993 } else {
1994 for ([_]i64{ 0, 1, 13 }) |upper| {
1995 var expected = upper;
1996 for (1..count) |index| {
1997 const rotated = 1 + (index - 1 + 2 * @as(usize, @intCast(upper)) + 1) % (count - 1);
1998 expected += @intCast((index + 1) * rotated);
1999 }
2000 try std.testing.expectEqual(expected, (try witness.run(&.{upper})).asInt().?);
2001 }
2002 for ([_]i64{ 0, 1, 13 }) |upper| try expectEvaluator(&witness, &.{upper});
2003 }
2004 }
2005 }
2006
2007 test "aarch64 native control pressure across back edges reaches frame bound and refuses one past" {
2008 if (!supports_native_execution) return error.SkipZigTest;
2009 const full = placement.max_live_values - 4;
2010 for ([_]usize{ full + 1, full, placement.max_registers + 2 }) |count| {
2011 var witness: Witness = undefined;
2012 try witness.init(.i64, 2, 1);
2013 defer witness.deinit();
2014 var captured: [full + 1]*ir.Value = undefined;
2015 for (captured[0..count], 0..) |*value, index| value.* = try witness.constant(@intCast(index + 1));
2016 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{witness.function.getArgument(0)}, witness.function.getResultTypes());
2017 try witness.function.getEntryBlock().addOperation(loop.op);
2018 const before = loop.getBeforeBlock();
2019 const cond = try blockCompare(&witness, before, .slt, before.arguments.items[0], witness.function.getArgument(1));
2020 try blockCondition(&witness, before, cond, before.arguments.items);
2021 const after = loop.getAfterBlock();
2022 var sum = after.arguments.items[0];
2023 for (captured[0..count]) |value| sum = try blockBinary(&witness, after, ArithDialect.AddOp, sum, value);
2024 try blockYield(&witness, after, &.{sum});
2025 try witness.ret(&.{loop.op.getResult(0).?});
2026 var plan: placement.Plan = .{};
2027 if (count > full) {
2028 try std.testing.expectError(error.FrameCapacity, plan.build(witness.function.getEntryBlock()));
2029 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{ 0, 10000 }));
2030 } else {
2031 try plan.build(witness.function.getEntryBlock());
2032 if (count == full) try std.testing.expectEqual(placement.max_frame_slots, plan.frame_slots);
2033 try std.testing.expect(plan.frameBytes() <= placement.max_frame_slots * placement.slot_bytes);
2034 for (captured[0..count]) |value| {
2035 for (plan.entries[0..plan.count]) |entry| {
2036 if (entry.value == value) try std.testing.expectEqual(plan.loops[0].trailing, entry.range.end);
2037 }
2038 }
2039 for ([_]i64{ 0, 1, 10000 }) |upper| {
2040 const increment: i64 = @intCast(count * (count + 1) / 2);
2041 const expected = @divTrunc(upper + increment - 1, increment) * increment;
2042 try std.testing.expectEqual(expected, (try witness.run(&.{ 0, upper })).asInt().?);
2043 }
2044 for ([_]i64{ 0, 1, 10000 }) |upper| try expectEvaluator(&witness, &.{ 0, upper });
2045 }
2046 }
2047 }
2048
2049 test "aarch64 native control traps only in the iteration reaching an invalid domain" {
2050 if (!supports_native_execution) return error.SkipZigTest;
2051 var witness: Witness = undefined;
2052 try witness.init(.i64, 1, 1);
2053 defer witness.deinit();
2054 const zero = try witness.constant(0);
2055 const one = try witness.constant(1);
2056 const three = try witness.constant(3);
2057 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{zero}, &.{zero.type});
2058 try witness.function.getEntryBlock().addOperation(loop.op);
2059 const before = loop.getBeforeBlock();
2060 const cond = try blockCompare(&witness, before, .slt, before.arguments.items[0], witness.function.getArgument(0));
2061 try blockCondition(&witness, before, cond, before.arguments.items);
2062 const after = loop.getAfterBlock();
2063 const denominator = try blockBinary(&witness, after, ArithDialect.SubOp, three, after.arguments.items[0]);
2064 _ = try blockBinary(&witness, after, ArithDialect.DivOp, one, denominator);
2065 const next = try blockBinary(&witness, after, ArithDialect.AddOp, after.arguments.items[0], one);
2066 try blockYield(&witness, after, &.{next});
2067 try witness.ret(&.{loop.op.getResult(0).?});
2068 for ([_]i64{ 0, 1, 3 }) |upper| try expectEvaluator(&witness, &.{upper});
2069 if (@import("builtin").os.tag == .linux) try expectTrap(&witness, &.{4});
2070 }
2071
2072 test "aarch64 native control nesting depth at limit and one past" {
2073 if (!supports_native_execution) return error.SkipZigTest;
2074 for ([_]usize{ control.max_depth, control.max_depth + 1 }) |depth| {
2075 var witness: Witness = undefined;
2076 try witness.initSignature(.bool, .i64, 1, 1);
2077 defer witness.deinit();
2078 var blocks: [control.max_depth + 1]*ir.Block = undefined;
2079 var block = witness.function.getEntryBlock();
2080 for (blocks[0..depth]) |*nested| {
2081 const branch = try ScfDialect.IfOp.createWithoutElse(&witness.ctx, .unknown, witness.function.getArgument(0));
2082 try block.addOperation(branch.op);
2083 block = branch.getThenBlock();
2084 nested.* = block;
2085 }
2086 for (blocks[0..depth]) |nested| try blockYield(&witness, nested, &.{});
2087 try witness.ret(&.{try witness.constant(42)});
2088 if (depth > control.max_depth) {
2089 try std.testing.expectError(BackendError.UnsupportedOperation, witness.run(&.{1}));
2090 try std.testing.expectEqualStrings("scf.if", witness.diagnostic_operation);
2091 try std.testing.expect(std.mem.indexOf(u8, witness.diagnostic_message, "32 nested") != null);
2092 } else {
2093 try expectEvaluator(&witness, &.{0});
2094 try expectEvaluator(&witness, &.{1});
2095 }
2096 }
2097 }
2098
2099 test "aarch64 control branch displacement at signed encoding limits and one past" {
2100 var witness: Witness = undefined;
2101 try witness.init(.i64, 0, 1);
2102 defer witness.deinit();
2103 var emitter = Emitter.init(witness.arena.allocator());
2104 defer emitter.deinit();
2105 _ = try emitter.branch();
2106 const conditional_limit = @as(usize, std.math.maxInt(i19)) * Instruction.size;
2107 const branch_limit = @as(usize, std.math.maxInt(i26)) * Instruction.size;
2108 try emitter.patchBranch(witness.function.op, 0, conditional_limit, .eq);
2109 try std.testing.expectError(BackendError.UnsupportedOperation, emitter.patchBranch(witness.function.op, 0, conditional_limit + Instruction.size, .eq));
2110 try emitter.patchBranch(witness.function.op, 0, branch_limit, null);
2111 try std.testing.expectError(BackendError.UnsupportedOperation, emitter.patchBranch(witness.function.op, 0, branch_limit + Instruction.size, null));
2112 }
2113
2114 test "aarch64 native control while admits every scalar carried type" {
2115 if (!supports_native_execution) return error.SkipZigTest;
2116 for (integer_kinds ++ [_]dialects.arith.ScalarKind{.bool}) |kind| {
2117 var witness: Witness = undefined;
2118 try witness.init(kind, 1, 1);
2119 defer witness.deinit();
2120 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{witness.function.getArgument(0)}, witness.function.getResultTypes());
2121 try witness.function.getEntryBlock().addOperation(loop.op);
2122 const before = loop.getBeforeBlock();
2123 const cond = try blockConstant(&witness, before, .bool, 0);
2124 try blockCondition(&witness, before, cond, before.arguments.items);
2125 try blockYield(&witness, loop.getAfterBlock(), loop.getAfterBlock().arguments.items);
2126 try witness.ret(&.{loop.op.getResult(0).?});
2127 for ([_]i64{ 0, 1, -1 }) |value| try expectEvaluator(&witness, &.{value});
2128 }
2129 }
2130
2131 test "aarch64 native control zero carried loops and resultless else regions" {
2132 if (!supports_native_execution) return error.SkipZigTest;
2133 var witness: Witness = undefined;
2134 try witness.initSignature(.bool, .i64, 1, 1);
2135 defer witness.deinit();
2136 const branch = try ScfDialect.IfOp.create(&witness.ctx, .unknown, witness.function.getArgument(0), &.{});
2137 try witness.function.getEntryBlock().addOperation(branch.op);
2138 try blockYield(&witness, branch.getThenBlock(), &.{});
2139 try blockYield(&witness, branch.getElseBlock().?, &.{});
2140 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{}, &.{});
2141 try witness.function.getEntryBlock().addOperation(loop.op);
2142 const cond = try blockConstant(&witness, loop.getBeforeBlock(), .bool, 0);
2143 try blockCondition(&witness, loop.getBeforeBlock(), cond, &.{});
2144 try blockYield(&witness, loop.getAfterBlock(), &.{});
2145 try witness.ret(&.{try witness.constant(7)});
2146 try expectEvaluator(&witness, &.{0});
2147 try expectEvaluator(&witness, &.{1});
2148 }
2149
2150 test "aarch64 control if and for carried limits accept full and refuse one past" {
2151 for ([_]usize{ control.max_carried_values, control.max_carried_values + 1 }) |count| {
2152 for ([_]bool{ false, true }) |counted| {
2153 var witness: Witness = undefined;
2154 try witness.init(.index, 0, 1);
2155 defer witness.deinit();
2156 const entry = witness.function.getEntryBlock();
2157 var op: *ir.Operation = undefined;
2158 if (counted) {
2159 const upper = try witness.constant(3);
2160 op = (try makeCounted(&witness, entry, upper, count)).op;
2161 } else {
2162 const cond = try blockConstant(&witness, entry, .bool, 1);
2163 var types: [control.max_carried_values + 1]ir.Type = @splat(witness.function.getResultTypes()[0]);
2164 const branch = try ScfDialect.IfOp.create(&witness.ctx, .unknown, cond, types[0..count]);
2165 try entry.addOperation(branch.op);
2166 for ([_]*ir.Block{ branch.getThenBlock(), branch.getElseBlock().? }, 0..) |block, alternative| {
2167 var values: [control.max_carried_values + 1]*ir.Value = undefined;
2168 for (values[0..count], 0..) |*value, index| value.* = try blockConstant(&witness, block, .index, @intCast(index + alternative));
2169 try blockYield(&witness, block, values[0..count]);
2170 }
2171 op = branch.op;
2172 }
2173 var sum = op.getResult(0).?;
2174 for (1..count) |index| sum = try witness.binary(true, sum, op.getResult(index).?);
2175 try witness.ret(&.{sum});
2176 if (count > control.max_carried_values) {
2177 try std.testing.expectError(BackendError.UnsupportedOperation, witness.backend.compileFunctionToMachineCode(witness.module, "witness"));
2178 try std.testing.expectEqualStrings(if (counted) "scf.for" else "scf.if", witness.diagnostic_operation);
2179 } else {
2180 const bytes = try witness.backend.compileFunctionToMachineCode(witness.module, "witness");
2181 witness.arena.allocator().free(bytes);
2182 if (supports_native_execution) try expectEvaluator(&witness, &.{});
2183 }
2184 }
2185 }
2186 }
2187
2188 test "aarch64 control rejects floating carried values and mismatched condition edges by name" {
2189 for ([_]bool{ false, true }) |floating| {
2190 var witness: Witness = undefined;
2191 try witness.init(.i64, 0, 1);
2192 defer witness.deinit();
2193 const initial = if (floating) value: {
2194 const typ = try ArithDialect.getScalarType(&witness.ctx, .f64);
2195 const constant = try ArithDialect.ConstantOp.createFloat(&witness.ctx, .unknown, typ, 1.0);
2196 try witness.function.getEntryBlock().addOperation(constant.op);
2197 break :value constant.getResult();
2198 } else try witness.constant(1);
2199 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{initial}, &.{initial.type});
2200 try witness.function.getEntryBlock().addOperation(loop.op);
2201 const cond = try blockConstant(&witness, loop.getBeforeBlock(), .bool, 0);
2202 try blockCondition(&witness, loop.getBeforeBlock(), cond, &.{if (floating) loop.getBeforeBlock().arguments.items[0] else cond});
2203 try blockYield(&witness, loop.getAfterBlock(), loop.getAfterBlock().arguments.items);
2204 try witness.ret(&.{try witness.constant(0)});
2205 try std.testing.expectError(BackendError.UnsupportedOperation, witness.backend.compileFunctionToMachineCode(witness.module, "witness"));
2206 try std.testing.expectEqualStrings("scf.while", witness.diagnostic_operation);
2207 }
2208 }
2209
2210 test "aarch64 native control for parallel carried rotations and nested counted loops" {
2211 if (!supports_native_execution) return error.SkipZigTest;
2212 for ([_]bool{ true, false }) |nested| {
2213 var witness: Witness = undefined;
2214 try witness.init(.index, 1, 1);
2215 defer witness.deinit();
2216 const entry = witness.function.getEntryBlock();
2217 const zero = try witness.constant(0);
2218 const one = try witness.constant(1);
2219 const count = control.max_carried_values;
2220 var initial: [count]*ir.Value = undefined;
2221 var types: [count]ir.Type = @splat(zero.type);
2222 for (&initial, 0..) |*value, index| value.* = try witness.constant(@intCast(index + 1));
2223 const loop = try ScfDialect.ForOp.create(&witness.ctx, .unknown, zero, witness.function.getArgument(0), one, &initial, &types);
2224 try entry.addOperation(loop.op);
2225 const body = loop.getBodyBlock();
2226 var yielded: [count]*ir.Value = undefined;
2227 for (&yielded, 0..) |*value, index| value.* = body.arguments.items[1 + (index + 1) % count];
2228 if (nested) {
2229 const inner = try makeCounted(&witness, body, body.arguments.items[0], 1);
2230 yielded[0] = try blockBinary(&witness, body, ArithDialect.AddOp, yielded[0], inner.op.getResult(0).?);
2231 }
2232 try blockYield(&witness, body, &yielded);
2233 var sum = loop.op.getResult(0).?;
2234 for (1..count) |index| {
2235 const weight = try witness.constant(@intCast(index + 1));
2236 const product = try blockBinary(&witness, entry, ArithDialect.MulOp, loop.op.getResult(index).?, weight);
2237 sum = try witness.binary(true, sum, product);
2238 }
2239 try witness.ret(&.{sum});
2240 for ([_]i64{ 0, 1, 7 }) |upper| {
2241 var expected_values: [count]i64 = undefined;
2242 for (&expected_values, 0..) |*value, index| value.* = @intCast(index + 1);
2243 for (0..@intCast(upper)) |iteration| {
2244 const saved = expected_values[0];
2245 for (0..count - 1) |index| expected_values[index] = expected_values[index + 1];
2246 expected_values[count - 1] = saved;
2247 if (nested and iteration > 0) expected_values[0] += @intCast(iteration * (iteration - 1) / 2);
2248 }
2249 var expected: i64 = 0;
2250 for (expected_values, 0..) |value, index| expected += value * @as(i64, @intCast(index + 1));
2251 try std.testing.expectEqual(expected, (try witness.run(&.{upper})).asInt().?);
2252 }
2253 for ([_]i64{ 0, 1, 7 }) |upper| try expectEvaluator(&witness, &.{upper});
2254 }
2255 }
2256
2257 test "aarch64 native control while with constant bounds matches evaluator" {
2258 if (!supports_native_execution) return error.SkipZigTest;
2259 for ([_]i64{ 0, 1, 19 }) |upper_value| {
2260 var witness: Witness = undefined;
2261 try witness.init(.i64, 0, 1);
2262 defer witness.deinit();
2263 const zero = try witness.constant(0);
2264 const one = try witness.constant(1);
2265 const upper = try witness.constant(upper_value);
2266 const loop = try makeWhile(&witness, witness.function.getEntryBlock(), zero, upper, one);
2267 try witness.ret(&.{loop.op.getResult(0).?});
2268 try expectEvaluator(&witness, &.{});
2269 }
2270 }
2271
2272 test "aarch64 native control constant bounded loop reaches its trap only on iteration four" {
2273 if (!supports_native_execution) return error.SkipZigTest;
2274 for ([_]i64{ 0, 1, 3, 4 }) |upper_value| {
2275 var witness: Witness = undefined;
2276 try witness.init(.i64, 0, 1);
2277 defer witness.deinit();
2278 const zero = try witness.constant(0);
2279 const one = try witness.constant(1);
2280 const three = try witness.constant(3);
2281 const upper = try witness.constant(upper_value);
2282 const loop = try ScfDialect.WhileOp.create(&witness.ctx, .unknown, &.{zero}, &.{zero.type});
2283 try witness.function.getEntryBlock().addOperation(loop.op);
2284 const before = loop.getBeforeBlock();
2285 const cond = try blockCompare(&witness, before, .slt, before.arguments.items[0], upper);
2286 try blockCondition(&witness, before, cond, before.arguments.items);
2287 const after = loop.getAfterBlock();
2288 const denominator = try blockBinary(&witness, after, ArithDialect.SubOp, three, after.arguments.items[0]);
2289 _ = try blockBinary(&witness, after, ArithDialect.DivOp, one, denominator);
2290 const next = try blockBinary(&witness, after, ArithDialect.AddOp, after.arguments.items[0], one);
2291 try blockYield(&witness, after, &.{next});
2292 try witness.ret(&.{loop.op.getResult(0).?});
2293 if (upper_value < 4) {
2294 try expectEvaluator(&witness, &.{});
2295 } else if (@import("builtin").os.tag == .linux) {
2296 try expectTrap(&witness, &.{});
2297 }
2298 }
2299 }
2300
2301 test "aarch64 native control counted induction bounds step and empty yield match evaluator" {
2302 if (!supports_native_execution) return error.SkipZigTest;
2303 const Case = struct { lower: i64, upper: i64, step: i64, carried: bool };
2304 for ([_]Case{
2305 .{ .lower = -3, .upper = 4, .step = 2, .carried = true },
2306 .{ .lower = 5, .upper = 5, .step = 1, .carried = true },
2307 .{ .lower = 5, .upper = 3, .step = 1, .carried = true },
2308 .{ .lower = 0, .upper = 7, .step = 2, .carried = false },
2309 }) |case| {
2310 var witness: Witness = undefined;
2311 try witness.init(.index, 0, 1);
2312 defer witness.deinit();
2313 const lower = try witness.constant(case.lower);
2314 const upper = try witness.constant(case.upper);
2315 const step = try witness.constant(case.step);
2316 const initial = try witness.constant(7);
2317 const loop = try ScfDialect.ForOp.create(&witness.ctx, .unknown, lower, upper, step, if (case.carried) &.{initial} else &.{}, if (case.carried) &.{initial.type} else &.{});
2318 try witness.function.getEntryBlock().addOperation(loop.op);
2319 const body = loop.getBodyBlock();
2320 if (case.carried) {
2321 const sum = try blockBinary(&witness, body, ArithDialect.AddOp, body.arguments.items[1], body.arguments.items[0]);
2322 try blockYield(&witness, body, &.{sum});
2323 } else try blockYield(&witness, body, &.{});
2324 try witness.ret(&.{if (case.carried) loop.op.getResult(0).? else initial});
2325 try expectEvaluator(&witness, &.{});
2326 }
2327 }