lib/choir/src/backends/x64/scalar.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const zig_builtin = @import("builtin");
3 const ir = @import("../../core/root.zig");
4 const dialects = @import("../../dialects/root.zig");
5 const machine = @import("../root.zig").machine_code;
6 const call_plan = @import("calls.zig");
7 const cast = @import("cast.zig");
8 const data = @import("data.zig");
9 const encoding = @import("encoding.zig");
10 const labels = @import("labels.zig");
11 const registers = @import("registers/root.zig");
12 const slot_layout = @import("slots.zig");
13
14 const cpu_has_fma3 = blk: {
15 const features = zig_builtin.cpu.features;
16 const FeatureType = std.Target.x86.Feature;
17 break :blk std.Target.x86.featureSetHas(features, FeatureType.fma);
18 };
19
20 const cpu_has_popcnt = blk: {
21 const features = zig_builtin.cpu.features;
22 const FeatureType = std.Target.x86.Feature;
23 break :blk std.Target.x86.featureSetHas(features, FeatureType.popcnt);
24 };
25
26 const arith = dialects.arith;
27 const ArithDialect = arith.ArithDialect;
28 const CmpPredicate = arith.CmpPredicate;
29 const Condition = encoding.Condition;
30 const Label = labels.Label;
31 const Slot = slot_layout.Slot;
32 const ExternArg = call_plan.ExternArg;
33 const ExternResult = call_plan.ExternResult;
34
35 pub fn isUnsignedIntegerTypeName(type_name: []const u8) bool {
36 return std.mem.eql(u8, type_name, "arith.u8") or
37 std.mem.eql(u8, type_name, "arith.u16") or
38 std.mem.eql(u8, type_name, "arith.u32") or
39 std.mem.eql(u8, type_name, "arith.u64") or
40 std.mem.eql(u8, type_name, "arith.index");
41 }
42
43 fn isUnsignedScalarTypeName(type_name: []const u8) bool {
44 return std.mem.eql(u8, type_name, "arith.u8") or
45 std.mem.eql(u8, type_name, "arith.u16") or
46 std.mem.eql(u8, type_name, "arith.u32") or
47 std.mem.eql(u8, type_name, "arith.u64");
48 }
49
50 pub fn isIntegerScalarTypeName(type_name: []const u8) bool {
51 return std.mem.eql(u8, type_name, "arith.i8") or
52 std.mem.eql(u8, type_name, "arith.i16") or
53 std.mem.eql(u8, type_name, "arith.i32") or
54 std.mem.eql(u8, type_name, "arith.i64") or
55 isUnsignedScalarTypeName(type_name);
56 }
57
58 fn isFloatScalarTypeName(type_name: []const u8) bool {
59 return std.mem.eql(u8, type_name, "arith.f16") or
60 std.mem.eql(u8, type_name, "arith.bf16") or
61 std.mem.eql(u8, type_name, "arith.f32") or
62 std.mem.eql(u8, type_name, "arith.f64");
63 }
64
65 pub fn emitArithConstant(self: anytype, op: *ir.Operation) !void {
66 if (try data.symbolOf(op)) |symbol| return emitDataAddress(self, op, symbol);
67 const const_op = ArithDialect.ConstantOp{ .op = op };
68 const result = const_op.getResult();
69 const slot = try self.slotFor(result);
70 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
71
72 if (std.mem.eql(u8, type_name, "arith.bool")) {
73 const bool_attr = op.getAttrAs(ir.Attribute.BoolAttr, "value") orelse return error.InvalidConstant;
74 const bool_val = bool_attr.getValue();
75 const imm: u64 = if (bool_val) 1 else 0;
76 try self.emitEncoding(encoding.movRegImm64(.rax, imm));
77 try self.storeFrom(result, .rax);
78 return;
79 }
80
81 if (const_op.getIntValue()) |int_val| {
82 const bits: u64 = @bitCast(int_val);
83 const masked: u64 = switch (slot.width) {
84 8 => bits & 0xFF,
85 16 => bits & 0xFFFF,
86 32 => bits & 0xFFFF_FFFF,
87 64 => bits,
88 else => return error.UnsupportedType,
89 };
90 if (slot.width <= 32) {
91 try self.emitEncoding(encoding.movRegImm32(.rax, @intCast(masked)));
92 } else {
93 try self.emitEncoding(encoding.movRegImm64(.rax, masked));
94 }
95 try self.storeFrom(result, .rax);
96 return;
97 }
98
99 if (const_op.getFloatValue()) |float_val| {
100 if (slot.width == 32) {
101 const bits: u32 = @bitCast(@as(f32, @floatCast(float_val)));
102 try self.emitEncoding(encoding.movRegImm32(.rax, bits));
103 } else if (slot.width == 64) {
104 const bits: u64 = @bitCast(float_val);
105 try self.emitEncoding(encoding.movRegImm64(.rax, bits));
106 } else {
107 return error.UnsupportedType;
108 }
109 if (self.xmmHomeForPhase(result, .definition)) |home| {
110 try self.emitEncoding(encoding.movqXmmFromReg64(home, .rax));
111 } else {
112 try self.storeSlot(slot, .rax);
113 }
114 return;
115 }
116
117 return error.UnsupportedType;
118 }
119
120 /// Loads the address of `symbol` through a 64-bit immediate that a data relocation fills.
121 fn emitDataAddress(self: anytype, op: *ir.Operation, symbol: machine.DataSymbol) !void {
122 const result = (ArithDialect.ConstantOp{ .op = op }).getResult();
123 try data.emitAddress(self, result, symbol);
124 }
125
126 pub const BinOp = enum {
127 add,
128 sub,
129 mul,
130 div,
131 rem,
132 };
133
134 pub fn emitIntegerDiv(self: anytype, op_width: u8, is_unsigned: bool) !void {
135 if (is_unsigned) {
136 if (op_width == 32) {
137 try self.emitEncoding(encoding.xorRegReg32(.rdx, .rdx));
138 try self.emitEncoding(encoding.divReg32(.rcx));
139 } else {
140 try self.emitEncoding(encoding.xorRegReg(.rdx, .rdx));
141 try self.emitEncoding(encoding.divReg(.rcx));
142 }
143 return;
144 }
145
146 if (op_width == 32) {
147 try self.emitEncoding(encoding.cdq());
148 try self.emitEncoding(encoding.idivReg32(.rcx));
149 } else {
150 try self.emitEncoding(encoding.cqo());
151 try self.emitEncoding(encoding.idivReg(.rcx));
152 }
153 }
154
155 pub fn shouldSkipArithConstant(op: *ir.Operation) bool {
156 const result = op.getResult(0) orelse return false;
157 if (constantIntI32(result) == null) return false;
158
159 var use = result.first_use;
160 while (use) |operand| {
161 const user: *ir.Operation = @ptrCast(@alignCast(operand.owner));
162 if (!immediateUseSupported(user, operand.operand_number)) return false;
163 use = operand.next_use;
164 }
165 return true;
166 }
167
168 pub fn emitBinaryIntOp(self: anytype, op: *ir.Operation, kind: BinOp) !void {
169 const result = op.getResult(0) orelse return error.MissingResult;
170 const lhs = op.getOperand(0) orelse return error.MissingOperand;
171 const rhs = op.getOperand(1) orelse return error.MissingOperand;
172
173 const result_slot = try self.slotFor(result);
174
175 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
176 const is_float = std.mem.eql(u8, type_name, "arith.f32") or std.mem.eql(u8, type_name, "arith.f64");
177 const is_unsigned = isUnsignedIntegerTypeName(type_name);
178 if ((kind == .div or kind == .rem) and std.mem.eql(u8, type_name, "arith.bool")) return error.UnsupportedType;
179
180 if (is_float) {
181 try self.loadIntoXmm(lhs, .xmm0);
182 try self.loadIntoXmm(rhs, .xmm1);
183 if (result_slot.width == 32) {
184 switch (kind) {
185 .add => try self.emitEncoding(encoding.addss(.xmm0, .{ .reg = .xmm1 })),
186 .sub => try self.emitEncoding(encoding.subss(.xmm0, .{ .reg = .xmm1 })),
187 .mul => try self.emitEncoding(encoding.mulss(.xmm0, .{ .reg = .xmm1 })),
188 .div => try self.emitEncoding(encoding.divss(.xmm0, .{ .reg = .xmm1 })),
189 .rem => return error.UnsupportedType,
190 }
191 } else if (result_slot.width == 64) {
192 switch (kind) {
193 .add => try self.emitEncoding(encoding.addsd(.xmm0, .{ .reg = .xmm1 })),
194 .sub => try self.emitEncoding(encoding.subsd(.xmm0, .{ .reg = .xmm1 })),
195 .mul => try self.emitEncoding(encoding.mulsd(.xmm0, .{ .reg = .xmm1 })),
196 .div => try self.emitEncoding(encoding.divsd(.xmm0, .{ .reg = .xmm1 })),
197 .rem => return error.UnsupportedType,
198 }
199 } else {
200 return error.UnsupportedType;
201 }
202 try self.storeFromXmm(result, .xmm0);
203 return;
204 }
205
206 const width = result_slot.width;
207 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
208 const op_width: u8 = if (width <= 32) 32 else 64;
209
210 if (kind == .sub) {
211 if (constantIntI32(rhs)) |imm| {
212 try self.loadInto(lhs, .rax);
213 if (op_width == 32) {
214 try self.emitEncoding(encoding.subRegImm32(.rax, imm));
215 } else {
216 try self.emitEncoding(encoding.subRegImm(.rax, imm));
217 }
218 try self.storeFrom(result, .rax);
219 return;
220 }
221 }
222
223 try self.loadInto(lhs, .rax);
224 try self.loadInto(rhs, .rcx);
225
226 switch (kind) {
227 .add => {
228 if (op_width == 32) {
229 try self.emitEncoding(encoding.addRegReg32(.rax, .rcx));
230 } else {
231 try self.emitEncoding(encoding.addRegReg(.rax, .rcx));
232 }
233 try self.storeFrom(result, .rax);
234 },
235 .sub => {
236 if (op_width == 32) {
237 try self.emitEncoding(encoding.subRegReg32(.rax, .rcx));
238 } else {
239 try self.emitEncoding(encoding.subRegReg(.rax, .rcx));
240 }
241 try self.storeFrom(result, .rax);
242 },
243 .mul => {
244 if (op_width == 32) {
245 try self.emitEncoding(encoding.imulRegReg32(.rax, .rcx));
246 } else {
247 try self.emitEncoding(encoding.imulRegReg(.rax, .rcx));
248 }
249 try self.storeFrom(result, .rax);
250 },
251 .div => {
252 try emitIntegerDiv(self, op_width, is_unsigned);
253 try self.storeFrom(result, .rax);
254 },
255 .rem => {
256 try emitIntegerDiv(self, op_width, is_unsigned);
257 try self.storeFrom(result, .rdx);
258 },
259 }
260 }
261
262 /// The arithmetic instruction must write OF even for constant operands. In particular, this
263 /// path never calls the ordinary binary emitter, whose lea/shift forms do not answer overflow.
264 /// The allocator fixes the two definitions to rax and rcx, so settling the value cannot
265 /// overwrite the flag. A materialized bool captures OF before either result settles.
266 /// A flags-only bool leaves OF live through storeFrom's mov (or no instruction), then
267 /// the adjacent branch's allocator transitions, which emit only flag-preserving moves.
268 pub fn emitOverflow(self: anytype, op: *ir.Operation, kind: enum { add, sub, mul }) !void {
269 const result = op.getResult(0) orelse return error.MissingResult;
270 const overflow = op.getResult(1) orelse return error.MissingResult;
271 const lhs = op.getOperand(0) orelse return error.MissingOperand;
272 const rhs = op.getOperand(1) orelse return error.MissingOperand;
273 if (op.getNumResults() != 2 or op.getNumOperands() != 2) return error.UnsupportedOperation;
274 if (arith.scalarKindFromType(lhs.type) != .i64 or
275 arith.scalarKindFromType(rhs.type) != .i64 or
276 arith.scalarKindFromType(result.type) != .i64 or
277 arith.scalarKindFromType(overflow.type) != .bool) return error.UnsupportedType;
278
279 try self.loadInto(lhs, .rax);
280 try self.loadInto(rhs, .rcx);
281 try self.emitEncoding(switch (kind) {
282 .add => encoding.addRegReg(.rax, .rcx),
283 .sub => encoding.subRegReg(.rax, .rcx),
284 .mul => encoding.imulRegReg(.rax, .rcx),
285 });
286 const flags_only = self.flagsOnly(overflow);
287 if (!flags_only) {
288 try self.emitEncoding(encoding.setcc(.rcx, .o));
289 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
290 }
291 try self.storeFrom(result, .rax);
292 if (!flags_only) try self.storeFrom(overflow, .rcx);
293 }
294
295 pub fn emitArithUmulhi(self: anytype, op: *ir.Operation) !void {
296 const result = op.getResult(0) orelse return error.MissingResult;
297 const lhs = op.getOperand(0) orelse return error.MissingOperand;
298 const rhs = op.getOperand(1) orelse return error.MissingOperand;
299
300 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
301 const lhs_type_name = lhs.type.getDialectTypeName() orelse return error.UnsupportedType;
302 const rhs_type_name = rhs.type.getDialectTypeName() orelse return error.UnsupportedType;
303 if (!std.mem.eql(u8, type_name, lhs_type_name)) return error.UnsupportedType;
304 if (!std.mem.eql(u8, type_name, rhs_type_name)) return error.UnsupportedType;
305 if (arith.parseVectorTypeName(type_name) != null) return error.UnsupportedType;
306 if (!isIntegerScalarTypeName(type_name)) return error.UnsupportedType;
307
308 const result_slot = try self.slotFor(result);
309 const width = result_slot.width;
310 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
311
312 try self.loadIntoUnsigned(lhs, .rax);
313 try self.loadIntoUnsigned(rhs, .rcx);
314 try emitUnsignedHighProductFromRaxRcx(self, width);
315 try self.storeFrom(result, .rax);
316 }
317
318 pub fn emitUnsignedHighProductFromRaxRcx(self: anytype, width: u8) !void {
319 switch (width) {
320 8, 16 => {
321 try self.emitEncoding(encoding.imulRegReg32(.rax, .rcx));
322 try self.emitEncoding(encoding.shrRegImm(.rax, width));
323 },
324 32 => {
325 try self.emitEncoding(encoding.mulReg32(.rcx));
326 try self.emitEncoding(encoding.movRegReg32(.rax, .rdx));
327 },
328 64 => {
329 try self.emitEncoding(encoding.mulReg(.rcx));
330 try self.emitEncoding(encoding.movRegReg(.rax, .rdx));
331 },
332 else => return error.UnsupportedType,
333 }
334 }
335
336 pub fn emitArithPopCount(self: anytype, op: *ir.Operation) !void {
337 const result = op.getResult(0) orelse return error.MissingResult;
338 const input = op.getOperand(0) orelse return error.MissingOperand;
339
340 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
341 const input_type_name = input.type.getDialectTypeName() orelse return error.UnsupportedType;
342 if (!std.mem.eql(u8, type_name, input_type_name)) return error.UnsupportedType;
343 if (arith.parseVectorTypeName(type_name) != null) return error.UnsupportedType;
344 if (std.mem.eql(u8, type_name, "arith.bool") or isFloatScalarTypeName(type_name)) return error.UnsupportedType;
345 if (!isIntegerScalarTypeName(type_name) and !std.mem.eql(u8, type_name, "arith.index")) return error.UnsupportedType;
346
347 const result_slot = try self.slotFor(result);
348 const width = result_slot.width;
349 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
350
351 try self.loadIntoUnsigned(input, .rax);
352 try emitPopCountFromRax(self, width);
353 try self.storeFrom(result, .rax);
354 }
355
356 pub fn emitPopCountFromRax(self: anytype, width: u8) !void {
357 if (cpu_has_popcnt) {
358 if (width == 64) {
359 try self.emitEncoding(encoding.popcntRegReg(.rax, .rax));
360 } else {
361 try self.emitEncoding(encoding.popcntRegReg32(.rax, .rax));
362 }
363 } else {
364 try emitPopCountSoftware(self);
365 }
366 }
367
368 fn emitPopCountSoftware(self: anytype) !void {
369 try self.emitEncoding(encoding.movRegReg(.rdx, .rax));
370 try self.emitEncoding(encoding.shrRegImm(.rdx, 1));
371 try self.emitEncoding(encoding.movRegImm64(.rcx, 0x5555_5555_5555_5555));
372 try self.emitEncoding(encoding.andRegReg(.rdx, .rcx));
373 try self.emitEncoding(encoding.subRegReg(.rax, .rdx));
374
375 try self.emitEncoding(encoding.movRegImm64(.rcx, 0x3333_3333_3333_3333));
376 try self.emitEncoding(encoding.movRegReg(.rdx, .rax));
377 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
378 try self.emitEncoding(encoding.shrRegImm(.rdx, 2));
379 try self.emitEncoding(encoding.andRegReg(.rdx, .rcx));
380 try self.emitEncoding(encoding.addRegReg(.rax, .rdx));
381
382 try self.emitEncoding(encoding.movRegReg(.rdx, .rax));
383 try self.emitEncoding(encoding.shrRegImm(.rdx, 4));
384 try self.emitEncoding(encoding.addRegReg(.rax, .rdx));
385 try self.emitEncoding(encoding.movRegImm64(.rcx, 0x0f0f_0f0f_0f0f_0f0f));
386 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
387 try self.emitEncoding(encoding.movRegImm64(.rcx, 0x0101_0101_0101_0101));
388 try self.emitEncoding(encoding.imulRegReg(.rax, .rcx));
389 try self.emitEncoding(encoding.shrRegImm(.rax, 56));
390 }
391
392 pub fn emitArithNeg(self: anytype, op: *ir.Operation) !void {
393 const result = op.getResult(0) orelse return error.MissingResult;
394 const operand = op.getOperand(0) orelse return error.MissingOperand;
395
396 const result_slot = try self.slotFor(result);
397
398 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
399 if (std.mem.eql(u8, type_name, "arith.index")) return error.UnsupportedType;
400 if (std.mem.eql(u8, type_name, "arith.f16") or
401 std.mem.eql(u8, type_name, "arith.bf16") or
402 std.mem.eql(u8, type_name, "arith.bool")) return error.UnsupportedType;
403
404 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
405 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
406 if (is_f32 or is_f64) {
407 return emitArithNegFloat(self, result, operand, is_f32);
408 }
409
410 const width = result_slot.width;
411 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
412 const op_width: u8 = if (width <= 32) 32 else 64;
413
414 try self.loadInto(operand, .rax);
415 if (op_width == 32) {
416 try self.emitEncoding(encoding.negReg32(.rax));
417 } else {
418 try self.emitEncoding(encoding.negReg(.rax));
419 }
420 try self.storeFrom(result, .rax);
421 }
422
423 fn emitArithNegFloat(
424 self: anytype,
425 result: *ir.Value,
426 operand: *ir.Value,
427 is_f32: bool,
428 ) !void {
429 try self.loadIntoXmm(operand, .xmm0);
430 try self.emitEncoding(encoding.pcmpeqd(.xmm1, .{ .reg = .xmm1 }));
431 if (is_f32) {
432 try self.emitEncoding(encoding.pslldImm(.xmm1, 31));
433 try self.emitEncoding(encoding.xorps(.xmm0, .{ .reg = .xmm1 }));
434 } else {
435 try self.emitEncoding(encoding.psllqImm(.xmm1, 63));
436 try self.emitEncoding(encoding.xorpd(.xmm0, .{ .reg = .xmm1 }));
437 }
438 try self.storeFromXmm(result, .xmm0);
439 }
440
441 pub const MinMaxKind = enum { max, min };
442
443 pub fn emitArithMax(self: anytype, op: *ir.Operation) !void {
444 return emitArithMinMax(self, op, .max);
445 }
446
447 pub fn emitArithMin(self: anytype, op: *ir.Operation) !void {
448 return emitArithMinMax(self, op, .min);
449 }
450
451 fn emitArithMinMax(self: anytype, op: *ir.Operation, kind: MinMaxKind) !void {
452 const result = op.getResult(0) orelse return error.MissingResult;
453 const lhs = op.getOperand(0) orelse return error.MissingOperand;
454 const rhs = op.getOperand(1) orelse return error.MissingOperand;
455
456 const result_slot = try self.slotFor(result);
457
458 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
459 const lhs_type_name = lhs.type.getDialectTypeName() orelse return error.UnsupportedType;
460 const rhs_type_name = rhs.type.getDialectTypeName() orelse return error.UnsupportedType;
461 if (!std.mem.eql(u8, type_name, lhs_type_name)) return error.UnsupportedType;
462 if (!std.mem.eql(u8, type_name, rhs_type_name)) return error.UnsupportedType;
463
464 if (std.mem.eql(u8, type_name, "arith.f16") or
465 std.mem.eql(u8, type_name, "arith.bf16") or
466 std.mem.eql(u8, type_name, "arith.bool")) return error.UnsupportedType;
467
468 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
469 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
470 if (is_f32 or is_f64) {
471 return emitArithMinMaxFloat(self, result, lhs, rhs, is_f32, kind);
472 }
473
474 const is_unsigned = isUnsignedIntegerTypeName(type_name);
475
476 const width = result_slot.width;
477 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
478 const op_width: u8 = if (width <= 32) 32 else 64;
479
480 if (is_unsigned) {
481 try self.loadIntoUnsigned(lhs, .rax);
482 try self.loadIntoUnsigned(rhs, .rcx);
483 } else {
484 try self.loadInto(lhs, .rax);
485 try self.loadInto(rhs, .rcx);
486 }
487
488 try emitIntegerMinMax(self, op_width, is_unsigned, kind);
489 try self.storeFrom(result, .rax);
490 }
491
492 pub fn emitIntegerMinMax(self: anytype, op_width: u8, is_unsigned: bool, kind: MinMaxKind) !void {
493 if (op_width == 32) {
494 try self.emitEncoding(encoding.cmpRegReg32(.rax, .rcx));
495 if (is_unsigned) {
496 switch (kind) {
497 .max => try self.emitEncoding(encoding.cmovbRegReg32(.rax, .rcx)),
498 .min => try self.emitEncoding(encoding.cmovaRegReg32(.rax, .rcx)),
499 }
500 } else {
501 switch (kind) {
502 .max => try self.emitEncoding(encoding.cmovlRegReg32(.rax, .rcx)),
503 .min => try self.emitEncoding(encoding.cmovgRegReg32(.rax, .rcx)),
504 }
505 }
506 } else {
507 try self.emitEncoding(encoding.cmpRegReg(.rax, .rcx));
508 if (is_unsigned) {
509 switch (kind) {
510 .max => try self.emitEncoding(encoding.cmovbRegReg(.rax, .rcx)),
511 .min => try self.emitEncoding(encoding.cmovaRegReg(.rax, .rcx)),
512 }
513 } else {
514 switch (kind) {
515 .max => try self.emitEncoding(encoding.cmovlRegReg(.rax, .rcx)),
516 .min => try self.emitEncoding(encoding.cmovgRegReg(.rax, .rcx)),
517 }
518 }
519 }
520 }
521
522 fn emitArithMinMaxFloat(
523 self: anytype,
524 result: *ir.Value,
525 lhs: *ir.Value,
526 rhs: *ir.Value,
527 is_f32: bool,
528 kind: MinMaxKind,
529 ) !void {
530 try self.loadIntoXmm(lhs, .xmm0);
531 try self.loadIntoXmm(rhs, .xmm1);
532 try emitFloatMinMax(self, is_f32, kind);
533 try self.storeFromXmm(result, .xmm0);
534 }
535
536 pub fn emitFloatMinMax(
537 self: anytype,
538 is_f32: bool,
539 kind: MinMaxKind,
540 ) !void {
541 if (is_f32) {
542 try self.emitEncoding(encoding.movss(.xmm2, .{ .reg = .xmm0 }));
543 } else {
544 try self.emitEncoding(encoding.movsd(.xmm2, .{ .reg = .xmm0 }));
545 }
546
547 switch (kind) {
548 .max => {
549 if (is_f32) {
550 try self.emitEncoding(encoding.maxss(.xmm0, .{ .reg = .xmm1 }));
551 } else {
552 try self.emitEncoding(encoding.maxsd(.xmm0, .{ .reg = .xmm1 }));
553 }
554 },
555 .min => {
556 if (is_f32) {
557 try self.emitEncoding(encoding.minss(.xmm0, .{ .reg = .xmm1 }));
558 } else {
559 try self.emitEncoding(encoding.minsd(.xmm0, .{ .reg = .xmm1 }));
560 }
561 },
562 }
563
564 if (is_f32) {
565 try self.emitEncoding(encoding.ucomiss(.xmm0, .{ .reg = .xmm0 }));
566 } else {
567 try self.emitEncoding(encoding.ucomisd(.xmm0, .{ .reg = .xmm0 }));
568 }
569
570 var done_label = Label{};
571 defer done_label.deinit(self.allocator);
572 try self.emitJccLabel(.np, &done_label);
573
574 if (is_f32) {
575 try self.emitEncoding(encoding.movss(.xmm0, .{ .reg = .xmm2 }));
576 } else {
577 try self.emitEncoding(encoding.movsd(.xmm0, .{ .reg = .xmm2 }));
578 }
579
580 try self.bindLabel(&done_label);
581 }
582
583 pub fn emitArithSqrt(self: anytype, op: *ir.Operation) !void {
584 const result = op.getResult(0) orelse return error.MissingResult;
585 const operand = op.getOperand(0) orelse return error.MissingOperand;
586
587 const result_slot = try self.slotFor(result);
588 const operand_slot = try self.slotFor(operand);
589
590 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
591 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
592 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
593 if (!is_f32 and !is_f64) return error.UnsupportedType;
594
595 try self.loadSlotXmm(operand_slot, .xmm0);
596 if (is_f32) {
597 try self.emitEncoding(encoding.sqrtss(.xmm0, .{ .reg = .xmm0 }));
598 } else {
599 try self.emitEncoding(encoding.sqrtsd(.xmm0, .{ .reg = .xmm0 }));
600 }
601 try self.storeSlotXmm(result_slot, .xmm0);
602 }
603
604 pub fn emitArithAbs(self: anytype, op: *ir.Operation) !void {
605 const result = op.getResult(0) orelse return error.MissingResult;
606 const operand = op.getOperand(0) orelse return error.MissingOperand;
607
608 const result_slot = try self.slotFor(result);
609
610 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
611 if (std.mem.eql(u8, type_name, "arith.index")) return error.UnsupportedType;
612 if (std.mem.eql(u8, type_name, "arith.f16") or
613 std.mem.eql(u8, type_name, "arith.bf16") or
614 std.mem.eql(u8, type_name, "arith.bool")) return error.UnsupportedType;
615
616 if (isUnsignedScalarTypeName(type_name)) {
617 try self.loadIntoUnsigned(operand, .rax);
618 try self.storeFrom(result, .rax);
619 return;
620 }
621
622 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
623 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
624 if (is_f32 or is_f64) {
625 return emitArithAbsFloat(self, result_slot, try self.slotFor(operand), is_f32);
626 }
627
628 const width = result_slot.width;
629 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
630 const op_width: u8 = if (width <= 32) 32 else 64;
631
632 try self.loadInto(operand, .rax);
633 if (op_width == 32) {
634 try self.emitEncoding(encoding.movRegReg32(.rcx, .rax));
635 try self.emitEncoding(encoding.negReg32(.rcx));
636 try self.emitEncoding(encoding.cmovnsRegReg32(.rax, .rcx));
637 } else {
638 try self.emitEncoding(encoding.movRegReg(.rcx, .rax));
639 try self.emitEncoding(encoding.negReg(.rcx));
640 try self.emitEncoding(encoding.cmovnsRegReg(.rax, .rcx));
641 }
642 try self.storeFrom(result, .rax);
643 }
644
645 fn emitArithAbsFloat(
646 self: anytype,
647 result_slot: anytype,
648 operand_slot: anytype,
649 is_f32: bool,
650 ) !void {
651 try self.loadSlotXmm(operand_slot, .xmm0);
652 try self.emitEncoding(encoding.pcmpeqd(.xmm1, .{ .reg = .xmm1 }));
653 if (is_f32) {
654 try self.emitEncoding(encoding.psrldImm(.xmm1, 1));
655 try self.emitEncoding(encoding.andps(.xmm0, .{ .reg = .xmm1 }));
656 } else {
657 try self.emitEncoding(encoding.psrlqImm(.xmm1, 1));
658 try self.emitEncoding(encoding.andpd(.xmm0, .{ .reg = .xmm1 }));
659 }
660 try self.storeSlotXmm(result_slot, .xmm0);
661 }
662
663 pub const LibmUnaryKind = enum { sin, cos, tan, exp, log, tanh, floor };
664 pub const LibmBinaryKind = enum { pow };
665 const LibmTernaryKind = enum { fma };
666
667 fn libmUnaryCallee(kind: LibmUnaryKind, is_f32: bool) []const u8 {
668 return switch (kind) {
669 .sin => if (is_f32) "sinf" else "sin",
670 .cos => if (is_f32) "cosf" else "cos",
671 .tan => if (is_f32) "tanf" else "tan",
672 .exp => if (is_f32) "expf" else "exp",
673 .log => if (is_f32) "logf" else "log",
674 .tanh => if (is_f32) "tanhf" else "tanh",
675 .floor => if (is_f32) "floorf" else "floor",
676 };
677 }
678
679 fn libmBinaryCallee(kind: LibmBinaryKind, is_f32: bool) []const u8 {
680 return switch (kind) {
681 .pow => if (is_f32) "powf" else "pow",
682 };
683 }
684
685 fn libmTernaryCallee(kind: LibmTernaryKind, is_f32: bool) []const u8 {
686 return switch (kind) {
687 .fma => if (is_f32) "fmaf" else "fma",
688 };
689 }
690
691 pub fn emitArithLibmUnary(self: anytype, op: *ir.Operation, kind: LibmUnaryKind) !void {
692 const result = op.getResult(0) orelse return error.MissingResult;
693 const operand = op.getOperand(0) orelse return error.MissingOperand;
694
695 const result_slot = try self.slotFor(result);
696 const operand_slot = try self.slotFor(operand);
697
698 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
699 const operand_type_name = operand.type.getDialectTypeName() orelse return error.UnsupportedType;
700 if (!std.mem.eql(u8, type_name, operand_type_name)) return error.UnsupportedType;
701
702 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
703 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
704 if (!is_f32 and !is_f64) return error.UnsupportedType;
705
706 const callee = libmUnaryCallee(kind, is_f32);
707 const args = [_]ExternArg{
708 .{ .type_name = type_name, .source = .{ .slot = operand_slot } },
709 };
710 const call_result: ExternResult = .{ .type_name = type_name, .slot = result_slot };
711 try call_plan.emitExtern(self, callee, &args, &.{call_result});
712 }
713
714 pub fn emitArithLibmBinary(self: anytype, op: *ir.Operation, kind: LibmBinaryKind) !void {
715 const result = op.getResult(0) orelse return error.MissingResult;
716 const base = op.getOperand(0) orelse return error.MissingOperand;
717 const exponent = op.getOperand(1) orelse return error.MissingOperand;
718
719 const result_slot = try self.slotFor(result);
720 const base_slot = try self.slotFor(base);
721 const exponent_slot = try self.slotFor(exponent);
722
723 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
724 const base_type_name = base.type.getDialectTypeName() orelse return error.UnsupportedType;
725 const exponent_type_name = exponent.type.getDialectTypeName() orelse return error.UnsupportedType;
726
727 if (!std.mem.eql(u8, type_name, base_type_name)) return error.UnsupportedType;
728 if (!std.mem.eql(u8, type_name, exponent_type_name)) return error.UnsupportedType;
729
730 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
731 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
732 if (!is_f32 and !is_f64) return error.UnsupportedType;
733
734 const callee = libmBinaryCallee(kind, is_f32);
735 const args = [_]ExternArg{
736 .{ .type_name = type_name, .source = .{ .slot = base_slot } },
737 .{ .type_name = type_name, .source = .{ .slot = exponent_slot } },
738 };
739 const call_result: ExternResult = .{ .type_name = type_name, .slot = result_slot };
740 try call_plan.emitExtern(self, callee, &args, &.{call_result});
741 }
742
743 fn emitArithLibmTernary(
744 self: anytype,
745 op: *ir.Operation,
746 kind: LibmTernaryKind,
747 ) !void {
748 const result = op.getResult(0) orelse return error.MissingResult;
749 const a = op.getOperand(0) orelse return error.MissingOperand;
750 const b = op.getOperand(1) orelse return error.MissingOperand;
751 const c = op.getOperand(2) orelse return error.MissingOperand;
752
753 const result_slot = try self.slotFor(result);
754 const a_slot = try self.slotFor(a);
755 const b_slot = try self.slotFor(b);
756 const c_slot = try self.slotFor(c);
757
758 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
759 const a_type_name = a.type.getDialectTypeName() orelse return error.UnsupportedType;
760 const b_type_name = b.type.getDialectTypeName() orelse return error.UnsupportedType;
761 const c_type_name = c.type.getDialectTypeName() orelse return error.UnsupportedType;
762
763 if (!std.mem.eql(u8, type_name, a_type_name)) return error.UnsupportedType;
764 if (!std.mem.eql(u8, type_name, b_type_name)) return error.UnsupportedType;
765 if (!std.mem.eql(u8, type_name, c_type_name)) return error.UnsupportedType;
766
767 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
768 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
769 if (!is_f32 and !is_f64) return error.UnsupportedType;
770
771 const callee = libmTernaryCallee(kind, is_f32);
772 const args = [_]ExternArg{
773 .{ .type_name = type_name, .source = .{ .slot = a_slot } },
774 .{ .type_name = type_name, .source = .{ .slot = b_slot } },
775 .{ .type_name = type_name, .source = .{ .slot = c_slot } },
776 };
777 const call_result: ExternResult = .{ .type_name = type_name, .slot = result_slot };
778 try call_plan.emitExtern(self, callee, &args, &.{call_result});
779 }
780
781 pub fn emitArithFma(self: anytype, op: *ir.Operation) !void {
782 if (comptime cpu_has_fma3) {
783 return emitArithFmaNative(self, op);
784 }
785 return emitArithLibmTernary(self, op, .fma);
786 }
787
788 fn emitArithFmaNative(self: anytype, op: *ir.Operation) !void {
789 const result = op.getResult(0) orelse return error.MissingResult;
790 const a = op.getOperand(0) orelse return error.MissingOperand;
791 const b = op.getOperand(1) orelse return error.MissingOperand;
792 const c = op.getOperand(2) orelse return error.MissingOperand;
793
794 const result_slot = try self.slotFor(result);
795 const a_slot = try self.slotFor(a);
796 const b_slot = try self.slotFor(b);
797 const c_slot = try self.slotFor(c);
798
799 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
800 const a_type_name = a.type.getDialectTypeName() orelse return error.UnsupportedType;
801 const b_type_name = b.type.getDialectTypeName() orelse return error.UnsupportedType;
802 const c_type_name = c.type.getDialectTypeName() orelse return error.UnsupportedType;
803
804 if (!std.mem.eql(u8, type_name, a_type_name)) return error.UnsupportedType;
805 if (!std.mem.eql(u8, type_name, b_type_name)) return error.UnsupportedType;
806 if (!std.mem.eql(u8, type_name, c_type_name)) return error.UnsupportedType;
807
808 const is_f32 = std.mem.eql(u8, type_name, "arith.f32");
809 const is_f64 = std.mem.eql(u8, type_name, "arith.f64");
810 if (!is_f32 and !is_f64) return error.UnsupportedType;
811
812 try self.loadSlotXmm(a_slot, .xmm0);
813 try self.loadSlotXmm(b_slot, .xmm1);
814 try self.loadSlotXmm(c_slot, .xmm2);
815
816 if (is_f32) {
817 try self.emitEncoding(encoding.vfmadd231ss(.xmm2, .xmm0, .xmm1));
818 } else {
819 try self.emitEncoding(encoding.vfmadd231sd(.xmm2, .xmm0, .xmm1));
820 }
821
822 try self.storeSlotXmm(result_slot, .xmm2);
823 }
824
825 pub const BitwiseBinaryKind = enum { band, bor, bxor };
826 pub const ShiftKind = enum { shl, shr, ushr };
827
828 pub fn emitArithBitwiseBinary(
829 self: anytype,
830 op: *ir.Operation,
831 kind: BitwiseBinaryKind,
832 ) !void {
833 const result = op.getResult(0) orelse return error.MissingResult;
834 const lhs = op.getOperand(0) orelse return error.MissingOperand;
835 const rhs = op.getOperand(1) orelse return error.MissingOperand;
836
837 const result_slot = try self.slotFor(result);
838
839 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
840 const lhs_type_name = lhs.type.getDialectTypeName() orelse return error.UnsupportedType;
841 const rhs_type_name = rhs.type.getDialectTypeName() orelse return error.UnsupportedType;
842 if (!std.mem.eql(u8, type_name, lhs_type_name)) return error.UnsupportedType;
843 if (!std.mem.eql(u8, type_name, rhs_type_name)) return error.UnsupportedType;
844
845 if (std.mem.eql(u8, type_name, "arith.f16") or
846 std.mem.eql(u8, type_name, "arith.bf16") or
847 std.mem.eql(u8, type_name, "arith.f32") or
848 std.mem.eql(u8, type_name, "arith.f64")) return error.UnsupportedType;
849
850 const width = result_slot.width;
851 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
852 const op_width: u8 = if (width <= 32) 32 else 64;
853
854 try self.loadInto(lhs, .rax);
855 try self.loadInto(rhs, .rcx);
856 if (op_width == 32) {
857 switch (kind) {
858 .band => try self.emitEncoding(encoding.andRegReg32(.rax, .rcx)),
859 .bor => try self.emitEncoding(encoding.orRegReg32(.rax, .rcx)),
860 .bxor => try self.emitEncoding(encoding.xorRegReg32(.rax, .rcx)),
861 }
862 } else {
863 switch (kind) {
864 .band => try self.emitEncoding(encoding.andRegReg(.rax, .rcx)),
865 .bor => try self.emitEncoding(encoding.orRegReg(.rax, .rcx)),
866 .bxor => try self.emitEncoding(encoding.xorRegReg(.rax, .rcx)),
867 }
868 }
869 try self.storeFrom(result, .rax);
870 }
871
872 pub fn emitArithNot(self: anytype, op: *ir.Operation) !void {
873 const result = op.getResult(0) orelse return error.MissingResult;
874 const operand = op.getOperand(0) orelse return error.MissingOperand;
875
876 const result_slot = try self.slotFor(result);
877
878 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
879 const operand_type_name = operand.type.getDialectTypeName() orelse return error.UnsupportedType;
880 if (!std.mem.eql(u8, type_name, operand_type_name)) return error.UnsupportedType;
881
882 if (std.mem.eql(u8, type_name, "arith.bool")) {
883 try self.loadInto(operand, .rax);
884 try self.emitEncoding(encoding.cmpRegImm(.rax, 0));
885 try self.emitEncoding(encoding.setcc(.rax, .e));
886 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
887 try self.storeFrom(result, .rax);
888 return;
889 }
890
891 if (std.mem.eql(u8, type_name, "arith.f16") or
892 std.mem.eql(u8, type_name, "arith.bf16") or
893 std.mem.eql(u8, type_name, "arith.f32") or
894 std.mem.eql(u8, type_name, "arith.f64")) return error.UnsupportedType;
895
896 const width = result_slot.width;
897 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
898 const op_width: u8 = if (width <= 32) 32 else 64;
899
900 try self.loadInto(operand, .rax);
901 if (op_width == 32) {
902 try self.emitEncoding(encoding.notReg32(.rax));
903 } else {
904 try self.emitEncoding(encoding.notReg(.rax));
905 }
906 try self.storeFrom(result, .rax);
907 }
908
909 pub fn emitArithShift(self: anytype, op: *ir.Operation, kind: ShiftKind) !void {
910 const result = op.getResult(0) orelse return error.MissingResult;
911 const value = op.getOperand(0) orelse return error.MissingOperand;
912 const count = op.getOperand(1) orelse return error.MissingOperand;
913
914 const result_slot = try self.slotFor(result);
915
916 const type_name = result.type.getDialectTypeName() orelse return error.UnsupportedType;
917 const value_type_name = value.type.getDialectTypeName() orelse return error.UnsupportedType;
918 const count_type_name = count.type.getDialectTypeName() orelse return error.UnsupportedType;
919 if (!std.mem.eql(u8, type_name, value_type_name)) return error.UnsupportedType;
920 if (!std.mem.eql(u8, type_name, count_type_name)) return error.UnsupportedType;
921
922 if (std.mem.eql(u8, type_name, "arith.f16") or
923 std.mem.eql(u8, type_name, "arith.bf16") or
924 std.mem.eql(u8, type_name, "arith.f32") or
925 std.mem.eql(u8, type_name, "arith.f64") or
926 std.mem.eql(u8, type_name, "arith.bool")) return error.UnsupportedType;
927
928 if (isUnsignedIntegerTypeName(type_name) and kind == .shr) return error.UnsupportedType;
929
930 const width = result_slot.width;
931 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
932 const op_width: u8 = if (width <= 32) 32 else 64;
933
934 if (kind == .ushr) {
935 try self.loadIntoUnsigned(value, .rax);
936 } else {
937 try self.loadInto(value, .rax);
938 }
939 try self.loadInto(count, .rcx);
940
941 if (op_width == 32) {
942 switch (kind) {
943 .shl => try self.emitEncoding(encoding.shlRegCl32(.rax)),
944 .shr => try self.emitEncoding(encoding.sarRegCl32(.rax)),
945 .ushr => try self.emitEncoding(encoding.shrRegCl32(.rax)),
946 }
947 } else {
948 switch (kind) {
949 .shl => try self.emitEncoding(encoding.shlRegCl(.rax)),
950 .shr => try self.emitEncoding(encoding.sarRegCl(.rax)),
951 .ushr => try self.emitEncoding(encoding.shrRegCl(.rax)),
952 }
953 }
954 try self.storeFrom(result, .rax);
955 }
956
957 pub fn emitArithSelect(self: anytype, op: *ir.Operation) !void {
958 if (op.operands.items.len != 3) return error.UnsupportedOperation;
959 const cond = op.getOperand(0) orelse return error.MissingOperand;
960 const true_val = op.getOperand(1) orelse return error.MissingOperand;
961 const false_val = op.getOperand(2) orelse return error.MissingOperand;
962 const result = op.getResult(0) orelse return error.MissingResult;
963
964 const cond_slot = try self.slotFor(cond);
965 const true_slot = try self.slotFor(true_val);
966 const false_slot = try self.slotFor(false_val);
967 const result_slot = try self.slotFor(result);
968
969 const cond_name = cond.type.getDialectTypeName() orelse return error.UnsupportedType;
970 if (!std.mem.eql(u8, cond_name, "arith.bool")) return error.UnsupportedType;
971
972 const result_kind = cast.Kind.fromTypeName(result.type.getDialectTypeName() orelse return error.UnsupportedType) orelse return error.UnsupportedType;
973 const true_kind = cast.Kind.fromTypeName(true_val.type.getDialectTypeName() orelse return error.UnsupportedType) orelse return error.UnsupportedType;
974 const false_kind = cast.Kind.fromTypeName(false_val.type.getDialectTypeName() orelse return error.UnsupportedType) orelse return error.UnsupportedType;
975 if (result_kind != true_kind or result_kind != false_kind) return error.UnsupportedType;
976 if (!result.type.eql(true_val.type)) return error.UnsupportedType;
977 if (!result.type.eql(false_val.type)) return error.UnsupportedType;
978
979 const is_f32 = result_kind == .f32_;
980 const is_f64 = result_kind == .f64_;
981
982 if (is_f32 or is_f64) {
983 try self.loadInto(cond, .rax);
984 try self.loadSlotXmm(true_slot, .xmm0);
985 try self.loadSlotXmm(false_slot, .xmm1);
986 try self.emitEncoding(encoding.cmpRegImm(.rax, 0));
987
988 var done_label = Label{};
989 defer done_label.deinit(self.allocator);
990 try self.emitJccLabel(.ne, &done_label);
991
992 if (is_f32) {
993 try self.emitEncoding(encoding.movss(.xmm0, .{ .reg = .xmm1 }));
994 } else {
995 try self.emitEncoding(encoding.movsd(.xmm0, .{ .reg = .xmm1 }));
996 }
997
998 try self.bindLabel(&done_label);
999 try self.storeSlotXmm(result_slot, .xmm0);
1000 return;
1001 }
1002
1003 if (cond_slot.width == 64 and true_slot.width == 64 and false_slot.width == 64 and result_slot.width == 64) {
1004 try self.loadInto(cond, .rax);
1005 try self.loadInto(true_val, .rcx);
1006 try self.loadInto(false_val, .rdx);
1007 } else {
1008 try self.loadIntoUnsigned(cond, .rax);
1009 try self.loadIntoUnsigned(true_val, .rcx);
1010 try self.loadIntoUnsigned(false_val, .rdx);
1011 }
1012 try self.emitEncoding(encoding.cmpRegImm(.rax, 0));
1013 try self.emitEncoding(encoding.cmovneRegReg(.rdx, .rcx));
1014 try self.storeFrom(result, .rdx);
1015 }
1016
1017 pub fn emitArithCmp(self: anytype, op: *ir.Operation) !void {
1018 const wrapped = ArithDialect.CmpOp{ .op = op };
1019 const result = wrapped.getResult();
1020 const pred = wrapped.getPredicate() orelse return error.MissingPredicate;
1021
1022 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1023 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1024
1025 const result_slot = try self.slotFor(result);
1026 const lhs_slot = try self.slotFor(lhs);
1027 const type_name = lhs.type.getDialectTypeName() orelse return error.UnsupportedType;
1028 const is_float = std.mem.eql(u8, type_name, "arith.f32") or std.mem.eql(u8, type_name, "arith.f64");
1029
1030 if (is_float) {
1031 try self.loadIntoXmm(lhs, .xmm0);
1032 try self.loadIntoXmm(rhs, .xmm1);
1033 if (lhs_slot.width == 32) {
1034 try self.emitEncoding(encoding.ucomiss(.xmm0, .{ .reg = .xmm1 }));
1035 } else if (lhs_slot.width == 64) {
1036 try self.emitEncoding(encoding.ucomisd(.xmm0, .{ .reg = .xmm1 }));
1037 } else {
1038 return error.UnsupportedType;
1039 }
1040
1041 switch (pred) {
1042 .eq => {
1043 try self.emitEncoding(encoding.setcc(.rax, .e));
1044 try self.emitEncoding(encoding.setcc(.rcx, .np));
1045 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1046 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1047 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
1048 try self.storeSlot(result_slot, .rax);
1049 },
1050 .ne => {
1051 try self.emitEncoding(encoding.setcc(.rax, .ne));
1052 try self.emitEncoding(encoding.setcc(.rcx, .p));
1053 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1054 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1055 try self.emitEncoding(encoding.orRegReg(.rax, .rcx));
1056 try self.storeSlot(result_slot, .rax);
1057 },
1058 .lt, .slt, .ult => {
1059 try self.emitEncoding(encoding.setcc(.rax, .b));
1060 try self.emitEncoding(encoding.setcc(.rcx, .np));
1061 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1062 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1063 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
1064 try self.storeSlot(result_slot, .rax);
1065 },
1066 .le, .sle, .ule => {
1067 try self.emitEncoding(encoding.setcc(.rax, .be));
1068 try self.emitEncoding(encoding.setcc(.rcx, .np));
1069 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1070 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1071 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
1072 try self.storeSlot(result_slot, .rax);
1073 },
1074 .gt, .sgt, .ugt => {
1075 try self.emitEncoding(encoding.setcc(.rax, .a));
1076 try self.emitEncoding(encoding.setcc(.rcx, .np));
1077 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1078 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1079 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
1080 try self.storeSlot(result_slot, .rax);
1081 },
1082 .ge, .sge, .uge => {
1083 try self.emitEncoding(encoding.setcc(.rax, .ae));
1084 try self.emitEncoding(encoding.setcc(.rcx, .np));
1085 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1086 try self.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1087 try self.emitEncoding(encoding.andRegReg(.rax, .rcx));
1088 try self.storeSlot(result_slot, .rax);
1089 },
1090 }
1091 return;
1092 }
1093
1094 const width = lhs_slot.width;
1095 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1096
1097 const is_unsigned = switch (pred) {
1098 .ult, .ule, .ugt, .uge => true,
1099 else => false,
1100 };
1101
1102 if (constantIntI32(rhs)) |imm| {
1103 if (is_unsigned) {
1104 try self.loadIntoUnsigned(lhs, .rax);
1105 } else {
1106 try self.loadInto(lhs, .rax);
1107 }
1108 if (width == 64) {
1109 try self.emitEncoding(encoding.cmpRegImm(.rax, imm));
1110 } else {
1111 try self.emitEncoding(encoding.cmpRegImm32(.rax, imm));
1112 }
1113
1114 const cond = conditionForIntPredicate(pred);
1115 try self.emitEncoding(encoding.setcc(.rax, cond));
1116 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1117 try self.storeFrom(result, .rax);
1118 return;
1119 }
1120
1121 if (is_unsigned) {
1122 try self.loadIntoUnsigned(lhs, .rax);
1123 try self.loadIntoUnsigned(rhs, .rcx);
1124 } else {
1125 try self.loadInto(lhs, .rax);
1126 try self.loadInto(rhs, .rcx);
1127 }
1128
1129 if (width == 64) {
1130 try self.emitEncoding(encoding.cmpRegReg(.rax, .rcx));
1131 } else {
1132 try self.emitEncoding(encoding.cmpRegReg32(.rax, .rcx));
1133 }
1134
1135 const cond = conditionForIntPredicate(pred);
1136 try self.emitEncoding(encoding.setcc(.rax, cond));
1137 try self.emitEncoding(encoding.movzxReg8(.rax, .rax));
1138 try self.storeFrom(result, .rax);
1139 }
1140
1141 /// Sets the flags for one integer comparison, and DOES NOT DECIDE WHAT ITS
1142 /// OPERANDS LOOK LIKE.
1143 ///
1144 /// It used to. It read the extension off the PREDICATE, taking `.ult`, `.ule`,
1145 /// `.ugt` and `.uge` as unsigned and everything else as signed, so a `u8`
1146 /// equality took the signed reload while `constantIntI32` handed back the face
1147 /// value, and `cmp $0xc8, %eax` met an `%eax` holding `0xFFFFFFC8`. That is
1148 /// the comparison where the operand's form came from the instruction reading
1149 /// it rather than from the value, so the two sides disagreed about what a byte
1150 /// means.
1151 ///
1152 /// Both operands now come back in the form their own slots carry, which is
1153 /// the form the immediate is already in, so the three agree.
1154 ///
1155 /// WHERE THE PREDICATE AND THE OPERAND TYPE DISAGREE, THE OPERAND WINS FOR
1156 /// RELOADS. A signed predicate on unsigned operands exercises that boundary. At
1157 /// widths 8 and 16 it costs nothing: a zero extended `u8` or
1158 /// `u16` cannot reach the sign bit of the 32 bit comparison it is widened
1159 /// into, so the signed comparison gives the unsigned answer anyway. At 32 and
1160 /// 64 bits no extension happens at all and the predicate is the whole story.
1161 /// Choosing the predicate for an unsigned comparison belongs to Yikes, not to
1162 /// this backend.
1163 pub fn emitArithCmpFlags(self: anytype, op: *ir.Operation) !bool {
1164 const wrapped = ArithDialect.CmpOp{ .op = op };
1165 if (wrapped.getPredicate() == null) return error.MissingPredicate;
1166
1167 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1168 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1169 const lhs_slot = try self.slotFor(lhs);
1170 const type_name = lhs.type.getDialectTypeName() orelse return error.UnsupportedType;
1171 const is_float = std.mem.eql(u8, type_name, "arith.f32") or std.mem.eql(u8, type_name, "arith.f64");
1172 if (is_float) return false;
1173
1174 const width = lhs_slot.width;
1175 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1176
1177 if (constantIntI32(rhs)) |imm| {
1178 try self.loadInto(lhs, .rax);
1179 if (width == 64) {
1180 try self.emitEncoding(encoding.cmpRegImm(.rax, imm));
1181 } else {
1182 try self.emitEncoding(encoding.cmpRegImm32(.rax, imm));
1183 }
1184 return true;
1185 }
1186
1187 try self.loadInto(lhs, .rax);
1188 try self.loadInto(rhs, .rcx);
1189
1190 if (width == 64) {
1191 try self.emitEncoding(encoding.cmpRegReg(.rax, .rcx));
1192 } else {
1193 try self.emitEncoding(encoding.cmpRegReg32(.rax, .rcx));
1194 }
1195 return true;
1196 }
1197
1198 pub fn conditionForIntPredicate(pred: CmpPredicate) Condition {
1199 return switch (pred) {
1200 .eq => .e,
1201 .ne => .ne,
1202 .lt, .slt => .l,
1203 .le, .sle => .le,
1204 .gt, .sgt => .g,
1205 .ge, .sge => .ge,
1206 .ult => .b,
1207 .ule => .be,
1208 .ugt => .a,
1209 .uge => .ae,
1210 };
1211 }
1212
1213 fn immediateUseSupported(op: *ir.Operation, operand_number: u32) bool {
1214 const name = op.name.name;
1215 if (operand_number != 1) return false;
1216 if (std.mem.eql(u8, name, ArithDialect.SubOp.operation_name)) return true;
1217 if (std.mem.eql(u8, name, ArithDialect.CmpOp.operation_name)) return true;
1218 return false;
1219 }
1220
1221 pub fn constantIntI32(value: *ir.Value) ?i32 {
1222 const op = definingOp(value) orelse return null;
1223 if (!std.mem.eql(u8, op.name.name, ArithDialect.ConstantOp.operation_name)) return null;
1224 if (data.hasAttributes(op)) return null;
1225 const constant = ArithDialect.ConstantOp{ .op = op };
1226 const int_value = constant.getIntValue() orelse return null;
1227 return std.math.cast(i32, int_value);
1228 }
1229
1230 fn definingOp(value: *ir.Value) ?*ir.Operation {
1231 return switch (value.kind) {
1232 .op_result => |info| @ptrCast(@alignCast(info.owner)),
1233 .block_argument => null,
1234 };
1235 }
1236
1237 const BinaryRecorder = struct {
1238 lhs: *ir.Value,
1239 rhs: *ir.Value,
1240 result: *ir.Value,
1241 lhs_slot: Slot,
1242 rhs_slot: Slot,
1243 result_slot: Slot,
1244 gpr_loads: usize = 0,
1245 xmm_loads: usize = 0,
1246 gpr_stores: usize = 0,
1247 xmm_stores: usize = 0,
1248 encodings: usize = 0,
1249
1250 pub fn slotFor(self: *BinaryRecorder, value: *ir.Value) !Slot {
1251 if (value == self.lhs) return self.lhs_slot;
1252 if (value == self.rhs) return self.rhs_slot;
1253 if (value == self.result) return self.result_slot;
1254 return error.MissingSlot;
1255 }
1256
1257 pub fn loadSlot(self: *BinaryRecorder, slot: Slot, reg: registers.GPR) !void {
1258 _ = slot;
1259 _ = reg;
1260 self.gpr_loads += 1;
1261 }
1262
1263 pub fn loadSlotUnsigned(self: *BinaryRecorder, slot: Slot, reg: registers.GPR) !void {
1264 try self.loadSlot(slot, reg);
1265 }
1266
1267 pub fn loadSlotXmm(self: *BinaryRecorder, slot: Slot, reg: registers.XMM) !void {
1268 _ = slot;
1269 _ = reg;
1270 self.xmm_loads += 1;
1271 }
1272
1273 pub fn storeSlot(self: *BinaryRecorder, slot: Slot, reg: registers.GPR) !void {
1274 _ = slot;
1275 _ = reg;
1276 self.gpr_stores += 1;
1277 }
1278
1279 pub fn storeSlotXmm(self: *BinaryRecorder, slot: Slot, reg: registers.XMM) !void {
1280 _ = slot;
1281 _ = reg;
1282 self.xmm_stores += 1;
1283 }
1284
1285 pub fn loadInto(self: *BinaryRecorder, value: *ir.Value, reg: registers.GPR) !void {
1286 try self.loadSlot(try self.slotFor(value), reg);
1287 }
1288
1289 pub fn loadIntoXmm(self: *BinaryRecorder, value: *ir.Value, reg: registers.XMM) !void {
1290 try self.loadSlotXmm(try self.slotFor(value), reg);
1291 }
1292
1293 pub fn storeFromXmm(self: *BinaryRecorder, value: *ir.Value, reg: registers.XMM) !void {
1294 try self.storeSlotXmm(try self.slotFor(value), reg);
1295 }
1296
1297 pub fn xmmHomeForPhase(self: *BinaryRecorder, value: *ir.Value, phase: anytype) ?registers.XMM {
1298 _ = self;
1299 _ = value;
1300 _ = phase;
1301 return null;
1302 }
1303
1304 pub fn loadIntoUnsigned(self: *BinaryRecorder, value: *ir.Value, reg: registers.GPR) !void {
1305 try self.loadSlotUnsigned(try self.slotFor(value), reg);
1306 }
1307
1308 pub fn storeFrom(self: *BinaryRecorder, value: *ir.Value, reg: registers.GPR) !void {
1309 try self.storeSlot(try self.slotFor(value), reg);
1310 }
1311
1312 pub fn emitEncoding(self: *BinaryRecorder, enc: encoding.Encoding) !void {
1313 try std.testing.expect(enc.len > 0);
1314 self.encodings += 1;
1315 }
1316 };
1317
1318 test "x86_64 scalar owner emits floating add" {
1319 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
1320 defer ctx.deinit(std.testing.allocator);
1321 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1322
1323 const loc = ir.Location.getUnknown();
1324 const f32_type = try ArithDialect.getScalarType(&ctx, .f32);
1325 const c1 = try ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 1.5);
1326 const c2 = try ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 2.5);
1327 const add = try ArithDialect.AddOp.create(&ctx, loc, c1.getResult(), c2.getResult());
1328
1329 var recorder = BinaryRecorder{
1330 .lhs = c1.getResult(),
1331 .rhs = c2.getResult(),
1332 .result = add.getResult(),
1333 .lhs_slot = .{ .offset = -8, .width = 32, .ext = .unsigned },
1334 .rhs_slot = .{ .offset = -16, .width = 32, .ext = .unsigned },
1335 .result_slot = .{ .offset = -24, .width = 32, .ext = .unsigned },
1336 };
1337
1338 try emitBinaryIntOp(&recorder, add.op, .add);
1339
1340 try std.testing.expectEqual(@as(usize, 2), recorder.xmm_loads);
1341 try std.testing.expectEqual(@as(usize, 1), recorder.xmm_stores);
1342 try std.testing.expectEqual(@as(usize, 1), recorder.encodings);
1343 }
1344
1345 test "x86_64 scalar owner emits integer sub immediate" {
1346 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
1347 defer ctx.deinit(std.testing.allocator);
1348 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1349
1350 const loc = ir.Location.getUnknown();
1351 const i64_type = try ArithDialect.getScalarType(&ctx, .i64);
1352 const lhs = try ArithDialect.ConstantOp.createInt(&ctx, loc, i64_type, 9);
1353 const rhs = try ArithDialect.ConstantOp.createInt(&ctx, loc, i64_type, 1);
1354 const sub = try ArithDialect.SubOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
1355
1356 try std.testing.expect(shouldSkipArithConstant(rhs.op));
1357
1358 var recorder = BinaryRecorder{
1359 .lhs = lhs.getResult(),
1360 .rhs = rhs.getResult(),
1361 .result = sub.getResult(),
1362 .lhs_slot = .{ .offset = -8, .width = 64, .ext = .signed },
1363 .rhs_slot = .{ .offset = -16, .width = 64, .ext = .signed },
1364 .result_slot = .{ .offset = -24, .width = 64, .ext = .signed },
1365 };
1366
1367 try emitBinaryIntOp(&recorder, sub.op, .sub);
1368
1369 try std.testing.expectEqual(@as(usize, 1), recorder.gpr_loads);
1370 try std.testing.expectEqual(@as(usize, 1), recorder.gpr_stores);
1371 try std.testing.expectEqual(@as(usize, 1), recorder.encodings);
1372 }
1373
1374 test "x86_64 scalar owner emits integer cmp immediate" {
1375 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
1376 defer ctx.deinit(std.testing.allocator);
1377 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1378
1379 const loc = ir.Location.getUnknown();
1380 const i64_type = try ArithDialect.getScalarType(&ctx, .i64);
1381 const lhs = try ArithDialect.ConstantOp.createInt(&ctx, loc, i64_type, 9);
1382 const rhs = try ArithDialect.ConstantOp.createInt(&ctx, loc, i64_type, 0);
1383 const cmp = try ArithDialect.CmpOp.create(&ctx, loc, .eq, lhs.getResult(), rhs.getResult());
1384
1385 try std.testing.expect(shouldSkipArithConstant(rhs.op));
1386
1387 var recorder = BinaryRecorder{
1388 .lhs = lhs.getResult(),
1389 .rhs = rhs.getResult(),
1390 .result = cmp.getResult(),
1391 .lhs_slot = .{ .offset = -8, .width = 64, .ext = .signed },
1392 .rhs_slot = .{ .offset = -16, .width = 64, .ext = .signed },
1393 .result_slot = .{ .offset = -24, .width = 64, .ext = .signed },
1394 };
1395
1396 try emitArithCmp(&recorder, cmp.op);
1397
1398 try std.testing.expectEqual(@as(usize, 1), recorder.gpr_loads);
1399 try std.testing.expectEqual(@as(usize, 1), recorder.gpr_stores);
1400 try std.testing.expectEqual(@as(usize, 3), recorder.encodings);
1401 }
1402
1403 test "x86_64 scalar owner maps integer predicates" {
1404 try std.testing.expectEqual(Condition.l, conditionForIntPredicate(.slt));
1405 try std.testing.expectEqual(Condition.ae, conditionForIntPredicate(.uge));
1406 }