lib/choir/src/backends/x64/vector.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const ir = @import("../../core/root.zig");
3 const dialects = @import("../../dialects/root.zig");
4 const data = @import("data.zig");
5 const encoding = @import("encoding.zig");
6 const labels = @import("labels.zig");
7 const registers = @import("registers/root.zig");
8 const scalar_backend = @import("scalar.zig");
9 const slot_layout = @import("slots.zig");
10
11 const ArithDialect = dialects.arith.ArithDialect;
12 const CmpPredicate = dialects.arith.CmpPredicate;
13
14 pub const BinOp = enum {
15 add,
16 sub,
17 mul,
18 div,
19 };
20
21 pub const BitwiseBinaryOp = enum {
22 band,
23 bor,
24 bxor,
25 };
26
27 pub const PackedFloat = enum {
28 f32x4,
29 f64x2,
30 };
31
32 pub const PackedInt = enum {
33 dwordx4,
34 qwordx2,
35 };
36
37 pub const PackedKind = union(enum) {
38 float: PackedFloat,
39 int: PackedInt,
40 };
41
42 pub fn packedFloatKind(slot: slot_layout.VectorSlot) ?PackedFloat {
43 if (!slot.is_float) return null;
44 if (slot.lanes == 4 and
45 slot.base.width == 32 and
46 std.mem.eql(u8, slot.elem_type_name, dialects.arith.type_names.float32))
47 {
48 return .f32x4;
49 }
50 if (slot.lanes == 2 and
51 slot.base.width == 64 and
52 std.mem.eql(u8, slot.elem_type_name, dialects.arith.type_names.float64))
53 {
54 return .f64x2;
55 }
56 return null;
57 }
58
59 pub fn packedIntKind(slot: slot_layout.VectorSlot) ?PackedInt {
60 if (slot.is_float) return null;
61 if (slot.lanes == 4 and
62 slot.base.width == 32 and
63 (std.mem.eql(u8, slot.elem_type_name, dialects.arith.type_names.int32) or
64 std.mem.eql(u8, slot.elem_type_name, dialects.arith.type_names.uint32)))
65 {
66 return .dwordx4;
67 }
68 if (slot.lanes == 2 and
69 slot.base.width == 64 and
70 (std.mem.eql(u8, slot.elem_type_name, dialects.arith.type_names.int64) or
71 std.mem.eql(u8, slot.elem_type_name, dialects.arith.type_names.uint64)))
72 {
73 return .qwordx2;
74 }
75 return null;
76 }
77
78 pub fn packedKind(slot: slot_layout.VectorSlot) ?PackedKind {
79 if (packedFloatKind(slot)) |kind| return .{ .float = kind };
80 if (packedIntKind(slot)) |kind| return .{ .int = kind };
81 return null;
82 }
83
84 fn samePackedFloatKind(lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot) ?PackedFloat {
85 const lhs_kind = packedFloatKind(lhs) orelse return null;
86 const rhs_kind = packedFloatKind(rhs) orelse return null;
87 const result_kind = packedFloatKind(result) orelse return null;
88 if (lhs_kind != rhs_kind or lhs_kind != result_kind) return null;
89 return lhs_kind;
90 }
91
92 fn samePackedIntKind(lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot) ?PackedInt {
93 if (!std.mem.eql(u8, lhs.elem_type_name, rhs.elem_type_name) or
94 !std.mem.eql(u8, lhs.elem_type_name, result.elem_type_name)) return null;
95 const lhs_kind = packedIntKind(lhs) orelse return null;
96 const rhs_kind = packedIntKind(rhs) orelse return null;
97 const result_kind = packedIntKind(result) orelse return null;
98 if (lhs_kind != rhs_kind or lhs_kind != result_kind) return null;
99 return lhs_kind;
100 }
101
102 fn samePackedResultShape(input: slot_layout.VectorSlot, result: slot_layout.VectorSlot) bool {
103 return input.lanes == result.lanes and
104 input.base.width == result.base.width and
105 packedKind(result) != null;
106 }
107
108 fn sameVectorStorageShape(lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot) bool {
109 return lhs.lanes == rhs.lanes and
110 lhs.base.width == rhs.base.width and
111 lhs.is_float == rhs.is_float and
112 std.mem.eql(u8, lhs.elem_type_name, rhs.elem_type_name);
113 }
114
115 fn samePackedFloatCmpKind(lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot) ?PackedFloat {
116 const lhs_kind = packedFloatKind(lhs) orelse return null;
117 const rhs_kind = packedFloatKind(rhs) orelse return null;
118 if (lhs_kind != rhs_kind) return null;
119 if (!samePackedResultShape(lhs, result)) return null;
120 return lhs_kind;
121 }
122
123 fn samePackedIntCmpKind(lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot) ?PackedInt {
124 if (!std.mem.eql(u8, lhs.elem_type_name, rhs.elem_type_name)) return null;
125 const lhs_kind = packedIntKind(lhs) orelse return null;
126 const rhs_kind = packedIntKind(rhs) orelse return null;
127 if (lhs_kind != rhs_kind) return null;
128 if (!samePackedResultShape(lhs, result)) return null;
129 return lhs_kind;
130 }
131
132 fn packedMem(slot: slot_layout.VectorSlot) encoding.Mem {
133 const low_lane = slot_layout.laneSlot(slot.base, slot.lanes - 1);
134 return encoding.Mem.baseDisp(.rbp, low_lane.offset);
135 }
136
137 pub fn emitPackedLoadFromMem(emitter: anytype, kind: PackedKind, mem: encoding.Mem, reg: registers.XMM) !void {
138 switch (kind) {
139 .float => |float_kind| switch (float_kind) {
140 .f32x4 => try emitter.emitEncoding(encoding.movups(reg, .{ .mem = mem })),
141 .f64x2 => try emitter.emitEncoding(encoding.movupd(reg, .{ .mem = mem })),
142 },
143 .int => |int_kind| switch (int_kind) {
144 .dwordx4 => try emitter.emitEncoding(encoding.movups(reg, .{ .mem = mem })),
145 .qwordx2 => try emitter.emitEncoding(encoding.movupd(reg, .{ .mem = mem })),
146 },
147 }
148 }
149
150 pub fn emitPackedStoreToMem(emitter: anytype, kind: PackedKind, mem: encoding.Mem, reg: registers.XMM) !void {
151 switch (kind) {
152 .float => |float_kind| switch (float_kind) {
153 .f32x4 => try emitter.emitEncoding(encoding.movupsStore(mem, reg)),
154 .f64x2 => try emitter.emitEncoding(encoding.movupdStore(mem, reg)),
155 },
156 .int => |int_kind| switch (int_kind) {
157 .dwordx4 => try emitter.emitEncoding(encoding.movupsStore(mem, reg)),
158 .qwordx2 => try emitter.emitEncoding(encoding.movupdStore(mem, reg)),
159 },
160 }
161 }
162
163 pub fn emitPackedLoad(emitter: anytype, slot: slot_layout.VectorSlot, reg: registers.XMM) !void {
164 const kind = packedKind(slot) orelse return error.UnsupportedType;
165 try emitPackedLoadFromMem(emitter, kind, packedMem(slot), reg);
166 }
167
168 pub fn emitPackedStore(emitter: anytype, slot: slot_layout.VectorSlot, reg: registers.XMM) !void {
169 const kind = packedKind(slot) orelse return error.UnsupportedType;
170 try emitPackedStoreToMem(emitter, kind, packedMem(slot), reg);
171 }
172
173 fn emitPackedFloatBinary(emitter: anytype, lhs: *ir.Value, rhs: *ir.Value, result: *ir.Value, op_kind: BinOp, float_kind: PackedFloat) !void {
174 try emitter.loadIntoXmmPacked(lhs, .xmm0);
175 try emitter.loadIntoXmmPacked(rhs, .xmm1);
176 switch (float_kind) {
177 .f32x4 => switch (op_kind) {
178 .add => try emitter.emitEncoding(encoding.addps(.xmm0, .{ .reg = .xmm1 })),
179 .sub => try emitter.emitEncoding(encoding.subps(.xmm0, .{ .reg = .xmm1 })),
180 .mul => try emitter.emitEncoding(encoding.mulps(.xmm0, .{ .reg = .xmm1 })),
181 .div => try emitter.emitEncoding(encoding.divps(.xmm0, .{ .reg = .xmm1 })),
182 },
183 .f64x2 => switch (op_kind) {
184 .add => try emitter.emitEncoding(encoding.addpd(.xmm0, .{ .reg = .xmm1 })),
185 .sub => try emitter.emitEncoding(encoding.subpd(.xmm0, .{ .reg = .xmm1 })),
186 .mul => try emitter.emitEncoding(encoding.mulpd(.xmm0, .{ .reg = .xmm1 })),
187 .div => try emitter.emitEncoding(encoding.divpd(.xmm0, .{ .reg = .xmm1 })),
188 },
189 }
190 try emitter.storeFromXmmPacked(result, .xmm0);
191 }
192
193 fn emitPackedDwordMul(emitter: anytype) !void {
194 try emitter.emitEncoding(encoding.movups(.xmm2, .{ .reg = .xmm0 }));
195 try emitter.emitEncoding(encoding.pmuludq(.xmm0, .{ .reg = .xmm1 }));
196 try emitter.emitEncoding(encoding.psrlqImm(.xmm2, 32));
197 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 32));
198 try emitter.emitEncoding(encoding.pmuludq(.xmm2, .{ .reg = .xmm1 }));
199 try emitter.emitEncoding(encoding.pcmpeqd(.xmm3, .{ .reg = .xmm3 }));
200 try emitter.emitEncoding(encoding.psrlqImm(.xmm3, 32));
201 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm3 }));
202 try emitter.emitEncoding(encoding.pand(.xmm2, .{ .reg = .xmm3 }));
203 try emitter.emitEncoding(encoding.psllqImm(.xmm2, 32));
204 try emitter.emitEncoding(encoding.por(.xmm0, .{ .reg = .xmm2 }));
205 }
206
207 fn emitPackedQwordMul(emitter: anytype) !void {
208 try emitter.emitEncoding(encoding.movupd(.xmm2, .{ .reg = .xmm0 }));
209 try emitter.emitEncoding(encoding.psrlqImm(.xmm2, 32));
210 try emitter.emitEncoding(encoding.movupd(.xmm3, .{ .reg = .xmm1 }));
211 try emitter.emitEncoding(encoding.psrlqImm(.xmm3, 32));
212 try emitter.emitEncoding(encoding.movupd(.xmm4, .{ .reg = .xmm0 }));
213 try emitter.emitEncoding(encoding.pmuludq(.xmm2, .{ .reg = .xmm1 }));
214 try emitter.emitEncoding(encoding.pmuludq(.xmm4, .{ .reg = .xmm3 }));
215 try emitter.emitEncoding(encoding.paddq(.xmm2, .{ .reg = .xmm4 }));
216 try emitter.emitEncoding(encoding.psllqImm(.xmm2, 32));
217 try emitter.emitEncoding(encoding.pmuludq(.xmm0, .{ .reg = .xmm1 }));
218 try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm2 }));
219 }
220
221 fn emitPackedIntBinary(emitter: anytype, lhs: *ir.Value, rhs: *ir.Value, result: *ir.Value, op_kind: BinOp, int_kind: PackedInt) !void {
222 try emitter.loadIntoXmmPacked(lhs, .xmm0);
223 try emitter.loadIntoXmmPacked(rhs, .xmm1);
224 switch (int_kind) {
225 .dwordx4 => switch (op_kind) {
226 .add => try emitter.emitEncoding(encoding.paddd(.xmm0, .{ .reg = .xmm1 })),
227 .sub => try emitter.emitEncoding(encoding.psubd(.xmm0, .{ .reg = .xmm1 })),
228 .mul => try emitPackedDwordMul(emitter),
229 .div => return error.UnsupportedOperation,
230 },
231 .qwordx2 => switch (op_kind) {
232 .add => try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm1 })),
233 .sub => try emitter.emitEncoding(encoding.psubq(.xmm0, .{ .reg = .xmm1 })),
234 .mul => try emitPackedQwordMul(emitter),
235 .div => return error.UnsupportedOperation,
236 },
237 }
238 try emitter.storeFromXmmPacked(result, .xmm0);
239 }
240
241 fn emitPackedIntBitwiseBinary(emitter: anytype, lhs: *ir.Value, rhs: *ir.Value, result: *ir.Value, op_kind: BitwiseBinaryOp) !void {
242 try emitter.loadIntoXmmPacked(lhs, .xmm0);
243 try emitter.loadIntoXmmPacked(rhs, .xmm1);
244 switch (op_kind) {
245 .band => try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm1 })),
246 .bor => try emitter.emitEncoding(encoding.por(.xmm0, .{ .reg = .xmm1 })),
247 .bxor => try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm1 })),
248 }
249 try emitter.storeFromXmmPacked(result, .xmm0);
250 }
251
252 fn emitPackedDwordConstantReg(emitter: anytype, reg: registers.XMM, value: u32) !void {
253 try emitter.emitEncoding(encoding.movRegImm32(.rax, value));
254 try emitter.emitEncoding(encoding.movdXmmFromReg32(reg, .rax));
255 try emitter.emitEncoding(encoding.pshufd(reg, .{ .reg = reg }, 0));
256 }
257
258 fn emitPackedQwordConstantReg(emitter: anytype, reg: registers.XMM, value: u64) !void {
259 try emitter.emitEncoding(encoding.movRegImm64(.rax, value));
260 try emitter.emitEncoding(encoding.movqXmmFromReg64(reg, .rax));
261 try emitter.emitEncoding(encoding.shufpd(reg, .{ .reg = reg }, 0));
262 }
263
264 fn emitPackedDwordPopCount(emitter: anytype) !void {
265 try emitter.emitEncoding(encoding.movups(.xmm1, .{ .reg = .xmm0 }));
266 try emitter.emitEncoding(encoding.psrldImm(.xmm1, 1));
267 try emitPackedDwordConstantReg(emitter, .xmm2, 0x5555_5555);
268 try emitter.emitEncoding(encoding.pand(.xmm1, .{ .reg = .xmm2 }));
269 try emitter.emitEncoding(encoding.psubd(.xmm0, .{ .reg = .xmm1 }));
270
271 try emitPackedDwordConstantReg(emitter, .xmm2, 0x3333_3333);
272 try emitter.emitEncoding(encoding.movups(.xmm1, .{ .reg = .xmm0 }));
273 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm2 }));
274 try emitter.emitEncoding(encoding.psrldImm(.xmm1, 2));
275 try emitter.emitEncoding(encoding.pand(.xmm1, .{ .reg = .xmm2 }));
276 try emitter.emitEncoding(encoding.paddd(.xmm0, .{ .reg = .xmm1 }));
277
278 try emitter.emitEncoding(encoding.movups(.xmm1, .{ .reg = .xmm0 }));
279 try emitter.emitEncoding(encoding.psrldImm(.xmm1, 4));
280 try emitter.emitEncoding(encoding.paddd(.xmm0, .{ .reg = .xmm1 }));
281 try emitPackedDwordConstantReg(emitter, .xmm2, 0x0f0f_0f0f);
282 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm2 }));
283
284 try emitter.emitEncoding(encoding.movups(.xmm1, .{ .reg = .xmm0 }));
285 try emitter.emitEncoding(encoding.psrldImm(.xmm1, 8));
286 try emitter.emitEncoding(encoding.paddd(.xmm0, .{ .reg = .xmm1 }));
287 try emitter.emitEncoding(encoding.movups(.xmm1, .{ .reg = .xmm0 }));
288 try emitter.emitEncoding(encoding.psrldImm(.xmm1, 16));
289 try emitter.emitEncoding(encoding.paddd(.xmm0, .{ .reg = .xmm1 }));
290 try emitPackedDwordConstantReg(emitter, .xmm2, 0x0000_003f);
291 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm2 }));
292 }
293
294 fn emitPackedQwordPopCount(emitter: anytype) !void {
295 try emitter.emitEncoding(encoding.movupd(.xmm1, .{ .reg = .xmm0 }));
296 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 1));
297 try emitPackedQwordConstantReg(emitter, .xmm2, 0x5555_5555_5555_5555);
298 try emitter.emitEncoding(encoding.pand(.xmm1, .{ .reg = .xmm2 }));
299 try emitter.emitEncoding(encoding.psubq(.xmm0, .{ .reg = .xmm1 }));
300
301 try emitPackedQwordConstantReg(emitter, .xmm2, 0x3333_3333_3333_3333);
302 try emitter.emitEncoding(encoding.movupd(.xmm1, .{ .reg = .xmm0 }));
303 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm2 }));
304 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 2));
305 try emitter.emitEncoding(encoding.pand(.xmm1, .{ .reg = .xmm2 }));
306 try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm1 }));
307
308 try emitter.emitEncoding(encoding.movupd(.xmm1, .{ .reg = .xmm0 }));
309 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 4));
310 try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm1 }));
311 try emitPackedQwordConstantReg(emitter, .xmm2, 0x0f0f_0f0f_0f0f_0f0f);
312 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm2 }));
313
314 try emitter.emitEncoding(encoding.movupd(.xmm1, .{ .reg = .xmm0 }));
315 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 8));
316 try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm1 }));
317 try emitter.emitEncoding(encoding.movupd(.xmm1, .{ .reg = .xmm0 }));
318 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 16));
319 try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm1 }));
320 try emitter.emitEncoding(encoding.movupd(.xmm1, .{ .reg = .xmm0 }));
321 try emitter.emitEncoding(encoding.psrlqImm(.xmm1, 32));
322 try emitter.emitEncoding(encoding.paddq(.xmm0, .{ .reg = .xmm1 }));
323 try emitPackedQwordConstantReg(emitter, .xmm2, 0x0000_0000_0000_007f);
324 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm2 }));
325 }
326
327 fn emitPackedIntPopCount(emitter: anytype, input: slot_layout.VectorSlot, result: slot_layout.VectorSlot, int_kind: PackedInt) !void {
328 try emitPackedLoad(emitter, input, .xmm0);
329 switch (int_kind) {
330 .dwordx4 => try emitPackedDwordPopCount(emitter),
331 .qwordx2 => try emitPackedQwordPopCount(emitter),
332 }
333 try emitPackedStore(emitter, result, .xmm0);
334 }
335
336 fn emitPackedDwordUmulhi(emitter: anytype) !void {
337 try emitter.emitEncoding(encoding.movups(.xmm2, .{ .reg = .xmm0 }));
338 try emitter.emitEncoding(encoding.movups(.xmm3, .{ .reg = .xmm1 }));
339 try emitter.emitEncoding(encoding.pmuludq(.xmm0, .{ .reg = .xmm1 }));
340 try emitter.emitEncoding(encoding.psrlqImm(.xmm2, 32));
341 try emitter.emitEncoding(encoding.psrlqImm(.xmm3, 32));
342 try emitter.emitEncoding(encoding.pmuludq(.xmm2, .{ .reg = .xmm3 }));
343 try emitter.emitEncoding(encoding.psrlqImm(.xmm0, 32));
344 try emitter.emitEncoding(encoding.psrlqImm(.xmm2, 32));
345 try emitter.emitEncoding(encoding.psllqImm(.xmm2, 32));
346 try emitter.emitEncoding(encoding.por(.xmm0, .{ .reg = .xmm2 }));
347 }
348
349 fn emitPackedIntUmulhi(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot, int_kind: PackedInt) !bool {
350 if (int_kind != .dwordx4) return false;
351 try emitPackedLoad(emitter, lhs, .xmm0);
352 try emitPackedLoad(emitter, rhs, .xmm1);
353 try emitPackedDwordUmulhi(emitter);
354 try emitPackedStore(emitter, result, .xmm0);
355 return true;
356 }
357
358 fn emitCopyVectorSlot(emitter: anytype, src: slot_layout.VectorSlot, dst: slot_layout.VectorSlot) !void {
359 if (!sameVectorStorageShape(src, dst)) return error.UnsupportedType;
360
361 if (packedKind(src) != null) {
362 if (packedKind(dst) != null) {
363 try emitPackedLoad(emitter, src, .xmm0);
364 try emitPackedStore(emitter, dst, .xmm0);
365 return;
366 }
367 }
368
369 for (0..src.lanes) |lane| {
370 const src_lane = slot_layout.laneSlot(src.base, lane);
371 const dst_lane = slot_layout.laneSlot(dst.base, lane);
372 if (src.is_float) {
373 try emitter.loadSlotXmm(src_lane, .xmm0);
374 try emitter.storeSlotXmm(dst_lane, .xmm0);
375 } else {
376 try emitter.loadSlot(src_lane, .rax);
377 try emitter.storeSlot(dst_lane, .rax);
378 }
379 }
380 }
381
382 fn definingOp(value: *ir.Value) ?*ir.Operation {
383 return switch (value.kind) {
384 .op_result => |info| @ptrCast(@alignCast(info.owner)),
385 .block_argument => null,
386 };
387 }
388
389 fn constantIntValue(value: *ir.Value) ?i64 {
390 const op = definingOp(value) orelse return null;
391 const name = op.name.name;
392 if (std.mem.eql(u8, name, ArithDialect.ConstantOp.operation_name)) {
393 if (data.hasAttributes(op)) return null;
394 return (ArithDialect.ConstantOp{ .op = op }).getIntValue();
395 }
396 if (std.mem.eql(u8, name, ArithDialect.VecConstantOp.operation_name)) {
397 return (ArithDialect.VecConstantOp{ .op = op }).getIntValue();
398 }
399 return null;
400 }
401
402 fn boundedShiftCount(value: i64, lane_width: u8) ?u8 {
403 if (value < 0) return null;
404 const count: u64 = @intCast(value);
405 if (count >= lane_width) return null;
406 return @intCast(count);
407 }
408
409 fn uniformShiftCount(value: *ir.Value, lane_width: u8) ?u8 {
410 if (constantIntValue(value)) |count| return boundedShiftCount(count, lane_width);
411 const op = definingOp(value) orelse return null;
412 if (!std.mem.eql(u8, op.name.name, ArithDialect.SplatOp.operation_name)) return null;
413 const scalar = op.getOperand(0) orelse return null;
414 const count = constantIntValue(scalar) orelse return null;
415 return boundedShiftCount(count, lane_width);
416 }
417
418 fn emitPackedIntShiftUniform(
419 emitter: anytype,
420 value: slot_layout.VectorSlot,
421 result: slot_layout.VectorSlot,
422 kind: scalar_backend.ShiftKind,
423 int_kind: PackedInt,
424 count: u8,
425 ) !bool {
426 if (int_kind == .qwordx2 and kind == .shr) return false;
427
428 try emitPackedLoad(emitter, value, .xmm0);
429 switch (int_kind) {
430 .dwordx4 => switch (kind) {
431 .shl => try emitter.emitEncoding(encoding.pslldImm(.xmm0, count)),
432 .shr => try emitter.emitEncoding(encoding.psradImm(.xmm0, count)),
433 .ushr => try emitter.emitEncoding(encoding.psrldImm(.xmm0, count)),
434 },
435 .qwordx2 => switch (kind) {
436 .shl => try emitter.emitEncoding(encoding.psllqImm(.xmm0, count)),
437 .shr => unreachable,
438 .ushr => try emitter.emitEncoding(encoding.psrlqImm(.xmm0, count)),
439 },
440 }
441 try emitPackedStore(emitter, result, .xmm0);
442 return true;
443 }
444
445 const FloatCmpPlan = struct {
446 imm: u8,
447 swap: bool = false,
448 };
449
450 fn floatCmpPlan(pred: CmpPredicate) FloatCmpPlan {
451 return switch (pred) {
452 .eq => .{ .imm = 0 },
453 .ne => .{ .imm = 4 },
454 .lt, .slt, .ult => .{ .imm = 1 },
455 .le, .sle, .ule => .{ .imm = 2 },
456 .gt, .sgt, .ugt => .{ .imm = 1, .swap = true },
457 .ge, .sge, .uge => .{ .imm = 2, .swap = true },
458 };
459 }
460
461 fn emitPackedFloatCmp(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot, pred: CmpPredicate, float_kind: PackedFloat) !void {
462 const plan = floatCmpPlan(pred);
463 if (plan.swap) {
464 try emitPackedLoad(emitter, rhs, .xmm0);
465 try emitPackedLoad(emitter, lhs, .xmm1);
466 } else {
467 try emitPackedLoad(emitter, lhs, .xmm0);
468 try emitPackedLoad(emitter, rhs, .xmm1);
469 }
470 switch (float_kind) {
471 .f32x4 => {
472 try emitter.emitEncoding(encoding.cmpps(.xmm0, .{ .reg = .xmm1 }, plan.imm));
473 try emitter.emitEncoding(encoding.psrldImm(.xmm0, 31));
474 },
475 .f64x2 => {
476 try emitter.emitEncoding(encoding.cmppd(.xmm0, .{ .reg = .xmm1 }, plan.imm));
477 try emitter.emitEncoding(encoding.psrlqImm(.xmm0, 63));
478 },
479 }
480 try emitPackedStore(emitter, result, .xmm0);
481 }
482
483 const IntCmpRelation = enum {
484 eq,
485 gt,
486 };
487
488 const IntCmpPlan = struct {
489 relation: IntCmpRelation,
490 swap: bool = false,
491 invert: bool = false,
492 unsigned: bool = false,
493 };
494
495 fn intCmpPlan(pred: CmpPredicate) IntCmpPlan {
496 return switch (pred) {
497 .eq => .{ .relation = .eq },
498 .ne => .{ .relation = .eq, .invert = true },
499 .gt, .sgt => .{ .relation = .gt },
500 .lt, .slt => .{ .relation = .gt, .swap = true },
501 .ge, .sge => .{ .relation = .gt, .swap = true, .invert = true },
502 .le, .sle => .{ .relation = .gt, .invert = true },
503 .ugt => .{ .relation = .gt, .unsigned = true },
504 .ult => .{ .relation = .gt, .swap = true, .unsigned = true },
505 .uge => .{ .relation = .gt, .swap = true, .invert = true, .unsigned = true },
506 .ule => .{ .relation = .gt, .invert = true, .unsigned = true },
507 };
508 }
509
510 fn emitPackedAllOnes(emitter: anytype, reg: registers.XMM) !void {
511 try emitter.emitEncoding(encoding.pcmpeqd(reg, .{ .reg = reg }));
512 }
513
514 fn emitPackedOneBits(emitter: anytype, kind: PackedInt, reg: registers.XMM) !void {
515 try emitPackedAllOnes(emitter, reg);
516 switch (kind) {
517 .dwordx4 => try emitter.emitEncoding(encoding.psrldImm(reg, 31)),
518 .qwordx2 => try emitter.emitEncoding(encoding.psrlqImm(reg, 63)),
519 }
520 }
521
522 fn emitPackedDwordSignMask(emitter: anytype, reg: registers.XMM) !void {
523 try emitPackedAllOnes(emitter, reg);
524 try emitter.emitEncoding(encoding.pslldImm(reg, 31));
525 }
526
527 fn emitPackedIntCmpInvert(emitter: anytype, kind: PackedInt) !void {
528 try emitPackedOneBits(emitter, kind, .xmm1);
529 try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm1 }));
530 }
531
532 fn emitPackedDwordCmpMask(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, plan: IntCmpPlan) !void {
533 if (plan.swap) {
534 try emitPackedLoad(emitter, rhs, .xmm0);
535 try emitPackedLoad(emitter, lhs, .xmm1);
536 } else {
537 try emitPackedLoad(emitter, lhs, .xmm0);
538 try emitPackedLoad(emitter, rhs, .xmm1);
539 }
540 switch (plan.relation) {
541 .eq => try emitter.emitEncoding(encoding.pcmpeqd(.xmm0, .{ .reg = .xmm1 })),
542 .gt => {
543 if (plan.unsigned) {
544 try emitPackedDwordSignMask(emitter, .xmm2);
545 try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm2 }));
546 try emitter.emitEncoding(encoding.pxor(.xmm1, .{ .reg = .xmm2 }));
547 }
548 try emitter.emitEncoding(encoding.pcmpgtd(.xmm0, .{ .reg = .xmm1 }));
549 },
550 }
551 }
552
553 fn emitPackedDwordCmpBits(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, plan: IntCmpPlan) !void {
554 try emitPackedDwordCmpMask(emitter, lhs, rhs, plan);
555 try emitter.emitEncoding(encoding.psrldImm(.xmm0, 31));
556 }
557
558 fn emitPackedQwordEqCmpBits(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot) !void {
559 try emitPackedLoad(emitter, lhs, .xmm0);
560 try emitPackedLoad(emitter, rhs, .xmm1);
561 try emitter.emitEncoding(encoding.pcmpeqd(.xmm0, .{ .reg = .xmm1 }));
562 try emitter.emitEncoding(encoding.pshufd(.xmm1, .{ .reg = .xmm0 }, 0xB1));
563 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm1 }));
564 try emitter.emitEncoding(encoding.psrlqImm(.xmm0, 63));
565 }
566
567 fn emitPackedQwordGtCmpBits(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, unsigned: bool) !void {
568 try emitPackedLoad(emitter, lhs, .xmm0);
569 try emitPackedLoad(emitter, rhs, .xmm1);
570
571 try emitter.emitEncoding(encoding.movupd(.xmm2, .{ .reg = .xmm0 }));
572 try emitter.emitEncoding(encoding.movupd(.xmm3, .{ .reg = .xmm1 }));
573 if (unsigned) {
574 try emitPackedDwordSignMask(emitter, .xmm4);
575 try emitter.emitEncoding(encoding.pxor(.xmm2, .{ .reg = .xmm4 }));
576 try emitter.emitEncoding(encoding.pxor(.xmm3, .{ .reg = .xmm4 }));
577 }
578 try emitter.emitEncoding(encoding.pcmpgtd(.xmm2, .{ .reg = .xmm3 }));
579
580 try emitter.emitEncoding(encoding.movupd(.xmm3, .{ .reg = .xmm0 }));
581 try emitter.emitEncoding(encoding.pcmpeqd(.xmm3, .{ .reg = .xmm1 }));
582
583 try emitPackedDwordSignMask(emitter, .xmm4);
584 try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm4 }));
585 try emitter.emitEncoding(encoding.pxor(.xmm1, .{ .reg = .xmm4 }));
586 try emitter.emitEncoding(encoding.pcmpgtd(.xmm0, .{ .reg = .xmm1 }));
587 try emitter.emitEncoding(encoding.psllqImm(.xmm0, 32));
588 try emitter.emitEncoding(encoding.pand(.xmm0, .{ .reg = .xmm3 }));
589 try emitter.emitEncoding(encoding.por(.xmm2, .{ .reg = .xmm0 }));
590 try emitter.emitEncoding(encoding.movupd(.xmm0, .{ .reg = .xmm2 }));
591 try emitter.emitEncoding(encoding.psrlqImm(.xmm0, 63));
592 }
593
594 fn emitPackedQwordCmpBits(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, plan: IntCmpPlan) !void {
595 switch (plan.relation) {
596 .eq => try emitPackedQwordEqCmpBits(emitter, lhs, rhs),
597 .gt => {
598 const gt_lhs = if (plan.swap) rhs else lhs;
599 const gt_rhs = if (plan.swap) lhs else rhs;
600 try emitPackedQwordGtCmpBits(emitter, gt_lhs, gt_rhs, plan.unsigned);
601 },
602 }
603 }
604
605 fn emitPackedQwordBitsToMask(emitter: anytype) !void {
606 try emitter.emitEncoding(encoding.pxor(.xmm1, .{ .reg = .xmm1 }));
607 try emitter.emitEncoding(encoding.psubq(.xmm1, .{ .reg = .xmm0 }));
608 try emitter.emitEncoding(encoding.movupd(.xmm0, .{ .reg = .xmm1 }));
609 }
610
611 fn emitPackedIntCmp(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot, pred: CmpPredicate, int_kind: PackedInt) !void {
612 const plan = intCmpPlan(pred);
613 switch (int_kind) {
614 .dwordx4 => try emitPackedDwordCmpBits(emitter, lhs, rhs, plan),
615 .qwordx2 => try emitPackedQwordCmpBits(emitter, lhs, rhs, plan),
616 }
617 if (plan.invert) try emitPackedIntCmpInvert(emitter, int_kind);
618 try emitPackedStore(emitter, result, .xmm0);
619 }
620
621 fn emitPackedIntBlendMask(emitter: anytype, true_slot: slot_layout.VectorSlot, false_slot: slot_layout.VectorSlot, result: slot_layout.VectorSlot, int_kind: PackedInt) !void {
622 try emitPackedLoad(emitter, true_slot, .xmm1);
623 try emitPackedLoad(emitter, false_slot, .xmm2);
624 switch (int_kind) {
625 .dwordx4 => try emitter.emitEncoding(encoding.movups(.xmm3, .{ .reg = .xmm0 })),
626 .qwordx2 => try emitter.emitEncoding(encoding.movupd(.xmm3, .{ .reg = .xmm0 })),
627 }
628 try emitter.emitEncoding(encoding.pand(.xmm1, .{ .reg = .xmm0 }));
629 try emitPackedAllOnes(emitter, .xmm4);
630 try emitter.emitEncoding(encoding.pxor(.xmm3, .{ .reg = .xmm4 }));
631 try emitter.emitEncoding(encoding.pand(.xmm2, .{ .reg = .xmm3 }));
632 try emitter.emitEncoding(encoding.por(.xmm1, .{ .reg = .xmm2 }));
633 try emitPackedStore(emitter, result, .xmm1);
634 }
635
636 fn emitPackedIntMinMax(emitter: anytype, lhs: slot_layout.VectorSlot, rhs: slot_layout.VectorSlot, result: slot_layout.VectorSlot, kind: scalar_backend.MinMaxKind, int_kind: PackedInt) !void {
637 const plan: IntCmpPlan = .{
638 .relation = .gt,
639 .unsigned = scalar_backend.isUnsignedIntegerTypeName(result.elem_type_name),
640 };
641 switch (int_kind) {
642 .dwordx4 => try emitPackedDwordCmpMask(emitter, lhs, rhs, plan),
643 .qwordx2 => {
644 try emitPackedQwordCmpBits(emitter, lhs, rhs, plan);
645 try emitPackedQwordBitsToMask(emitter);
646 },
647 }
648 const true_slot = switch (kind) {
649 .max => lhs,
650 .min => rhs,
651 };
652 const false_slot = switch (kind) {
653 .max => rhs,
654 .min => lhs,
655 };
656 try emitPackedIntBlendMask(emitter, true_slot, false_slot, result, int_kind);
657 }
658
659 fn packedIntCmpSupported(pred: CmpPredicate, int_kind: PackedInt) bool {
660 _ = pred;
661 return switch (int_kind) {
662 .dwordx4 => true,
663 .qwordx2 => true,
664 };
665 }
666
667 fn emitPackedFloatNeg(emitter: anytype, input: slot_layout.VectorSlot, result: slot_layout.VectorSlot, float_kind: PackedFloat) !void {
668 try emitPackedLoad(emitter, input, .xmm0);
669 try emitter.emitEncoding(encoding.pcmpeqd(.xmm1, .{ .reg = .xmm1 }));
670 switch (float_kind) {
671 .f32x4 => {
672 try emitter.emitEncoding(encoding.pslldImm(.xmm1, 31));
673 try emitter.emitEncoding(encoding.xorps(.xmm0, .{ .reg = .xmm1 }));
674 },
675 .f64x2 => {
676 try emitter.emitEncoding(encoding.psllqImm(.xmm1, 63));
677 try emitter.emitEncoding(encoding.xorpd(.xmm0, .{ .reg = .xmm1 }));
678 },
679 }
680 try emitPackedStore(emitter, result, .xmm0);
681 }
682
683 fn emitPackedIntNeg(emitter: anytype, input: slot_layout.VectorSlot, result: slot_layout.VectorSlot, int_kind: PackedInt) !void {
684 switch (int_kind) {
685 .dwordx4 => {
686 try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm0 }));
687 try emitPackedLoad(emitter, input, .xmm1);
688 try emitter.emitEncoding(encoding.psubd(.xmm0, .{ .reg = .xmm1 }));
689 },
690 .qwordx2 => {
691 try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm0 }));
692 try emitPackedLoad(emitter, input, .xmm1);
693 try emitter.emitEncoding(encoding.psubq(.xmm0, .{ .reg = .xmm1 }));
694 },
695 }
696 try emitPackedStore(emitter, result, .xmm0);
697 }
698
699 fn emitPackedIntNot(emitter: anytype, input: slot_layout.VectorSlot, result: slot_layout.VectorSlot) !void {
700 try emitPackedLoad(emitter, input, .xmm0);
701 try emitPackedAllOnes(emitter, .xmm1);
702 try emitter.emitEncoding(encoding.pxor(.xmm0, .{ .reg = .xmm1 }));
703 try emitPackedStore(emitter, result, .xmm0);
704 }
705
706 fn emitPackedFloatSplat(emitter: anytype, scalar: *ir.Value, result: *ir.Value, float_kind: PackedFloat) !void {
707 try emitter.loadIntoXmm(scalar, .xmm0);
708 try emitPackedFloatBroadcastStore(emitter, result, float_kind);
709 }
710
711 fn emitPackedFloatBroadcastStore(emitter: anytype, result: *ir.Value, float_kind: PackedFloat) !void {
712 switch (float_kind) {
713 .f32x4 => try emitter.emitEncoding(encoding.shufps(.xmm0, .{ .reg = .xmm0 }, 0)),
714 .f64x2 => try emitter.emitEncoding(encoding.shufpd(.xmm0, .{ .reg = .xmm0 }, 0)),
715 }
716 try emitter.storeFromXmmPacked(result, .xmm0);
717 }
718
719 fn emitPackedFloatConstant(emitter: anytype, result: *ir.Value, value: f64, float_kind: PackedFloat) !void {
720 switch (float_kind) {
721 .f32x4 => {
722 const bits: u32 = @bitCast(@as(f32, @floatCast(value)));
723 try emitter.emitEncoding(encoding.movRegImm32(.rax, bits));
724 try emitter.emitEncoding(encoding.movdXmmFromReg32(.xmm0, .rax));
725 },
726 .f64x2 => {
727 const bits: u64 = @bitCast(value);
728 try emitter.emitEncoding(encoding.movRegImm64(.rax, bits));
729 try emitter.emitEncoding(encoding.movqXmmFromReg64(.xmm0, .rax));
730 },
731 }
732 try emitPackedFloatBroadcastStore(emitter, result, float_kind);
733 }
734
735 fn emitPackedDwordBroadcastStore(emitter: anytype, result: *ir.Value) !void {
736 try emitter.emitEncoding(encoding.pshufd(.xmm0, .{ .reg = .xmm0 }, 0));
737 try emitter.storeFromXmmPacked(result, .xmm0);
738 }
739
740 fn emitPackedQwordBroadcastStore(emitter: anytype, result: *ir.Value) !void {
741 try emitter.emitEncoding(encoding.shufpd(.xmm0, .{ .reg = .xmm0 }, 0));
742 try emitter.storeFromXmmPacked(result, .xmm0);
743 }
744
745 fn emitPackedIntSplat(emitter: anytype, scalar: *ir.Value, result: *ir.Value, int_kind: PackedInt) !void {
746 const scalar_home = emitter.registerHome(scalar);
747 switch (int_kind) {
748 .dwordx4 => {
749 if (scalar_home) |home| {
750 try emitter.emitEncoding(encoding.movRegReg32(.rax, home));
751 try emitter.emitEncoding(encoding.movdXmmFromReg32(.xmm0, .rax));
752 } else {
753 const mem = encoding.Mem.baseDisp(.rbp, (try emitter.slotFor(scalar)).offset);
754 try emitter.emitEncoding(encoding.movdXmmFromMem32(.xmm0, mem));
755 }
756 try emitPackedDwordBroadcastStore(emitter, result);
757 },
758 .qwordx2 => {
759 if (scalar_home) |home| {
760 try emitter.emitEncoding(encoding.movqXmmFromReg64(.xmm0, home));
761 } else {
762 const mem = encoding.Mem.baseDisp(.rbp, (try emitter.slotFor(scalar)).offset);
763 try emitter.emitEncoding(encoding.movqXmmFromMem64(.xmm0, mem));
764 }
765 try emitPackedQwordBroadcastStore(emitter, result);
766 },
767 }
768 }
769
770 fn emitPackedIntConstant(emitter: anytype, result: *ir.Value, value: i64, int_kind: PackedInt) !void {
771 switch (int_kind) {
772 .dwordx4 => {
773 const bits: u64 = @bitCast(value);
774 const masked: u32 = @intCast(bits & 0xFFFF_FFFF);
775 try emitter.emitEncoding(encoding.movRegImm32(.rax, masked));
776 try emitter.emitEncoding(encoding.movdXmmFromReg32(.xmm0, .rax));
777 try emitPackedDwordBroadcastStore(emitter, result);
778 },
779 .qwordx2 => {
780 const bits: u64 = @bitCast(value);
781 try emitter.emitEncoding(encoding.movRegImm64(.rax, bits));
782 try emitter.emitEncoding(encoding.movqXmmFromReg64(.xmm0, .rax));
783 try emitPackedQwordBroadcastStore(emitter, result);
784 },
785 }
786 }
787
788 fn packedSourceElement(index: i64, lanes: usize) !u8 {
789 if (index < 0) return error.IndexOutOfBounds;
790 const lane: usize = @intCast(index);
791 if (lane >= lanes) return error.IndexOutOfBounds;
792 return @intCast(lanes - 1 - lane);
793 }
794
795 fn packedShuffleImm(indices: []const i64, lanes: usize, selector_bits: usize) !u8 {
796 if (indices.len != lanes) return error.VectorArityMismatch;
797 var imm: u8 = 0;
798 for (0..lanes) |physical_dest| {
799 const logical_dest = lanes - 1 - physical_dest;
800 const physical_source = try packedSourceElement(indices[logical_dest], lanes);
801 const shift: u3 = @intCast(physical_dest * selector_bits);
802 imm |= physical_source << shift;
803 }
804 return imm;
805 }
806
807 fn packedFloatShuffleImm(indices: []const i64, float_kind: PackedFloat) !u8 {
808 const lanes: usize = switch (float_kind) {
809 .f32x4 => 4,
810 .f64x2 => 2,
811 };
812 const selector_bits: usize = switch (float_kind) {
813 .f32x4 => 2,
814 .f64x2 => 1,
815 };
816 return packedShuffleImm(indices, lanes, selector_bits);
817 }
818
819 fn emitPackedFloatShuffle(emitter: anytype, input: slot_layout.VectorSlot, result: slot_layout.VectorSlot, indices: []const i64, float_kind: PackedFloat) !void {
820 const imm = try packedFloatShuffleImm(indices, float_kind);
821 try emitPackedLoad(emitter, input, .xmm0);
822 switch (float_kind) {
823 .f32x4 => try emitter.emitEncoding(encoding.shufps(.xmm0, .{ .reg = .xmm0 }, imm)),
824 .f64x2 => try emitter.emitEncoding(encoding.shufpd(.xmm0, .{ .reg = .xmm0 }, imm)),
825 }
826 try emitPackedStore(emitter, result, .xmm0);
827 }
828
829 fn emitPackedIntShuffle(emitter: anytype, input: slot_layout.VectorSlot, result: slot_layout.VectorSlot, indices: []const i64, int_kind: PackedInt) !void {
830 switch (int_kind) {
831 .dwordx4 => {
832 const imm = try packedShuffleImm(indices, 4, 2);
833 try emitPackedLoad(emitter, input, .xmm0);
834 try emitter.emitEncoding(encoding.pshufd(.xmm0, .{ .reg = .xmm0 }, imm));
835 try emitPackedStore(emitter, result, .xmm0);
836 },
837 .qwordx2 => {
838 const imm = try packedShuffleImm(indices, 2, 1);
839 try emitPackedLoad(emitter, input, .xmm0);
840 try emitter.emitEncoding(encoding.shufpd(.xmm0, .{ .reg = .xmm0 }, imm));
841 try emitPackedStore(emitter, result, .xmm0);
842 },
843 }
844 }
845
846 fn samePackedInsertShape(input: slot_layout.VectorSlot, result: slot_layout.VectorSlot) ?PackedKind {
847 if (input.lanes != result.lanes or input.base.width != result.base.width) return null;
848 if (input.is_float != result.is_float) return null;
849 if (!std.mem.eql(u8, input.elem_type_name, result.elem_type_name)) return null;
850 const input_kind = packedKind(input) orelse return null;
851 const result_kind = packedKind(result) orelse return null;
852 if (!std.meta.eql(input_kind, result_kind)) return null;
853 return input_kind;
854 }
855
856 fn emitPackedExtractLowStore(emitter: anytype, result: slot_layout.Slot, is_float: bool) !void {
857 const mem = encoding.Mem.baseDisp(.rbp, result.offset);
858 if (is_float) {
859 switch (result.width) {
860 32 => try emitter.emitEncoding(encoding.movssStore(mem, .xmm0)),
861 64 => try emitter.emitEncoding(encoding.movsdStore(mem, .xmm0)),
862 else => return error.UnsupportedType,
863 }
864 } else {
865 switch (result.width) {
866 32 => try emitter.emitEncoding(encoding.movdMemFromXmm32(mem, .xmm0)),
867 64 => try emitter.emitEncoding(encoding.movqMemFromXmm64(mem, .xmm0)),
868 else => return error.UnsupportedType,
869 }
870 }
871 }
872
873 fn emitPackedExtract(emitter: anytype, vector: slot_layout.VectorSlot, result: slot_layout.Slot, index: i64, kind: PackedKind) !void {
874 if (result.width != vector.base.width) return error.UnsupportedType;
875 const source = try packedSourceElement(index, vector.lanes);
876
877 try emitPackedLoad(emitter, vector, .xmm0);
878 switch (kind) {
879 .float => |float_kind| switch (float_kind) {
880 .f32x4 => try emitter.emitEncoding(encoding.shufps(.xmm0, .{ .reg = .xmm0 }, source)),
881 .f64x2 => try emitter.emitEncoding(encoding.shufpd(.xmm0, .{ .reg = .xmm0 }, source)),
882 },
883 .int => |int_kind| switch (int_kind) {
884 .dwordx4 => try emitter.emitEncoding(encoding.pshufd(.xmm0, .{ .reg = .xmm0 }, source)),
885 .qwordx2 => try emitter.emitEncoding(encoding.shufpd(.xmm0, .{ .reg = .xmm0 }, source)),
886 },
887 }
888 try emitPackedExtractLowStore(emitter, result, vector.is_float);
889 }
890
891 fn emitPackedInsert(emitter: anytype, vector: slot_layout.VectorSlot, scalar: slot_layout.Slot, result: slot_layout.VectorSlot, index: usize, kind: PackedKind) !void {
892 if (scalar.width != vector.base.width) return error.UnsupportedType;
893 try emitPackedLoad(emitter, vector, .xmm0);
894 try emitPackedStoreToMem(emitter, kind, packedMem(result), .xmm0);
895
896 const dst_lane = slot_layout.laneSlot(result.base, index);
897 try emitter.loadSlot(scalar, .rax);
898 try emitter.storeSlot(dst_lane, .rax);
899 }
900
901 pub fn emitConstant(emitter: anytype, op: *ir.Operation) !void {
902 const vec_op = ArithDialect.VecConstantOp{ .op = op };
903 const result = vec_op.getResult();
904 const vector = try emitter.vectorSlotFor(result);
905
906 const int_val = vec_op.getIntValue();
907 const float_val = vec_op.getFloatValue();
908 if (int_val == null and float_val == null) return error.InvalidConstant;
909
910 if (int_val) |ival| {
911 if (packedIntKind(vector)) |int_kind| {
912 try emitPackedIntConstant(emitter, result, ival, int_kind);
913 return;
914 }
915 }
916
917 if (float_val) |fval| {
918 if (packedFloatKind(vector)) |float_kind| {
919 try emitPackedFloatConstant(emitter, result, fval, float_kind);
920 return;
921 }
922 }
923
924 for (0..vector.lanes) |lane| {
925 const lane_slot = slot_layout.laneSlot(vector.base, lane);
926 if (int_val) |ival| {
927 const bits: u64 = @bitCast(ival);
928 const masked: u64 = switch (lane_slot.width) {
929 8 => bits & 0xFF,
930 16 => bits & 0xFFFF,
931 32 => bits & 0xFFFF_FFFF,
932 64 => bits,
933 else => return error.UnsupportedType,
934 };
935 if (lane_slot.width <= 32) {
936 try emitter.emitEncoding(encoding.movRegImm32(.rax, @intCast(masked)));
937 } else {
938 try emitter.emitEncoding(encoding.movRegImm64(.rax, masked));
939 }
940 try emitter.storeSlot(lane_slot, .rax);
941 } else if (float_val) |fval| {
942 if (lane_slot.width == 32) {
943 const bits: u32 = @bitCast(@as(f32, @floatCast(fval)));
944 try emitter.emitEncoding(encoding.movRegImm32(.rax, bits));
945 try emitter.storeSlot(lane_slot, .rax);
946 } else if (lane_slot.width == 64) {
947 const bits: u64 = @bitCast(fval);
948 try emitter.emitEncoding(encoding.movRegImm64(.rax, bits));
949 try emitter.storeSlot(lane_slot, .rax);
950 } else {
951 return error.UnsupportedType;
952 }
953 }
954 }
955 }
956
957 pub fn emitSplat(emitter: anytype, op: *ir.Operation) !void {
958 const splat_op = ArithDialect.SplatOp{ .op = op };
959 const result = splat_op.getResult();
960 const scalar = splat_op.getInput();
961 const scalar_slot = try emitter.slotFor(scalar);
962 const vector = try emitter.vectorSlotFor(result);
963
964 if (packedFloatKind(vector)) |float_kind| {
965 if (scalar_slot.width == vector.base.width) {
966 try emitPackedFloatSplat(emitter, scalar, result, float_kind);
967 return;
968 }
969 }
970
971 if (packedIntKind(vector)) |int_kind| {
972 if (scalar_slot.width == vector.base.width) {
973 try emitPackedIntSplat(emitter, scalar, result, int_kind);
974 return;
975 }
976 }
977
978 if (scalar_slot.width != vector.base.width) {
979 return error.UnsupportedType;
980 }
981
982 if (vector.base.width <= 32 and !vector.is_float) {
983 try emitter.loadSlot(scalar_slot, .rax);
984 for (0..vector.lanes) |lane| {
985 const lane_slot = slot_layout.laneSlot(vector.base, lane);
986 try emitter.storeSlot(lane_slot, .rax);
987 }
988 return;
989 }
990
991 try emitter.loadSlot(scalar_slot, .rax);
992 for (0..vector.lanes) |lane| {
993 const lane_slot = slot_layout.laneSlot(vector.base, lane);
994 try emitter.storeSlot(lane_slot, .rax);
995 }
996 }
997
998 pub fn emitExtract(emitter: anytype, op: *ir.Operation) !void {
999 const extract_op = ArithDialect.ExtractOp{ .op = op };
1000 const result = extract_op.getResult();
1001 const vector = extract_op.getVector();
1002 const index = extract_op.getIndex() orelse return error.MissingIndex;
1003
1004 if (index < 0) return error.IndexOutOfBounds;
1005 const vector_slot = try emitter.vectorSlotFor(vector);
1006 const lane_index: usize = @intCast(index);
1007 if (lane_index >= vector_slot.lanes) return error.IndexOutOfBounds;
1008
1009 const lane_slot = slot_layout.laneSlot(vector_slot.base, lane_index);
1010 const result_slot = try emitter.slotFor(result);
1011 if (packedKind(vector_slot)) |kind| {
1012 try emitPackedExtract(emitter, vector_slot, result_slot, index, kind);
1013 return;
1014 }
1015 try emitter.loadSlot(lane_slot, .rax);
1016 try emitter.storeSlot(result_slot, .rax);
1017 }
1018
1019 pub fn emitInsert(emitter: anytype, op: *ir.Operation) !void {
1020 const insert_op = ArithDialect.InsertOp{ .op = op };
1021 const result = insert_op.getResult();
1022 const vector = insert_op.getVector();
1023 const scalar = insert_op.getScalar();
1024 const index = insert_op.getIndex() orelse return error.MissingIndex;
1025
1026 if (index < 0) return error.IndexOutOfBounds;
1027 const vector_slot = try emitter.vectorSlotFor(vector);
1028 const lane_index: usize = @intCast(index);
1029 if (lane_index >= vector_slot.lanes) return error.IndexOutOfBounds;
1030
1031 const result_slot = try emitter.vectorSlotFor(result);
1032 const scalar_slot = try emitter.slotFor(scalar);
1033
1034 if (samePackedInsertShape(vector_slot, result_slot)) |kind| {
1035 try emitPackedInsert(emitter, vector_slot, scalar_slot, result_slot, lane_index, kind);
1036 return;
1037 }
1038
1039 for (0..vector_slot.lanes) |lane| {
1040 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1041 if (lane == lane_index) {
1042 try emitter.loadSlot(scalar_slot, .rax);
1043 try emitter.storeSlot(dst_lane, .rax);
1044 } else {
1045 const src_lane = slot_layout.laneSlot(vector_slot.base, lane);
1046 try emitter.loadSlot(src_lane, .rax);
1047 try emitter.storeSlot(dst_lane, .rax);
1048 }
1049 }
1050 }
1051
1052 pub fn emitShuffle(emitter: anytype, op: *ir.Operation) !void {
1053 const shuffle_op = ArithDialect.VecShuffleOp{ .op = op };
1054 const result = shuffle_op.getResult();
1055 const vector = shuffle_op.getVector();
1056
1057 const result_slot = try emitter.vectorSlotFor(result);
1058 const vector_slot = try emitter.vectorSlotFor(vector);
1059 if (!std.mem.eql(u8, result_slot.elem_type_name, vector_slot.elem_type_name)) return error.UnsupportedType;
1060
1061 var buf: [16]i64 = undefined;
1062 const indices = shuffle_op.getIndices(&buf) catch |err| switch (err) {
1063 error.MissingIndices => return error.MissingIndex,
1064 error.InvalidIndices => return error.UnsupportedOperation,
1065 error.TooManyIndices => return error.VectorArityMismatch,
1066 };
1067
1068 if (indices.len != result_slot.lanes) return error.VectorArityMismatch;
1069
1070 if (packedFloatKind(vector_slot)) |float_kind| {
1071 if (packedFloatKind(result_slot)) |result_kind| {
1072 if (result_kind == float_kind) {
1073 try emitPackedFloatShuffle(emitter, vector_slot, result_slot, indices, float_kind);
1074 return;
1075 }
1076 }
1077 }
1078
1079 if (packedIntKind(vector_slot)) |int_kind| {
1080 if (packedIntKind(result_slot)) |result_kind| {
1081 if (result_kind == int_kind) {
1082 try emitPackedIntShuffle(emitter, vector_slot, result_slot, indices, int_kind);
1083 return;
1084 }
1085 }
1086 }
1087
1088 if (vector_slot.base.width != result_slot.base.width) {
1089 return error.UnsupportedType;
1090 }
1091
1092 if (vector_slot.is_float != result_slot.is_float) {
1093 return error.UnsupportedType;
1094 }
1095
1096 if (result_slot.lanes != vector_slot.lanes and indices.len != result_slot.lanes) {
1097 return error.VectorArityMismatch;
1098 }
1099
1100 if (indices.len != result_slot.lanes) {
1101 return error.VectorArityMismatch;
1102 }
1103
1104 if (!std.mem.eql(u8, result_slot.elem_type_name, vector_slot.elem_type_name)) {
1105 return error.UnsupportedType;
1106 }
1107
1108 if (indices.len == 0) {
1109 return;
1110 }
1111
1112 for (indices, 0..) |idx, lane| {
1113 if (idx < 0) return error.IndexOutOfBounds;
1114 const src_index: usize = @intCast(idx);
1115 if (src_index >= vector_slot.lanes) return error.IndexOutOfBounds;
1116
1117 const src_lane = slot_layout.laneSlot(vector_slot.base, src_index);
1118 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1119
1120 if (vector_slot.is_float) {
1121 try emitter.loadSlotXmm(src_lane, .xmm0);
1122 try emitter.storeSlotXmm(dst_lane, .xmm0);
1123 } else {
1124 try emitter.loadSlot(src_lane, .rax);
1125 try emitter.storeSlot(dst_lane, .rax);
1126 }
1127 }
1128 }
1129
1130 pub fn emitBinaryOp(emitter: anytype, op: *ir.Operation, kind: BinOp) !void {
1131 const result = op.getResult(0) orelse return error.MissingResult;
1132 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1133 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1134
1135 const lhs_slot = try emitter.vectorSlotFor(lhs);
1136 const rhs_slot = try emitter.vectorSlotFor(rhs);
1137 const result_slot = try emitter.vectorSlotFor(result);
1138
1139 if (lhs_slot.lanes != rhs_slot.lanes or lhs_slot.lanes != result_slot.lanes) return error.VectorArityMismatch;
1140 if (lhs_slot.is_float != rhs_slot.is_float) return error.UnsupportedType;
1141 if (lhs_slot.is_float != rhs_slot.is_float or lhs_slot.is_float != result_slot.is_float) return error.UnsupportedType;
1142
1143 if (samePackedFloatKind(lhs_slot, rhs_slot, result_slot)) |float_kind| {
1144 try emitPackedFloatBinary(emitter, lhs, rhs, result, kind, float_kind);
1145 return;
1146 }
1147
1148 if (samePackedIntKind(lhs_slot, rhs_slot, result_slot)) |int_kind| {
1149 switch (kind) {
1150 .add, .sub, .mul => {
1151 try emitPackedIntBinary(emitter, lhs, rhs, result, kind, int_kind);
1152 return;
1153 },
1154 .div => {},
1155 }
1156 }
1157
1158 for (0..lhs_slot.lanes) |lane| {
1159 const lhs_lane = slot_layout.laneSlot(lhs_slot.base, lane);
1160 const rhs_lane = slot_layout.laneSlot(rhs_slot.base, lane);
1161 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1162
1163 if (lhs_slot.is_float) {
1164 try emitter.loadSlotXmm(lhs_lane, .xmm0);
1165 try emitter.loadSlotXmm(rhs_lane, .xmm1);
1166 if (lhs_lane.width == 32) {
1167 switch (kind) {
1168 .add => try emitter.emitEncoding(encoding.addss(.xmm0, .{ .reg = .xmm1 })),
1169 .sub => try emitter.emitEncoding(encoding.subss(.xmm0, .{ .reg = .xmm1 })),
1170 .mul => try emitter.emitEncoding(encoding.mulss(.xmm0, .{ .reg = .xmm1 })),
1171 .div => try emitter.emitEncoding(encoding.divss(.xmm0, .{ .reg = .xmm1 })),
1172 }
1173 } else if (lhs_lane.width == 64) {
1174 switch (kind) {
1175 .add => try emitter.emitEncoding(encoding.addsd(.xmm0, .{ .reg = .xmm1 })),
1176 .sub => try emitter.emitEncoding(encoding.subsd(.xmm0, .{ .reg = .xmm1 })),
1177 .mul => try emitter.emitEncoding(encoding.mulsd(.xmm0, .{ .reg = .xmm1 })),
1178 .div => try emitter.emitEncoding(encoding.divsd(.xmm0, .{ .reg = .xmm1 })),
1179 }
1180 } else {
1181 return error.UnsupportedType;
1182 }
1183 try emitter.storeSlotXmm(dst_lane, .xmm0);
1184 } else {
1185 const width = lhs_lane.width;
1186 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1187 const op_width: u8 = if (width <= 32) 32 else 64;
1188
1189 const is_unsigned = scalar_backend.isUnsignedIntegerTypeName(result_slot.elem_type_name);
1190 if (kind == .div and is_unsigned) {
1191 try emitter.loadSlotUnsigned(lhs_lane, .rax);
1192 try emitter.loadSlotUnsigned(rhs_lane, .rcx);
1193 } else {
1194 try emitter.loadSlot(lhs_lane, .rax);
1195 try emitter.loadSlot(rhs_lane, .rcx);
1196 }
1197
1198 switch (kind) {
1199 .add => {
1200 if (op_width == 32) {
1201 try emitter.emitEncoding(encoding.addRegReg32(.rax, .rcx));
1202 } else {
1203 try emitter.emitEncoding(encoding.addRegReg(.rax, .rcx));
1204 }
1205 try emitter.storeSlot(dst_lane, .rax);
1206 },
1207 .sub => {
1208 if (op_width == 32) {
1209 try emitter.emitEncoding(encoding.subRegReg32(.rax, .rcx));
1210 } else {
1211 try emitter.emitEncoding(encoding.subRegReg(.rax, .rcx));
1212 }
1213 try emitter.storeSlot(dst_lane, .rax);
1214 },
1215 .mul => {
1216 if (op_width == 32) {
1217 try emitter.emitEncoding(encoding.imulRegReg32(.rax, .rcx));
1218 } else {
1219 try emitter.emitEncoding(encoding.imulRegReg(.rax, .rcx));
1220 }
1221 try emitter.storeSlot(dst_lane, .rax);
1222 },
1223 .div => {
1224 try scalar_backend.emitIntegerDiv(emitter, op_width, is_unsigned);
1225 try emitter.storeSlot(dst_lane, .rax);
1226 },
1227 }
1228 }
1229 }
1230 }
1231
1232 pub fn emitBitwiseBinary(emitter: anytype, op: *ir.Operation, kind: BitwiseBinaryOp) !void {
1233 const result = op.getResult(0) orelse return error.MissingResult;
1234 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1235 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1236
1237 const lhs_slot = try emitter.vectorSlotFor(lhs);
1238 const rhs_slot = try emitter.vectorSlotFor(rhs);
1239 const result_slot = try emitter.vectorSlotFor(result);
1240
1241 if (lhs_slot.lanes != rhs_slot.lanes or lhs_slot.lanes != result_slot.lanes) return error.VectorArityMismatch;
1242 if (lhs_slot.is_float or rhs_slot.is_float or result_slot.is_float) return error.UnsupportedType;
1243 if (!std.mem.eql(u8, lhs_slot.elem_type_name, rhs_slot.elem_type_name)) return error.UnsupportedType;
1244 if (!std.mem.eql(u8, lhs_slot.elem_type_name, result_slot.elem_type_name)) return error.UnsupportedType;
1245 if (lhs_slot.base.width != rhs_slot.base.width or lhs_slot.base.width != result_slot.base.width) return error.UnsupportedType;
1246
1247 if (samePackedIntKind(lhs_slot, rhs_slot, result_slot)) |_| {
1248 try emitPackedIntBitwiseBinary(emitter, lhs, rhs, result, kind);
1249 return;
1250 }
1251
1252 for (0..lhs_slot.lanes) |lane| {
1253 const lhs_lane = slot_layout.laneSlot(lhs_slot.base, lane);
1254 const rhs_lane = slot_layout.laneSlot(rhs_slot.base, lane);
1255 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1256
1257 const width = lhs_lane.width;
1258 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1259 const op_width: u8 = if (width <= 32) 32 else 64;
1260
1261 try emitter.loadSlot(lhs_lane, .rax);
1262 try emitter.loadSlot(rhs_lane, .rcx);
1263 if (op_width == 32) {
1264 switch (kind) {
1265 .band => try emitter.emitEncoding(encoding.andRegReg32(.rax, .rcx)),
1266 .bor => try emitter.emitEncoding(encoding.orRegReg32(.rax, .rcx)),
1267 .bxor => try emitter.emitEncoding(encoding.xorRegReg32(.rax, .rcx)),
1268 }
1269 } else {
1270 switch (kind) {
1271 .band => try emitter.emitEncoding(encoding.andRegReg(.rax, .rcx)),
1272 .bor => try emitter.emitEncoding(encoding.orRegReg(.rax, .rcx)),
1273 .bxor => try emitter.emitEncoding(encoding.xorRegReg(.rax, .rcx)),
1274 }
1275 }
1276 try emitter.storeSlot(dst_lane, .rax);
1277 }
1278 }
1279
1280 pub fn emitUmulhi(emitter: anytype, op: *ir.Operation) !void {
1281 const result = op.getResult(0) orelse return error.MissingResult;
1282 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1283 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1284
1285 const lhs_slot = try emitter.vectorSlotFor(lhs);
1286 const rhs_slot = try emitter.vectorSlotFor(rhs);
1287 const result_slot = try emitter.vectorSlotFor(result);
1288
1289 if (lhs_slot.lanes != rhs_slot.lanes or lhs_slot.lanes != result_slot.lanes) return error.VectorArityMismatch;
1290 if (lhs_slot.is_float or rhs_slot.is_float or result_slot.is_float) return error.UnsupportedType;
1291 if (!std.mem.eql(u8, lhs_slot.elem_type_name, rhs_slot.elem_type_name)) return error.UnsupportedType;
1292 if (!std.mem.eql(u8, lhs_slot.elem_type_name, result_slot.elem_type_name)) return error.UnsupportedType;
1293 if (lhs_slot.base.width != rhs_slot.base.width or lhs_slot.base.width != result_slot.base.width) return error.UnsupportedType;
1294 if (!scalar_backend.isIntegerScalarTypeName(result_slot.elem_type_name)) return error.UnsupportedType;
1295
1296 const width = result_slot.base.width;
1297 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1298
1299 if (samePackedIntKind(lhs_slot, rhs_slot, result_slot)) |int_kind| {
1300 if (try emitPackedIntUmulhi(emitter, lhs_slot, rhs_slot, result_slot, int_kind)) return;
1301 }
1302
1303 for (0..lhs_slot.lanes) |lane| {
1304 const lhs_lane = slot_layout.laneSlot(lhs_slot.base, lane);
1305 const rhs_lane = slot_layout.laneSlot(rhs_slot.base, lane);
1306 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1307
1308 try emitter.loadSlotUnsigned(lhs_lane, .rax);
1309 try emitter.loadSlotUnsigned(rhs_lane, .rcx);
1310 try scalar_backend.emitUnsignedHighProductFromRaxRcx(emitter, width);
1311 try emitter.storeSlot(dst_lane, .rax);
1312 }
1313 }
1314
1315 pub fn emitPopCount(emitter: anytype, op: *ir.Operation) !void {
1316 const result = op.getResult(0) orelse return error.MissingResult;
1317 const input = op.getOperand(0) orelse return error.MissingOperand;
1318
1319 const in_slot = try emitter.vectorSlotFor(input);
1320 const out_slot = try emitter.vectorSlotFor(result);
1321
1322 if (in_slot.lanes != out_slot.lanes) return error.VectorArityMismatch;
1323 if (in_slot.is_float or out_slot.is_float) return error.UnsupportedType;
1324 if (!std.mem.eql(u8, in_slot.elem_type_name, out_slot.elem_type_name)) return error.UnsupportedType;
1325 if (in_slot.base.width != out_slot.base.width) return error.UnsupportedType;
1326 if (std.mem.eql(u8, out_slot.elem_type_name, dialects.arith.type_names.boolean)) return error.UnsupportedType;
1327 if (!scalar_backend.isIntegerScalarTypeName(out_slot.elem_type_name) and
1328 !std.mem.eql(u8, out_slot.elem_type_name, dialects.arith.type_names.index)) return error.UnsupportedType;
1329
1330 const width = out_slot.base.width;
1331 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1332
1333 if (packedIntKind(in_slot)) |in_kind| {
1334 if (packedIntKind(out_slot)) |out_kind| {
1335 if (in_kind == out_kind) {
1336 try emitPackedIntPopCount(emitter, in_slot, out_slot, in_kind);
1337 return;
1338 }
1339 }
1340 }
1341
1342 for (0..in_slot.lanes) |lane| {
1343 const src_lane = slot_layout.laneSlot(in_slot.base, lane);
1344 const dst_lane = slot_layout.laneSlot(out_slot.base, lane);
1345
1346 try emitter.loadSlotUnsigned(src_lane, .rax);
1347 try scalar_backend.emitPopCountFromRax(emitter, width);
1348 try emitter.storeSlot(dst_lane, .rax);
1349 }
1350 }
1351
1352 pub fn emitShift(emitter: anytype, op: *ir.Operation, kind: scalar_backend.ShiftKind) !void {
1353 const result = op.getResult(0) orelse return error.MissingResult;
1354 const value = op.getOperand(0) orelse return error.MissingOperand;
1355 const count = op.getOperand(1) orelse return error.MissingOperand;
1356
1357 const value_slot = try emitter.vectorSlotFor(value);
1358 const count_slot = try emitter.vectorSlotFor(count);
1359 const result_slot = try emitter.vectorSlotFor(result);
1360
1361 if (value_slot.lanes != count_slot.lanes or value_slot.lanes != result_slot.lanes) return error.VectorArityMismatch;
1362 if (value_slot.is_float or count_slot.is_float or result_slot.is_float) return error.UnsupportedType;
1363 if (!std.mem.eql(u8, value_slot.elem_type_name, count_slot.elem_type_name)) return error.UnsupportedType;
1364 if (!std.mem.eql(u8, value_slot.elem_type_name, result_slot.elem_type_name)) return error.UnsupportedType;
1365 if (value_slot.base.width != count_slot.base.width or value_slot.base.width != result_slot.base.width) return error.UnsupportedType;
1366
1367 const elem_type_name = result_slot.elem_type_name;
1368 if (std.mem.eql(u8, elem_type_name, dialects.arith.type_names.float16) or
1369 std.mem.eql(u8, elem_type_name, dialects.arith.type_names.bfloat16) or
1370 std.mem.eql(u8, elem_type_name, dialects.arith.type_names.boolean)) return error.UnsupportedType;
1371 if (scalar_backend.isUnsignedIntegerTypeName(elem_type_name) and kind == .shr) return error.UnsupportedType;
1372
1373 const width = result_slot.base.width;
1374 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1375
1376 if (samePackedIntKind(value_slot, count_slot, result_slot)) |int_kind| {
1377 if (uniformShiftCount(count, width)) |shift_count| {
1378 if (try emitPackedIntShiftUniform(emitter, value_slot, result_slot, kind, int_kind, shift_count)) return;
1379 }
1380 }
1381
1382 const op_width: u8 = if (width <= 32) 32 else 64;
1383 for (0..value_slot.lanes) |lane| {
1384 const value_lane = slot_layout.laneSlot(value_slot.base, lane);
1385 const count_lane = slot_layout.laneSlot(count_slot.base, lane);
1386 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1387
1388 if (kind == .ushr) {
1389 try emitter.loadSlotUnsigned(value_lane, .rax);
1390 } else {
1391 try emitter.loadSlot(value_lane, .rax);
1392 }
1393 try emitter.loadSlot(count_lane, .rcx);
1394
1395 if (op_width == 32) {
1396 switch (kind) {
1397 .shl => try emitter.emitEncoding(encoding.shlRegCl32(.rax)),
1398 .shr => try emitter.emitEncoding(encoding.sarRegCl32(.rax)),
1399 .ushr => try emitter.emitEncoding(encoding.shrRegCl32(.rax)),
1400 }
1401 } else {
1402 switch (kind) {
1403 .shl => try emitter.emitEncoding(encoding.shlRegCl(.rax)),
1404 .shr => try emitter.emitEncoding(encoding.sarRegCl(.rax)),
1405 .ushr => try emitter.emitEncoding(encoding.shrRegCl(.rax)),
1406 }
1407 }
1408 try emitter.storeSlot(dst_lane, .rax);
1409 }
1410 }
1411
1412 pub fn emitMinMax(emitter: anytype, op: *ir.Operation, kind: scalar_backend.MinMaxKind) !void {
1413 const result = op.getResult(0) orelse return error.MissingResult;
1414 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1415 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1416
1417 const lhs_slot = try emitter.vectorSlotFor(lhs);
1418 const rhs_slot = try emitter.vectorSlotFor(rhs);
1419 const result_slot = try emitter.vectorSlotFor(result);
1420
1421 if (lhs_slot.lanes != rhs_slot.lanes or lhs_slot.lanes != result_slot.lanes) return error.VectorArityMismatch;
1422 if (lhs_slot.is_float != rhs_slot.is_float or lhs_slot.is_float != result_slot.is_float) return error.UnsupportedType;
1423 if (!std.mem.eql(u8, lhs_slot.elem_type_name, rhs_slot.elem_type_name)) return error.UnsupportedType;
1424 if (!std.mem.eql(u8, lhs_slot.elem_type_name, result_slot.elem_type_name)) return error.UnsupportedType;
1425
1426 const elem_type_name = result_slot.elem_type_name;
1427 if (std.mem.eql(u8, elem_type_name, dialects.arith.type_names.float16) or
1428 std.mem.eql(u8, elem_type_name, dialects.arith.type_names.bfloat16) or
1429 std.mem.eql(u8, elem_type_name, dialects.arith.type_names.boolean)) return error.UnsupportedType;
1430
1431 if (!result_slot.is_float) {
1432 if (samePackedIntKind(lhs_slot, rhs_slot, result_slot)) |int_kind| {
1433 try emitPackedIntMinMax(emitter, lhs_slot, rhs_slot, result_slot, kind, int_kind);
1434 return;
1435 }
1436 }
1437
1438 for (0..lhs_slot.lanes) |lane| {
1439 const lhs_lane = slot_layout.laneSlot(lhs_slot.base, lane);
1440 const rhs_lane = slot_layout.laneSlot(rhs_slot.base, lane);
1441 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1442
1443 if (result_slot.is_float) {
1444 try emitter.loadSlotXmm(lhs_lane, .xmm0);
1445 try emitter.loadSlotXmm(rhs_lane, .xmm1);
1446 if (lhs_lane.width == 32) {
1447 try scalar_backend.emitFloatMinMax(emitter, true, kind);
1448 } else if (lhs_lane.width == 64) {
1449 try scalar_backend.emitFloatMinMax(emitter, false, kind);
1450 } else {
1451 return error.UnsupportedType;
1452 }
1453 try emitter.storeSlotXmm(dst_lane, .xmm0);
1454 } else {
1455 const width = lhs_lane.width;
1456 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1457 const op_width: u8 = if (width <= 32) 32 else 64;
1458 const is_unsigned = scalar_backend.isUnsignedIntegerTypeName(elem_type_name);
1459 if (is_unsigned) {
1460 try emitter.loadSlotUnsigned(lhs_lane, .rax);
1461 try emitter.loadSlotUnsigned(rhs_lane, .rcx);
1462 } else {
1463 try emitter.loadSlot(lhs_lane, .rax);
1464 try emitter.loadSlot(rhs_lane, .rcx);
1465 }
1466 try scalar_backend.emitIntegerMinMax(emitter, op_width, is_unsigned, kind);
1467 try emitter.storeSlot(dst_lane, .rax);
1468 }
1469 }
1470 }
1471
1472 pub fn emitSelect(emitter: anytype, op: *ir.Operation) !void {
1473 if (op.operands.items.len != 3) return error.UnsupportedOperation;
1474 const cond = op.getOperand(0) orelse return error.MissingOperand;
1475 const true_val = op.getOperand(1) orelse return error.MissingOperand;
1476 const false_val = op.getOperand(2) orelse return error.MissingOperand;
1477 const result = op.getResult(0) orelse return error.MissingResult;
1478
1479 const cond_name = cond.type.getDialectTypeName() orelse return error.UnsupportedType;
1480 if (!std.mem.eql(u8, cond_name, dialects.arith.type_names.boolean)) return error.UnsupportedType;
1481 if (!result.type.eql(true_val.type)) return error.UnsupportedType;
1482 if (!result.type.eql(false_val.type)) return error.UnsupportedType;
1483
1484 const cond_slot = try emitter.slotFor(cond);
1485 const true_slot = try emitter.vectorSlotFor(true_val);
1486 const false_slot = try emitter.vectorSlotFor(false_val);
1487 const result_slot = try emitter.vectorSlotFor(result);
1488 if (!sameVectorStorageShape(true_slot, false_slot)) return error.UnsupportedType;
1489 if (!sameVectorStorageShape(true_slot, result_slot)) return error.UnsupportedType;
1490
1491 try emitCopyVectorSlot(emitter, true_slot, result_slot);
1492 try emitter.loadSlotUnsigned(cond_slot, .rax);
1493 try emitter.emitEncoding(encoding.cmpRegImm(.rax, 0));
1494
1495 var done_label = labels.Label{};
1496 defer done_label.deinit(emitter.allocator);
1497 try emitter.emitJccLabel(.ne, &done_label);
1498 try emitCopyVectorSlot(emitter, false_slot, result_slot);
1499 try emitter.bindLabel(&done_label);
1500 }
1501
1502 pub fn emitNeg(emitter: anytype, op: *ir.Operation) !void {
1503 const vec_op = ArithDialect.NegOp{ .op = op };
1504 const result = vec_op.getResult();
1505 const input = vec_op.getInput();
1506
1507 const in_slot = try emitter.vectorSlotFor(input);
1508 const out_slot = try emitter.vectorSlotFor(result);
1509 if (in_slot.lanes != out_slot.lanes) return error.VectorArityMismatch;
1510 if (in_slot.is_float != out_slot.is_float) return error.UnsupportedType;
1511
1512 if (packedFloatKind(in_slot)) |in_kind| {
1513 if (packedFloatKind(out_slot)) |out_kind| {
1514 if (in_kind == out_kind) {
1515 try emitPackedFloatNeg(emitter, in_slot, out_slot, in_kind);
1516 return;
1517 }
1518 }
1519 }
1520
1521 if (packedIntKind(in_slot)) |in_kind| {
1522 if (packedIntKind(out_slot)) |out_kind| {
1523 if (in_kind == out_kind and std.mem.eql(u8, in_slot.elem_type_name, out_slot.elem_type_name)) {
1524 try emitPackedIntNeg(emitter, in_slot, out_slot, in_kind);
1525 return;
1526 }
1527 }
1528 }
1529
1530 for (0..in_slot.lanes) |lane| {
1531 const src_lane = slot_layout.laneSlot(in_slot.base, lane);
1532 const dst_lane = slot_layout.laneSlot(out_slot.base, lane);
1533
1534 if (in_slot.is_float) {
1535 try emitter.emitEncoding(encoding.movRegImm64(.rax, 0));
1536 try emitter.storeSlot(dst_lane, .rax);
1537 try emitter.loadSlotXmm(dst_lane, .xmm0);
1538 try emitter.loadSlotXmm(src_lane, .xmm1);
1539 if (src_lane.width == 32) {
1540 try emitter.emitEncoding(encoding.subss(.xmm0, .{ .reg = .xmm1 }));
1541 } else if (src_lane.width == 64) {
1542 try emitter.emitEncoding(encoding.subsd(.xmm0, .{ .reg = .xmm1 }));
1543 } else {
1544 return error.UnsupportedType;
1545 }
1546 try emitter.storeSlotXmm(dst_lane, .xmm0);
1547 } else {
1548 const width = src_lane.width;
1549 const op_width: u8 = if (width <= 32) 32 else 64;
1550 try emitter.emitEncoding(encoding.movRegImm64(.rax, 0));
1551 try emitter.loadSlot(src_lane, .rcx);
1552 if (op_width == 32) {
1553 try emitter.emitEncoding(encoding.subRegReg32(.rax, .rcx));
1554 } else {
1555 try emitter.emitEncoding(encoding.subRegReg(.rax, .rcx));
1556 }
1557 try emitter.storeSlot(dst_lane, .rax);
1558 }
1559 }
1560 }
1561
1562 pub fn emitNot(emitter: anytype, op: *ir.Operation) !void {
1563 const vec_op = ArithDialect.NotOp{ .op = op };
1564 const result = vec_op.getResult();
1565 const input = vec_op.getInput();
1566
1567 const in_slot = try emitter.vectorSlotFor(input);
1568 const out_slot = try emitter.vectorSlotFor(result);
1569 if (in_slot.lanes != out_slot.lanes) return error.VectorArityMismatch;
1570 if (in_slot.is_float or out_slot.is_float) return error.UnsupportedType;
1571 if (!std.mem.eql(u8, in_slot.elem_type_name, out_slot.elem_type_name)) return error.UnsupportedType;
1572 if (in_slot.base.width != out_slot.base.width) return error.UnsupportedType;
1573 if (std.mem.eql(u8, out_slot.elem_type_name, dialects.arith.type_names.boolean)) return error.UnsupportedType;
1574
1575 if (packedIntKind(in_slot)) |in_kind| {
1576 if (packedIntKind(out_slot)) |out_kind| {
1577 if (in_kind == out_kind) {
1578 try emitPackedIntNot(emitter, in_slot, out_slot);
1579 return;
1580 }
1581 }
1582 }
1583
1584 for (0..in_slot.lanes) |lane| {
1585 const src_lane = slot_layout.laneSlot(in_slot.base, lane);
1586 const dst_lane = slot_layout.laneSlot(out_slot.base, lane);
1587
1588 const width = src_lane.width;
1589 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1590 const op_width: u8 = if (width <= 32) 32 else 64;
1591
1592 try emitter.loadSlot(src_lane, .rax);
1593 if (op_width == 32) {
1594 try emitter.emitEncoding(encoding.notReg32(.rax));
1595 } else {
1596 try emitter.emitEncoding(encoding.notReg(.rax));
1597 }
1598 try emitter.storeSlot(dst_lane, .rax);
1599 }
1600 }
1601
1602 pub fn emitCmp(emitter: anytype, op: *ir.Operation) !void {
1603 const vec_cmp = ArithDialect.VecCmpOp{ .op = op };
1604 const result = vec_cmp.getResult();
1605 const pred = vec_cmp.getPredicate() orelse return error.MissingPredicate;
1606
1607 const lhs = op.getOperand(0) orelse return error.MissingOperand;
1608 const rhs = op.getOperand(1) orelse return error.MissingOperand;
1609
1610 const lhs_slot = try emitter.vectorSlotFor(lhs);
1611 const rhs_slot = try emitter.vectorSlotFor(rhs);
1612 const result_slot = try emitter.vectorSlotFor(result);
1613
1614 if (lhs_slot.lanes != rhs_slot.lanes or lhs_slot.lanes != result_slot.lanes) return error.VectorArityMismatch;
1615
1616 if (samePackedFloatCmpKind(lhs_slot, rhs_slot, result_slot)) |float_kind| {
1617 try emitPackedFloatCmp(emitter, lhs_slot, rhs_slot, result_slot, pred, float_kind);
1618 return;
1619 }
1620
1621 if (samePackedIntCmpKind(lhs_slot, rhs_slot, result_slot)) |int_kind| {
1622 if (packedIntCmpSupported(pred, int_kind)) {
1623 try emitPackedIntCmp(emitter, lhs_slot, rhs_slot, result_slot, pred, int_kind);
1624 return;
1625 }
1626 }
1627
1628 for (0..lhs_slot.lanes) |lane| {
1629 const lhs_lane = slot_layout.laneSlot(lhs_slot.base, lane);
1630 const rhs_lane = slot_layout.laneSlot(rhs_slot.base, lane);
1631 const dst_lane = slot_layout.laneSlot(result_slot.base, lane);
1632
1633 if (lhs_slot.is_float) {
1634 try emitter.loadSlotXmm(lhs_lane, .xmm0);
1635 try emitter.loadSlotXmm(rhs_lane, .xmm1);
1636 if (lhs_lane.width == 32) {
1637 try emitter.emitEncoding(encoding.ucomiss(.xmm0, .{ .reg = .xmm1 }));
1638 } else if (lhs_lane.width == 64) {
1639 try emitter.emitEncoding(encoding.ucomisd(.xmm0, .{ .reg = .xmm1 }));
1640 } else {
1641 return error.UnsupportedType;
1642 }
1643
1644 switch (pred) {
1645 .eq => {
1646 try emitter.emitEncoding(encoding.setcc(.rax, .e));
1647 try emitter.emitEncoding(encoding.setcc(.rcx, .np));
1648 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1649 try emitter.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1650 try emitter.emitEncoding(encoding.andRegReg(.rax, .rcx));
1651 },
1652 .ne => {
1653 try emitter.emitEncoding(encoding.setcc(.rax, .ne));
1654 try emitter.emitEncoding(encoding.setcc(.rcx, .p));
1655 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1656 try emitter.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1657 try emitter.emitEncoding(encoding.orRegReg(.rax, .rcx));
1658 },
1659 .lt, .slt, .ult => {
1660 try emitter.emitEncoding(encoding.setcc(.rax, .b));
1661 try emitter.emitEncoding(encoding.setcc(.rcx, .np));
1662 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1663 try emitter.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1664 try emitter.emitEncoding(encoding.andRegReg(.rax, .rcx));
1665 },
1666 .le, .sle, .ule => {
1667 try emitter.emitEncoding(encoding.setcc(.rax, .be));
1668 try emitter.emitEncoding(encoding.setcc(.rcx, .np));
1669 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1670 try emitter.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1671 try emitter.emitEncoding(encoding.andRegReg(.rax, .rcx));
1672 },
1673 .gt, .sgt, .ugt => {
1674 try emitter.emitEncoding(encoding.setcc(.rax, .a));
1675 try emitter.emitEncoding(encoding.setcc(.rcx, .np));
1676 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1677 try emitter.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1678 try emitter.emitEncoding(encoding.andRegReg(.rax, .rcx));
1679 },
1680 .ge, .sge, .uge => {
1681 try emitter.emitEncoding(encoding.setcc(.rax, .ae));
1682 try emitter.emitEncoding(encoding.setcc(.rcx, .np));
1683 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1684 try emitter.emitEncoding(encoding.movzxReg8(.rcx, .rcx));
1685 try emitter.emitEncoding(encoding.andRegReg(.rax, .rcx));
1686 },
1687 }
1688 try emitter.storeSlot(dst_lane, .rax);
1689 } else {
1690 const width = lhs_lane.width;
1691 if (width != 8 and width != 16 and width != 32 and width != 64) return error.UnsupportedType;
1692
1693 const is_unsigned = switch (pred) {
1694 .ult, .ule, .ugt, .uge => true,
1695 else => false,
1696 };
1697
1698 if (is_unsigned) {
1699 try emitter.loadSlotUnsigned(lhs_lane, .rax);
1700 try emitter.loadSlotUnsigned(rhs_lane, .rcx);
1701 } else {
1702 try emitter.loadSlot(lhs_lane, .rax);
1703 try emitter.loadSlot(rhs_lane, .rcx);
1704 }
1705
1706 if (width == 64) {
1707 try emitter.emitEncoding(encoding.cmpRegReg(.rax, .rcx));
1708 } else {
1709 try emitter.emitEncoding(encoding.cmpRegReg32(.rax, .rcx));
1710 }
1711
1712 const cond = predicateToConditionInt(pred);
1713 try emitter.emitEncoding(encoding.setcc(.rax, cond));
1714 try emitter.emitEncoding(encoding.movzxReg8(.rax, .rax));
1715 try emitter.storeSlot(dst_lane, .rax);
1716 }
1717 }
1718 }
1719
1720 fn predicateToConditionInt(pred: CmpPredicate) encoding.Condition {
1721 return switch (pred) {
1722 .eq => .e,
1723 .ne => .ne,
1724 .lt, .slt => .l,
1725 .le, .sle => .le,
1726 .gt, .sgt => .g,
1727 .ge, .sge => .ge,
1728 .ult => .b,
1729 .ule => .be,
1730 .ugt => .a,
1731 .uge => .ae,
1732 };
1733 }
1734
1735 const ConstantRecorder = struct {
1736 allocator: ?std.mem.Allocator = null,
1737 vector_value: *ir.Value,
1738 vector_slot: slot_layout.VectorSlot,
1739 bytes: std.ArrayListUnmanaged(u8) = .empty,
1740 encodings: usize = 0,
1741 stores: usize = 0,
1742
1743 pub fn vectorSlotFor(self: *ConstantRecorder, value: *ir.Value) !slot_layout.VectorSlot {
1744 if (value != self.vector_value) return error.MissingSlot;
1745 return self.vector_slot;
1746 }
1747
1748 pub fn emitEncoding(self: *ConstantRecorder, enc: encoding.Encoding) !void {
1749 if (self.allocator) |allocator| {
1750 try self.bytes.appendSlice(allocator, enc.slice());
1751 }
1752 self.encodings += 1;
1753 }
1754
1755 pub fn storeSlot(self: *ConstantRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1756 _ = slot;
1757 _ = reg;
1758 self.stores += 1;
1759 }
1760
1761 pub fn storeFromXmmPacked(self: *ConstantRecorder, value: *ir.Value, reg: registers.XMM) !void {
1762 try emitPackedStore(self, try self.vectorSlotFor(value), reg);
1763 }
1764 };
1765
1766 const SplatRecorder = struct {
1767 allocator: std.mem.Allocator,
1768 scalar_value: *ir.Value,
1769 scalar_slot: slot_layout.Slot,
1770 result_value: *ir.Value,
1771 result_slot: slot_layout.VectorSlot,
1772 bytes: std.ArrayListUnmanaged(u8) = .empty,
1773 xmm_loads: usize = 0,
1774 scalar_loads: usize = 0,
1775 scalar_stores: usize = 0,
1776
1777 pub fn slotFor(self: *SplatRecorder, value: *ir.Value) !slot_layout.Slot {
1778 if (value == self.scalar_value) return self.scalar_slot;
1779 return error.MissingSlot;
1780 }
1781
1782 pub fn vectorSlotFor(self: *SplatRecorder, value: *ir.Value) !slot_layout.VectorSlot {
1783 if (value == self.result_value) return self.result_slot;
1784 return error.MissingSlot;
1785 }
1786
1787 pub fn emitEncoding(self: *SplatRecorder, enc: encoding.Encoding) !void {
1788 try self.bytes.appendSlice(self.allocator, enc.slice());
1789 }
1790
1791 pub fn loadSlotXmm(self: *SplatRecorder, slot: slot_layout.Slot, reg: registers.XMM) !void {
1792 self.xmm_loads += 1;
1793 const mem = encoding.Mem.baseDisp(.rbp, slot.offset);
1794 if (slot.width == 32) {
1795 try self.emitEncoding(encoding.movss(reg, .{ .mem = mem }));
1796 } else if (slot.width == 64) {
1797 try self.emitEncoding(encoding.movsd(reg, .{ .mem = mem }));
1798 } else {
1799 return error.UnsupportedType;
1800 }
1801 }
1802
1803 pub fn loadSlot(self: *SplatRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1804 _ = slot;
1805 _ = reg;
1806 self.scalar_loads += 1;
1807 }
1808
1809 pub fn storeSlot(self: *SplatRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1810 _ = slot;
1811 _ = reg;
1812 self.scalar_stores += 1;
1813 }
1814
1815 pub fn loadIntoXmm(self: *SplatRecorder, value: *ir.Value, reg: registers.XMM) !void {
1816 try self.loadSlotXmm(try self.slotFor(value), reg);
1817 }
1818
1819 pub fn registerHome(self: *SplatRecorder, value: *ir.Value) ?registers.GPR {
1820 _ = self;
1821 _ = value;
1822 return null;
1823 }
1824
1825 pub fn storeFromXmmPacked(self: *SplatRecorder, value: *ir.Value, reg: registers.XMM) !void {
1826 try emitPackedStore(self, try self.vectorSlotFor(value), reg);
1827 }
1828 };
1829
1830 const BinaryRecorder = struct {
1831 allocator: std.mem.Allocator,
1832 lhs_value: *ir.Value,
1833 lhs_slot: slot_layout.VectorSlot,
1834 rhs_value: *ir.Value,
1835 rhs_slot: slot_layout.VectorSlot,
1836 result_value: *ir.Value,
1837 result_slot: slot_layout.VectorSlot,
1838 bytes: std.ArrayListUnmanaged(u8) = .empty,
1839 scalar_loads: usize = 0,
1840 scalar_stores: usize = 0,
1841
1842 pub fn vectorSlotFor(self: *BinaryRecorder, value: *ir.Value) !slot_layout.VectorSlot {
1843 if (value == self.lhs_value) return self.lhs_slot;
1844 if (value == self.rhs_value) return self.rhs_slot;
1845 if (value == self.result_value) return self.result_slot;
1846 return error.MissingSlot;
1847 }
1848
1849 pub fn emitEncoding(self: *BinaryRecorder, enc: encoding.Encoding) !void {
1850 try self.bytes.appendSlice(self.allocator, enc.slice());
1851 }
1852
1853 pub fn loadSlotXmm(self: *BinaryRecorder, slot: slot_layout.Slot, reg: registers.XMM) !void {
1854 _ = slot;
1855 _ = reg;
1856 self.scalar_loads += 1;
1857 }
1858
1859 pub fn storeSlotXmm(self: *BinaryRecorder, slot: slot_layout.Slot, reg: registers.XMM) !void {
1860 _ = slot;
1861 _ = reg;
1862 self.scalar_stores += 1;
1863 }
1864
1865 pub fn loadSlot(self: *BinaryRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1866 _ = slot;
1867 _ = reg;
1868 self.scalar_loads += 1;
1869 }
1870
1871 pub fn loadSlotUnsigned(self: *BinaryRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1872 _ = slot;
1873 _ = reg;
1874 self.scalar_loads += 1;
1875 }
1876
1877 pub fn storeSlot(self: *BinaryRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1878 _ = slot;
1879 _ = reg;
1880 self.scalar_stores += 1;
1881 }
1882
1883 pub fn emitJccLabel(_: *BinaryRecorder, _: encoding.Condition, _: *labels.Label) !void {}
1884
1885 pub fn bindLabel(_: *BinaryRecorder, _: *labels.Label) !void {}
1886
1887 pub fn loadIntoXmmPacked(self: *BinaryRecorder, value: *ir.Value, reg: registers.XMM) !void {
1888 try emitPackedLoad(self, try self.vectorSlotFor(value), reg);
1889 }
1890
1891 pub fn storeFromXmmPacked(self: *BinaryRecorder, value: *ir.Value, reg: registers.XMM) !void {
1892 try emitPackedStore(self, try self.vectorSlotFor(value), reg);
1893 }
1894 };
1895
1896 const LaneRecorder = struct {
1897 allocator: std.mem.Allocator,
1898 vector_value: *ir.Value,
1899 vector_slot: slot_layout.VectorSlot,
1900 slot_value: *ir.Value,
1901 slot: slot_layout.Slot,
1902 result_vector_value: ?*ir.Value = null,
1903 result_vector_slot: slot_layout.VectorSlot = undefined,
1904 bytes: std.ArrayListUnmanaged(u8) = .empty,
1905 scalar_loads: usize = 0,
1906 scalar_stores: usize = 0,
1907
1908 pub fn vectorSlotFor(self: *LaneRecorder, value: *ir.Value) !slot_layout.VectorSlot {
1909 if (value == self.vector_value) return self.vector_slot;
1910 if (self.result_vector_value) |result_value| {
1911 if (value == result_value) return self.result_vector_slot;
1912 }
1913 return error.MissingSlot;
1914 }
1915
1916 pub fn slotFor(self: *LaneRecorder, value: *ir.Value) !slot_layout.Slot {
1917 if (value == self.slot_value) return self.slot;
1918 return error.MissingSlot;
1919 }
1920
1921 pub fn emitEncoding(self: *LaneRecorder, enc: encoding.Encoding) !void {
1922 try self.bytes.appendSlice(self.allocator, enc.slice());
1923 }
1924
1925 pub fn loadSlot(self: *LaneRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1926 _ = slot;
1927 _ = reg;
1928 self.scalar_loads += 1;
1929 }
1930
1931 pub fn storeSlot(self: *LaneRecorder, slot: slot_layout.Slot, reg: registers.GPR) !void {
1932 _ = slot;
1933 _ = reg;
1934 self.scalar_stores += 1;
1935 }
1936 };
1937
1938 fn f64x2Slot(offset: i32) slot_layout.VectorSlot {
1939 return .{
1940 .base = .{ .offset = offset, .width = 64, .ext = .unsigned },
1941 .lanes = 2,
1942 .elem_type_name = dialects.arith.type_names.float64,
1943 .is_float = true,
1944 };
1945 }
1946
1947 fn f32x4Slot(offset: i32) slot_layout.VectorSlot {
1948 return .{
1949 .base = .{ .offset = offset, .width = 32, .ext = .unsigned },
1950 .lanes = 4,
1951 .elem_type_name = dialects.arith.type_names.float32,
1952 .is_float = true,
1953 };
1954 }
1955
1956 fn i32x4Slot(offset: i32) slot_layout.VectorSlot {
1957 return .{
1958 .base = .{ .offset = offset, .width = 32, .ext = .signed },
1959 .lanes = 4,
1960 .elem_type_name = dialects.arith.type_names.int32,
1961 .is_float = false,
1962 };
1963 }
1964
1965 fn u32x4Slot(offset: i32) slot_layout.VectorSlot {
1966 return .{
1967 .base = .{ .offset = offset, .width = 32, .ext = .unsigned },
1968 .lanes = 4,
1969 .elem_type_name = dialects.arith.type_names.uint32,
1970 .is_float = false,
1971 };
1972 }
1973
1974 fn i64x2Slot(offset: i32) slot_layout.VectorSlot {
1975 return .{
1976 .base = .{ .offset = offset, .width = 64, .ext = .signed },
1977 .lanes = 2,
1978 .elem_type_name = dialects.arith.type_names.int64,
1979 .is_float = false,
1980 };
1981 }
1982
1983 fn u64x2Slot(offset: i32) slot_layout.VectorSlot {
1984 return .{
1985 .base = .{ .offset = offset, .width = 64, .ext = .unsigned },
1986 .lanes = 2,
1987 .elem_type_name = dialects.arith.type_names.uint64,
1988 .is_float = false,
1989 };
1990 }
1991
1992 test "x86_64 vector owner emits lane constants" {
1993 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
1994 defer ctx.deinit(std.testing.allocator);
1995 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
1996
1997 const loc = ir.Location.getUnknown();
1998 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int32)).?;
1999 var constant = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
2000
2001 var recorder = ConstantRecorder{
2002 .vector_value = constant.getResult(),
2003 .vector_slot = .{
2004 .base = .{ .offset = -8, .width = 32, .ext = .signed },
2005 .lanes = 2,
2006 .elem_type_name = dialects.arith.type_names.int32,
2007 .is_float = false,
2008 },
2009 };
2010
2011 try emitConstant(&recorder, constant.op);
2012
2013 try std.testing.expectEqual(@as(usize, 2), recorder.encodings);
2014 try std.testing.expectEqual(@as(usize, 2), recorder.stores);
2015 }
2016
2017 test "x86_64 vector constant i32x4 emits packed SSE2" {
2018 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2019 defer ctx.deinit(std.testing.allocator);
2020 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2021
2022 const loc = ir.Location.getUnknown();
2023 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.int32)).?;
2024 var constant = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -7);
2025
2026 var recorder = ConstantRecorder{
2027 .allocator = std.testing.allocator,
2028 .vector_value = constant.getResult(),
2029 .vector_slot = i32x4Slot(-16),
2030 };
2031 defer recorder.bytes.deinit(std.testing.allocator);
2032
2033 try emitConstant(&recorder, constant.op);
2034
2035 try std.testing.expectEqual(@as(usize, 4), recorder.encodings);
2036 try std.testing.expectEqual(@as(usize, 0), recorder.stores);
2037 try std.testing.expectEqualSlices(u8, &.{
2038 0xB8, 0xF9, 0xFF, 0xFF, 0xFF,
2039 0x66, 0x0F, 0x6E, 0xC0, 0x66,
2040 0x0F, 0x70, 0xC0, 0x00, 0x0F,
2041 0x11, 0x45, 0xE4,
2042 }, recorder.bytes.items);
2043 }
2044
2045 test "x86_64 vector constant f32x4 emits packed SSE" {
2046 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2047 defer ctx.deinit(std.testing.allocator);
2048 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2049
2050 const loc = ir.Location.getUnknown();
2051 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
2052 var constant = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
2053
2054 var recorder = ConstantRecorder{
2055 .allocator = std.testing.allocator,
2056 .vector_value = constant.getResult(),
2057 .vector_slot = f32x4Slot(-16),
2058 };
2059 defer recorder.bytes.deinit(std.testing.allocator);
2060
2061 try emitConstant(&recorder, constant.op);
2062
2063 try std.testing.expectEqual(@as(usize, 4), recorder.encodings);
2064 try std.testing.expectEqual(@as(usize, 0), recorder.stores);
2065 try std.testing.expectEqualSlices(u8, &.{
2066 0xB8, 0x00, 0x00, 0xC0, 0x3F,
2067 0x66, 0x0F, 0x6E, 0xC0, 0x0F,
2068 0xC6, 0xC0, 0x00, 0x0F, 0x11,
2069 0x45, 0xE4,
2070 }, recorder.bytes.items);
2071 }
2072
2073 test "x86_64 vector constant f64x2 emits packed SSE2" {
2074 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2075 defer ctx.deinit(std.testing.allocator);
2076 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2077
2078 const loc = ir.Location.getUnknown();
2079 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
2080 var constant = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
2081
2082 var recorder = ConstantRecorder{
2083 .allocator = std.testing.allocator,
2084 .vector_value = constant.getResult(),
2085 .vector_slot = f64x2Slot(-16),
2086 };
2087 defer recorder.bytes.deinit(std.testing.allocator);
2088
2089 try emitConstant(&recorder, constant.op);
2090
2091 try std.testing.expectEqual(@as(usize, 4), recorder.encodings);
2092 try std.testing.expectEqual(@as(usize, 0), recorder.stores);
2093 try std.testing.expectEqualSlices(u8, &.{
2094 0x48, 0xB8, 0x00, 0x00, 0x00,
2095 0x00, 0x00, 0x00, 0xF8, 0x3F,
2096 0x66, 0x48, 0x0F, 0x6E, 0xC0,
2097 0x66, 0x0F, 0xC6, 0xC0, 0x00,
2098 0x66, 0x0F, 0x11, 0x45, 0xE8,
2099 }, recorder.bytes.items);
2100 }
2101
2102 test "x86_64 vector constant i64x2 emits packed SSE2" {
2103 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2104 defer ctx.deinit(std.testing.allocator);
2105 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2106
2107 const loc = ir.Location.getUnknown();
2108 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
2109 var constant = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -7);
2110
2111 var recorder = ConstantRecorder{
2112 .allocator = std.testing.allocator,
2113 .vector_value = constant.getResult(),
2114 .vector_slot = i64x2Slot(-16),
2115 };
2116 defer recorder.bytes.deinit(std.testing.allocator);
2117
2118 try emitConstant(&recorder, constant.op);
2119
2120 try std.testing.expectEqual(@as(usize, 4), recorder.encodings);
2121 try std.testing.expectEqual(@as(usize, 0), recorder.stores);
2122 try std.testing.expectEqualSlices(u8, &.{
2123 0x48, 0xB8, 0xF9, 0xFF, 0xFF,
2124 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
2125 0x66, 0x48, 0x0F, 0x6E, 0xC0,
2126 0x66, 0x0F, 0xC6, 0xC0, 0x00,
2127 0x66, 0x0F, 0x11, 0x45, 0xE8,
2128 }, recorder.bytes.items);
2129 }
2130
2131 test "x86_64 vector splat f64x2 emits packed SSE" {
2132 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2133 defer ctx.deinit(std.testing.allocator);
2134 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2135
2136 const loc = ir.Location.getUnknown();
2137 const f64_type = try ArithDialect.getScalarType(&ctx, .f64);
2138 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
2139 var scalar = try ArithDialect.ConstantOp.createFloat(&ctx, loc, f64_type, 1.5);
2140 var splat = try ArithDialect.SplatOp.create(&ctx, loc, scalar.getResult(), vec_type);
2141
2142 var recorder = SplatRecorder{
2143 .allocator = std.testing.allocator,
2144 .scalar_value = scalar.getResult(),
2145 .scalar_slot = .{ .offset = -8, .width = 64, .ext = .unsigned },
2146 .result_value = splat.getResult(),
2147 .result_slot = f64x2Slot(-16),
2148 };
2149 defer recorder.bytes.deinit(std.testing.allocator);
2150
2151 try emitSplat(&recorder, splat.op);
2152
2153 try std.testing.expectEqual(@as(usize, 1), recorder.xmm_loads);
2154 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2155 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2156 try std.testing.expectEqualSlices(u8, &.{
2157 0xF2, 0x0F, 0x10, 0x45, 0xF8,
2158 0x66, 0x0F, 0xC6, 0xC0, 0x00,
2159 0x66, 0x0F, 0x11, 0x45, 0xE8,
2160 }, recorder.bytes.items);
2161 }
2162
2163 test "x86_64 vector splat f32x4 emits packed SSE" {
2164 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2165 defer ctx.deinit(std.testing.allocator);
2166 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2167
2168 const loc = ir.Location.getUnknown();
2169 const f32_type = try ArithDialect.getScalarType(&ctx, .f32);
2170 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
2171 var scalar = try ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 1.5);
2172 var splat = try ArithDialect.SplatOp.create(&ctx, loc, scalar.getResult(), vec_type);
2173
2174 var recorder = SplatRecorder{
2175 .allocator = std.testing.allocator,
2176 .scalar_value = scalar.getResult(),
2177 .scalar_slot = .{ .offset = -8, .width = 32, .ext = .unsigned },
2178 .result_value = splat.getResult(),
2179 .result_slot = f32x4Slot(-16),
2180 };
2181 defer recorder.bytes.deinit(std.testing.allocator);
2182
2183 try emitSplat(&recorder, splat.op);
2184
2185 try std.testing.expectEqual(@as(usize, 1), recorder.xmm_loads);
2186 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2187 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2188 try std.testing.expectEqualSlices(u8, &.{
2189 0xF3, 0x0F, 0x10, 0x45, 0xF8,
2190 0x0F, 0xC6, 0xC0, 0x00, 0x0F,
2191 0x11, 0x45, 0xE4,
2192 }, recorder.bytes.items);
2193 }
2194
2195 test "x86_64 vector splat u32x4 emits packed SSE2" {
2196 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2197 defer ctx.deinit(std.testing.allocator);
2198 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2199
2200 const loc = ir.Location.getUnknown();
2201 const u32_type = try ArithDialect.getScalarType(&ctx, .u32);
2202 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2203 var scalar = try ArithDialect.ConstantOp.createInt(&ctx, loc, u32_type, 7);
2204 var splat = try ArithDialect.SplatOp.create(&ctx, loc, scalar.getResult(), vec_type);
2205
2206 var recorder = SplatRecorder{
2207 .allocator = std.testing.allocator,
2208 .scalar_value = scalar.getResult(),
2209 .scalar_slot = .{ .offset = -8, .width = 32, .ext = .unsigned },
2210 .result_value = splat.getResult(),
2211 .result_slot = u32x4Slot(-16),
2212 };
2213 defer recorder.bytes.deinit(std.testing.allocator);
2214
2215 try emitSplat(&recorder, splat.op);
2216
2217 try std.testing.expectEqual(@as(usize, 0), recorder.xmm_loads);
2218 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2219 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2220 try std.testing.expectEqualSlices(u8, &.{
2221 0x66, 0x0F, 0x6E, 0x45, 0xF8,
2222 0x66, 0x0F, 0x70, 0xC0, 0x00,
2223 0x0F, 0x11, 0x45, 0xE4,
2224 }, recorder.bytes.items);
2225 }
2226
2227 test "x86_64 vector splat i64x2 emits packed SSE2" {
2228 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2229 defer ctx.deinit(std.testing.allocator);
2230 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2231
2232 const loc = ir.Location.getUnknown();
2233 const i64_type = try ArithDialect.getScalarType(&ctx, .i64);
2234 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
2235 var scalar = try ArithDialect.ConstantOp.createInt(&ctx, loc, i64_type, 7);
2236 var splat = try ArithDialect.SplatOp.create(&ctx, loc, scalar.getResult(), vec_type);
2237
2238 var recorder = SplatRecorder{
2239 .allocator = std.testing.allocator,
2240 .scalar_value = scalar.getResult(),
2241 .scalar_slot = .{ .offset = -8, .width = 64, .ext = .signed },
2242 .result_value = splat.getResult(),
2243 .result_slot = i64x2Slot(-16),
2244 };
2245 defer recorder.bytes.deinit(std.testing.allocator);
2246
2247 try emitSplat(&recorder, splat.op);
2248
2249 try std.testing.expectEqual(@as(usize, 0), recorder.xmm_loads);
2250 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2251 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2252 try std.testing.expectEqualSlices(u8, &.{
2253 0x66, 0x48, 0x0F, 0x6E, 0x45, 0xF8,
2254 0x66, 0x0F, 0xC6, 0xC0, 0x00, 0x66,
2255 0x0F, 0x11, 0x45, 0xE8,
2256 }, recorder.bytes.items);
2257 }
2258
2259 test "x86_64 vector extract u32x4 emits packed SSE2" {
2260 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2261 defer ctx.deinit(std.testing.allocator);
2262 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2263
2264 const loc = ir.Location.getUnknown();
2265 const scalar_type = try ArithDialect.getScalarType(&ctx, .u32);
2266 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2267 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
2268 const extract = try ArithDialect.ExtractOp.create(&ctx, loc, input.getResult(), 1, scalar_type);
2269
2270 var recorder = LaneRecorder{
2271 .allocator = std.testing.allocator,
2272 .vector_value = input.getResult(),
2273 .vector_slot = u32x4Slot(-8),
2274 .slot_value = extract.getResult(),
2275 .slot = .{ .offset = -40, .width = 32, .ext = .unsigned },
2276 };
2277 defer recorder.bytes.deinit(std.testing.allocator);
2278
2279 try emitExtract(&recorder, extract.op);
2280
2281 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2282 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2283 try std.testing.expectEqualSlices(u8, &.{
2284 0x0F, 0x10, 0x45, 0xEC,
2285 0x66, 0x0F, 0x70, 0xC0,
2286 0x02, 0x66, 0x0F, 0x7E,
2287 0x45, 0xD8,
2288 }, recorder.bytes.items);
2289 }
2290
2291 test "x86_64 vector extract f64x2 emits packed SSE2" {
2292 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2293 defer ctx.deinit(std.testing.allocator);
2294 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2295
2296 const loc = ir.Location.getUnknown();
2297 const scalar_type = try ArithDialect.getScalarType(&ctx, .f64);
2298 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
2299 var input = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 7.0);
2300 const extract = try ArithDialect.ExtractOp.create(&ctx, loc, input.getResult(), 0, scalar_type);
2301
2302 var recorder = LaneRecorder{
2303 .allocator = std.testing.allocator,
2304 .vector_value = input.getResult(),
2305 .vector_slot = f64x2Slot(-8),
2306 .slot_value = extract.getResult(),
2307 .slot = .{ .offset = -32, .width = 64, .ext = .unsigned },
2308 };
2309 defer recorder.bytes.deinit(std.testing.allocator);
2310
2311 try emitExtract(&recorder, extract.op);
2312
2313 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2314 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2315 try std.testing.expectEqualSlices(u8, &.{
2316 0x66, 0x0F, 0x10, 0x45, 0xF0,
2317 0x66, 0x0F, 0xC6, 0xC0, 0x01,
2318 0xF2, 0x0F, 0x11, 0x45, 0xE0,
2319 }, recorder.bytes.items);
2320 }
2321
2322 test "x86_64 vector insert u32x4 emits packed copy and lane store" {
2323 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2324 defer ctx.deinit(std.testing.allocator);
2325 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2326
2327 const loc = ir.Location.getUnknown();
2328 const scalar_type = try ArithDialect.getScalarType(&ctx, .u32);
2329 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2330 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
2331 var scalar = try ArithDialect.ConstantOp.createInt(&ctx, loc, scalar_type, 9);
2332 const insert = try ArithDialect.InsertOp.create(&ctx, loc, input.getResult(), scalar.getResult(), 2);
2333
2334 var recorder = LaneRecorder{
2335 .allocator = std.testing.allocator,
2336 .vector_value = input.getResult(),
2337 .vector_slot = u32x4Slot(-8),
2338 .slot_value = scalar.getResult(),
2339 .slot = .{ .offset = -64, .width = 32, .ext = .unsigned },
2340 .result_vector_value = insert.getResult(),
2341 .result_vector_slot = u32x4Slot(-40),
2342 };
2343 defer recorder.bytes.deinit(std.testing.allocator);
2344
2345 try emitInsert(&recorder, insert.op);
2346
2347 try std.testing.expectEqual(@as(usize, 1), recorder.scalar_loads);
2348 try std.testing.expectEqual(@as(usize, 1), recorder.scalar_stores);
2349 try std.testing.expectEqualSlices(u8, &.{
2350 0x0F, 0x10, 0x45, 0xEC,
2351 0x0F, 0x11, 0x45, 0xCC,
2352 }, recorder.bytes.items);
2353 }
2354
2355 test "x86_64 vector insert i64x2 emits packed copy and lane store" {
2356 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2357 defer ctx.deinit(std.testing.allocator);
2358 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2359
2360 const loc = ir.Location.getUnknown();
2361 const scalar_type = try ArithDialect.getScalarType(&ctx, .i64);
2362 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
2363 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
2364 var scalar = try ArithDialect.ConstantOp.createInt(&ctx, loc, scalar_type, 9);
2365 const insert = try ArithDialect.InsertOp.create(&ctx, loc, input.getResult(), scalar.getResult(), 0);
2366
2367 var recorder = LaneRecorder{
2368 .allocator = std.testing.allocator,
2369 .vector_value = input.getResult(),
2370 .vector_slot = i64x2Slot(-8),
2371 .slot_value = scalar.getResult(),
2372 .slot = .{ .offset = -40, .width = 64, .ext = .signed },
2373 .result_vector_value = insert.getResult(),
2374 .result_vector_slot = i64x2Slot(-24),
2375 };
2376 defer recorder.bytes.deinit(std.testing.allocator);
2377
2378 try emitInsert(&recorder, insert.op);
2379
2380 try std.testing.expectEqual(@as(usize, 1), recorder.scalar_loads);
2381 try std.testing.expectEqual(@as(usize, 1), recorder.scalar_stores);
2382 try std.testing.expectEqualSlices(u8, &.{
2383 0x66, 0x0F, 0x10, 0x45, 0xF0,
2384 0x66, 0x0F, 0x11, 0x45, 0xE0,
2385 }, recorder.bytes.items);
2386 }
2387
2388 test "x86_64 vector binary f64x2 emits packed SSE" {
2389 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2390 defer ctx.deinit(std.testing.allocator);
2391 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2392
2393 const loc = ir.Location.getUnknown();
2394 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
2395 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
2396 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.25);
2397 const add = try ArithDialect.AddOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2398
2399 var recorder = BinaryRecorder{
2400 .allocator = std.testing.allocator,
2401 .lhs_value = lhs.getResult(),
2402 .lhs_slot = f64x2Slot(-8),
2403 .rhs_value = rhs.getResult(),
2404 .rhs_slot = f64x2Slot(-24),
2405 .result_value = add.getResult(),
2406 .result_slot = f64x2Slot(-40),
2407 };
2408 defer recorder.bytes.deinit(std.testing.allocator);
2409
2410 try emitBinaryOp(&recorder, add.op, .add);
2411
2412 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2413 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2414 try std.testing.expectEqualSlices(u8, &.{
2415 0x66, 0x0F, 0x10, 0x45, 0xF0,
2416 0x66, 0x0F, 0x10, 0x4D, 0xE0,
2417 0x66, 0x0F, 0x58, 0xC1, 0x66,
2418 0x0F, 0x11, 0x45, 0xD0,
2419 }, recorder.bytes.items);
2420 }
2421
2422 test "x86_64 vector binary f64x2 div emits packed SSE2" {
2423 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2424 defer ctx.deinit(std.testing.allocator);
2425 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2426
2427 const loc = ir.Location.getUnknown();
2428 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
2429 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 9.0);
2430 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 3.0);
2431 const div = try ArithDialect.DivOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2432
2433 var recorder = BinaryRecorder{
2434 .allocator = std.testing.allocator,
2435 .lhs_value = lhs.getResult(),
2436 .lhs_slot = f64x2Slot(-8),
2437 .rhs_value = rhs.getResult(),
2438 .rhs_slot = f64x2Slot(-24),
2439 .result_value = div.getResult(),
2440 .result_slot = f64x2Slot(-40),
2441 };
2442 defer recorder.bytes.deinit(std.testing.allocator);
2443
2444 try emitBinaryOp(&recorder, div.op, .div);
2445
2446 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2447 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2448 try std.testing.expectEqualSlices(u8, &.{
2449 0x66, 0x0F, 0x10, 0x45, 0xF0,
2450 0x66, 0x0F, 0x10, 0x4D, 0xE0,
2451 0x66, 0x0F, 0x5E, 0xC1, 0x66,
2452 0x0F, 0x11, 0x45, 0xD0,
2453 }, recorder.bytes.items);
2454 }
2455
2456 test "x86_64 vector binary f64x2 mul emits packed SSE2" {
2457 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2458 defer ctx.deinit(std.testing.allocator);
2459 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2460
2461 const loc = ir.Location.getUnknown();
2462 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
2463 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
2464 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.0);
2465 const mul = try ArithDialect.MulOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2466
2467 var recorder = BinaryRecorder{
2468 .allocator = std.testing.allocator,
2469 .lhs_value = lhs.getResult(),
2470 .lhs_slot = f64x2Slot(-8),
2471 .rhs_value = rhs.getResult(),
2472 .rhs_slot = f64x2Slot(-24),
2473 .result_value = mul.getResult(),
2474 .result_slot = f64x2Slot(-40),
2475 };
2476 defer recorder.bytes.deinit(std.testing.allocator);
2477
2478 try emitBinaryOp(&recorder, mul.op, .mul);
2479
2480 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2481 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2482 try std.testing.expectEqualSlices(u8, &.{
2483 0x66, 0x0F, 0x10, 0x45, 0xF0,
2484 0x66, 0x0F, 0x10, 0x4D, 0xE0,
2485 0x66, 0x0F, 0x59, 0xC1, 0x66,
2486 0x0F, 0x11, 0x45, 0xD0,
2487 }, recorder.bytes.items);
2488 }
2489
2490 test "x86_64 vector binary f32x4 emits packed SSE" {
2491 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2492 defer ctx.deinit(std.testing.allocator);
2493 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2494
2495 const loc = ir.Location.getUnknown();
2496 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
2497 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
2498 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.25);
2499 const add = try ArithDialect.AddOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2500
2501 var recorder = BinaryRecorder{
2502 .allocator = std.testing.allocator,
2503 .lhs_value = lhs.getResult(),
2504 .lhs_slot = f32x4Slot(-8),
2505 .rhs_value = rhs.getResult(),
2506 .rhs_slot = f32x4Slot(-32),
2507 .result_value = add.getResult(),
2508 .result_slot = f32x4Slot(-56),
2509 };
2510 defer recorder.bytes.deinit(std.testing.allocator);
2511
2512 try emitBinaryOp(&recorder, add.op, .add);
2513
2514 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2515 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2516 try std.testing.expectEqualSlices(u8, &.{
2517 0x0F, 0x10, 0x45, 0xEC,
2518 0x0F, 0x10, 0x4D, 0xD4,
2519 0x0F, 0x58, 0xC1, 0x0F,
2520 0x11, 0x45, 0xBC,
2521 }, recorder.bytes.items);
2522 }
2523
2524 test "x86_64 vector binary f32x4 mul emits packed SSE" {
2525 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2526 defer ctx.deinit(std.testing.allocator);
2527 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2528
2529 const loc = ir.Location.getUnknown();
2530 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
2531 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
2532 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.0);
2533 const mul = try ArithDialect.MulOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2534
2535 var recorder = BinaryRecorder{
2536 .allocator = std.testing.allocator,
2537 .lhs_value = lhs.getResult(),
2538 .lhs_slot = f32x4Slot(-8),
2539 .rhs_value = rhs.getResult(),
2540 .rhs_slot = f32x4Slot(-32),
2541 .result_value = mul.getResult(),
2542 .result_slot = f32x4Slot(-56),
2543 };
2544 defer recorder.bytes.deinit(std.testing.allocator);
2545
2546 try emitBinaryOp(&recorder, mul.op, .mul);
2547
2548 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2549 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2550 try std.testing.expectEqualSlices(u8, &.{
2551 0x0F, 0x10, 0x45, 0xEC,
2552 0x0F, 0x10, 0x4D, 0xD4,
2553 0x0F, 0x59, 0xC1, 0x0F,
2554 0x11, 0x45, 0xBC,
2555 }, recorder.bytes.items);
2556 }
2557
2558 test "x86_64 vector binary f32x4 div emits packed SSE" {
2559 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2560 defer ctx.deinit(std.testing.allocator);
2561 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2562
2563 const loc = ir.Location.getUnknown();
2564 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
2565 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 8.0);
2566 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.0);
2567 const div = try ArithDialect.DivOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2568
2569 var recorder = BinaryRecorder{
2570 .allocator = std.testing.allocator,
2571 .lhs_value = lhs.getResult(),
2572 .lhs_slot = f32x4Slot(-8),
2573 .rhs_value = rhs.getResult(),
2574 .rhs_slot = f32x4Slot(-32),
2575 .result_value = div.getResult(),
2576 .result_slot = f32x4Slot(-56),
2577 };
2578 defer recorder.bytes.deinit(std.testing.allocator);
2579
2580 try emitBinaryOp(&recorder, div.op, .div);
2581
2582 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2583 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2584 try std.testing.expectEqualSlices(u8, &.{
2585 0x0F, 0x10, 0x45, 0xEC,
2586 0x0F, 0x10, 0x4D, 0xD4,
2587 0x0F, 0x5E, 0xC1, 0x0F,
2588 0x11, 0x45, 0xBC,
2589 }, recorder.bytes.items);
2590 }
2591
2592 test "x86_64 vector binary u32x4 add emits packed SSE2" {
2593 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2594 defer ctx.deinit(std.testing.allocator);
2595 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2596
2597 const loc = ir.Location.getUnknown();
2598 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2599 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2600 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2601 const add = try ArithDialect.AddOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2602
2603 var recorder = BinaryRecorder{
2604 .allocator = std.testing.allocator,
2605 .lhs_value = lhs.getResult(),
2606 .lhs_slot = u32x4Slot(-8),
2607 .rhs_value = rhs.getResult(),
2608 .rhs_slot = u32x4Slot(-32),
2609 .result_value = add.getResult(),
2610 .result_slot = u32x4Slot(-56),
2611 };
2612 defer recorder.bytes.deinit(std.testing.allocator);
2613
2614 try emitBinaryOp(&recorder, add.op, .add);
2615
2616 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2617 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2618 try std.testing.expectEqualSlices(u8, &.{
2619 0x0F, 0x10, 0x45, 0xEC,
2620 0x0F, 0x10, 0x4D, 0xD4,
2621 0x66, 0x0F, 0xFE, 0xC1,
2622 0x0F, 0x11, 0x45, 0xBC,
2623 }, recorder.bytes.items);
2624 }
2625
2626 test "x86_64 vector binary u32x4 sub emits packed SSE2" {
2627 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2628 defer ctx.deinit(std.testing.allocator);
2629 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2630
2631 const loc = ir.Location.getUnknown();
2632 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2633 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2634 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2635 const sub = try ArithDialect.SubOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2636
2637 var recorder = BinaryRecorder{
2638 .allocator = std.testing.allocator,
2639 .lhs_value = lhs.getResult(),
2640 .lhs_slot = u32x4Slot(-8),
2641 .rhs_value = rhs.getResult(),
2642 .rhs_slot = u32x4Slot(-32),
2643 .result_value = sub.getResult(),
2644 .result_slot = u32x4Slot(-56),
2645 };
2646 defer recorder.bytes.deinit(std.testing.allocator);
2647
2648 try emitBinaryOp(&recorder, sub.op, .sub);
2649
2650 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2651 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2652 try std.testing.expectEqualSlices(u8, &.{
2653 0x0F, 0x10, 0x45, 0xEC,
2654 0x0F, 0x10, 0x4D, 0xD4,
2655 0x66, 0x0F, 0xFA, 0xC1,
2656 0x0F, 0x11, 0x45, 0xBC,
2657 }, recorder.bytes.items);
2658 }
2659
2660 test "x86_64 vector bitwise u32x4 and emits packed SSE2" {
2661 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2662 defer ctx.deinit(std.testing.allocator);
2663 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2664
2665 const loc = ir.Location.getUnknown();
2666 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2667 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2668 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2669 const bitwise = try ArithDialect.AndOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2670
2671 var recorder = BinaryRecorder{
2672 .allocator = std.testing.allocator,
2673 .lhs_value = lhs.getResult(),
2674 .lhs_slot = u32x4Slot(-8),
2675 .rhs_value = rhs.getResult(),
2676 .rhs_slot = u32x4Slot(-32),
2677 .result_value = bitwise.getResult(),
2678 .result_slot = u32x4Slot(-56),
2679 };
2680 defer recorder.bytes.deinit(std.testing.allocator);
2681
2682 try emitBitwiseBinary(&recorder, bitwise.op, .band);
2683
2684 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2685 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2686 try std.testing.expectEqualSlices(u8, &.{
2687 0x0F, 0x10, 0x45, 0xEC,
2688 0x0F, 0x10, 0x4D, 0xD4,
2689 0x66, 0x0F, 0xDB, 0xC1,
2690 0x0F, 0x11, 0x45, 0xBC,
2691 }, recorder.bytes.items);
2692 }
2693
2694 test "x86_64 vector bitwise u32x4 or emits packed SSE2" {
2695 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2696 defer ctx.deinit(std.testing.allocator);
2697 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2698
2699 const loc = ir.Location.getUnknown();
2700 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2701 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2702 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2703 const bitwise = try ArithDialect.OrOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2704
2705 var recorder = BinaryRecorder{
2706 .allocator = std.testing.allocator,
2707 .lhs_value = lhs.getResult(),
2708 .lhs_slot = u32x4Slot(-8),
2709 .rhs_value = rhs.getResult(),
2710 .rhs_slot = u32x4Slot(-32),
2711 .result_value = bitwise.getResult(),
2712 .result_slot = u32x4Slot(-56),
2713 };
2714 defer recorder.bytes.deinit(std.testing.allocator);
2715
2716 try emitBitwiseBinary(&recorder, bitwise.op, .bor);
2717
2718 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2719 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2720 try std.testing.expectEqualSlices(u8, &.{
2721 0x0F, 0x10, 0x45, 0xEC,
2722 0x0F, 0x10, 0x4D, 0xD4,
2723 0x66, 0x0F, 0xEB, 0xC1,
2724 0x0F, 0x11, 0x45, 0xBC,
2725 }, recorder.bytes.items);
2726 }
2727
2728 test "x86_64 vector binary u32x4 mul emits packed SSE2" {
2729 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2730 defer ctx.deinit(std.testing.allocator);
2731 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2732
2733 const loc = ir.Location.getUnknown();
2734 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2735 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2736 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2737 const mul = try ArithDialect.MulOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2738
2739 var recorder = BinaryRecorder{
2740 .allocator = std.testing.allocator,
2741 .lhs_value = lhs.getResult(),
2742 .lhs_slot = u32x4Slot(-8),
2743 .rhs_value = rhs.getResult(),
2744 .rhs_slot = u32x4Slot(-32),
2745 .result_value = mul.getResult(),
2746 .result_slot = u32x4Slot(-56),
2747 };
2748 defer recorder.bytes.deinit(std.testing.allocator);
2749
2750 try emitBinaryOp(&recorder, mul.op, .mul);
2751
2752 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2753 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2754 try std.testing.expectEqualSlices(u8, &.{
2755 0x0F, 0x10, 0x45, 0xEC,
2756 0x0F, 0x10, 0x4D, 0xD4,
2757 0x0F, 0x10, 0xD0, 0x66,
2758 0x0F, 0xF4, 0xC1, 0x66,
2759 0x0F, 0x73, 0xD2, 0x20,
2760 0x66, 0x0F, 0x73, 0xD1,
2761 0x20, 0x66, 0x0F, 0xF4,
2762 0xD1, 0x66, 0x0F, 0x76,
2763 0xDB, 0x66, 0x0F, 0x73,
2764 0xD3, 0x20, 0x66, 0x0F,
2765 0xDB, 0xC3, 0x66, 0x0F,
2766 0xDB, 0xD3, 0x66, 0x0F,
2767 0x73, 0xF2, 0x20, 0x66,
2768 0x0F, 0xEB, 0xC2, 0x0F,
2769 0x11, 0x45, 0xBC,
2770 }, recorder.bytes.items);
2771 }
2772
2773 test "x86_64 vector umulhi u32x4 emits packed SSE2" {
2774 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2775 defer ctx.deinit(std.testing.allocator);
2776 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2777
2778 const loc = ir.Location.getUnknown();
2779 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2780 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 0xffff_ffff);
2781 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2782 const high = try ArithDialect.UmulhiOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2783
2784 var recorder = BinaryRecorder{
2785 .allocator = std.testing.allocator,
2786 .lhs_value = lhs.getResult(),
2787 .lhs_slot = u32x4Slot(-8),
2788 .rhs_value = rhs.getResult(),
2789 .rhs_slot = u32x4Slot(-32),
2790 .result_value = high.getResult(),
2791 .result_slot = u32x4Slot(-56),
2792 };
2793 defer recorder.bytes.deinit(std.testing.allocator);
2794
2795 try emitUmulhi(&recorder, high.op);
2796
2797 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2798 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2799 try std.testing.expect(std.mem.indexOf(u8, recorder.bytes.items, encoding.pmuludq(.xmm0, .{ .reg = .xmm1 }).slice()) != null);
2800 try std.testing.expect(std.mem.indexOf(u8, recorder.bytes.items, encoding.psrlqImm(.xmm0, 32).slice()) != null);
2801 }
2802
2803 test "x86_64 vector popcount u32x4 emits packed SSE2" {
2804 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2805 defer ctx.deinit(std.testing.allocator);
2806 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2807
2808 const loc = ir.Location.getUnknown();
2809 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2810 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 0xf0f0_00ff);
2811 const count = try ArithDialect.PopCountOp.create(&ctx, loc, input.getResult());
2812
2813 var recorder = BinaryRecorder{
2814 .allocator = std.testing.allocator,
2815 .lhs_value = input.getResult(),
2816 .lhs_slot = u32x4Slot(-8),
2817 .rhs_value = input.getResult(),
2818 .rhs_slot = u32x4Slot(-8),
2819 .result_value = count.getResult(),
2820 .result_slot = u32x4Slot(-32),
2821 };
2822 defer recorder.bytes.deinit(std.testing.allocator);
2823
2824 try emitPopCount(&recorder, count.op);
2825
2826 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2827 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2828 try std.testing.expect(std.mem.indexOf(u8, recorder.bytes.items, encoding.psubd(.xmm0, .{ .reg = .xmm1 }).slice()) != null);
2829 try std.testing.expect(std.mem.indexOf(u8, recorder.bytes.items, encoding.paddd(.xmm0, .{ .reg = .xmm1 }).slice()) != null);
2830 }
2831
2832 test "x86_64 vector popcount u64x2 emits packed SSE2" {
2833 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2834 defer ctx.deinit(std.testing.allocator);
2835 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2836
2837 const loc = ir.Location.getUnknown();
2838 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
2839 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -1);
2840 const count = try ArithDialect.PopCountOp.create(&ctx, loc, input.getResult());
2841
2842 var recorder = BinaryRecorder{
2843 .allocator = std.testing.allocator,
2844 .lhs_value = input.getResult(),
2845 .lhs_slot = u64x2Slot(-8),
2846 .rhs_value = input.getResult(),
2847 .rhs_slot = u64x2Slot(-8),
2848 .result_value = count.getResult(),
2849 .result_slot = u64x2Slot(-24),
2850 };
2851 defer recorder.bytes.deinit(std.testing.allocator);
2852
2853 try emitPopCount(&recorder, count.op);
2854
2855 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2856 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2857 try std.testing.expect(std.mem.indexOf(u8, recorder.bytes.items, encoding.psubq(.xmm0, .{ .reg = .xmm1 }).slice()) != null);
2858 try std.testing.expect(std.mem.indexOf(u8, recorder.bytes.items, encoding.paddq(.xmm0, .{ .reg = .xmm1 }).slice()) != null);
2859 }
2860
2861 test "x86_64 vector binary u32x4 div uses unsigned fallback" {
2862 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2863 defer ctx.deinit(std.testing.allocator);
2864 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2865
2866 const loc = ir.Location.getUnknown();
2867 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
2868 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 8);
2869 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2870 const div = try ArithDialect.DivOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2871
2872 var recorder = BinaryRecorder{
2873 .allocator = std.testing.allocator,
2874 .lhs_value = lhs.getResult(),
2875 .lhs_slot = u32x4Slot(-8),
2876 .rhs_value = rhs.getResult(),
2877 .rhs_slot = u32x4Slot(-32),
2878 .result_value = div.getResult(),
2879 .result_slot = u32x4Slot(-56),
2880 };
2881 defer recorder.bytes.deinit(std.testing.allocator);
2882
2883 try emitBinaryOp(&recorder, div.op, .div);
2884
2885 try std.testing.expectEqual(@as(usize, 8), recorder.scalar_loads);
2886 try std.testing.expectEqual(@as(usize, 4), recorder.scalar_stores);
2887 try std.testing.expectEqualSlices(u8, &.{
2888 0x31, 0xD2, 0xF7, 0xF1,
2889 0x31, 0xD2, 0xF7, 0xF1,
2890 0x31, 0xD2, 0xF7, 0xF1,
2891 0x31, 0xD2, 0xF7, 0xF1,
2892 }, recorder.bytes.items);
2893 }
2894
2895 test "x86_64 vector binary i32x4 sub emits packed SSE2" {
2896 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2897 defer ctx.deinit(std.testing.allocator);
2898 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2899
2900 const loc = ir.Location.getUnknown();
2901 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.int32)).?;
2902 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2903 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2904 const sub = try ArithDialect.SubOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2905
2906 var recorder = BinaryRecorder{
2907 .allocator = std.testing.allocator,
2908 .lhs_value = lhs.getResult(),
2909 .lhs_slot = i32x4Slot(-8),
2910 .rhs_value = rhs.getResult(),
2911 .rhs_slot = i32x4Slot(-32),
2912 .result_value = sub.getResult(),
2913 .result_slot = i32x4Slot(-56),
2914 };
2915 defer recorder.bytes.deinit(std.testing.allocator);
2916
2917 try emitBinaryOp(&recorder, sub.op, .sub);
2918
2919 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2920 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2921 try std.testing.expectEqualSlices(u8, &.{
2922 0x0F, 0x10, 0x45, 0xEC,
2923 0x0F, 0x10, 0x4D, 0xD4,
2924 0x66, 0x0F, 0xFA, 0xC1,
2925 0x0F, 0x11, 0x45, 0xBC,
2926 }, recorder.bytes.items);
2927 }
2928
2929 test "x86_64 vector binary i32x4 mul emits packed SSE2" {
2930 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2931 defer ctx.deinit(std.testing.allocator);
2932 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2933
2934 const loc = ir.Location.getUnknown();
2935 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.int32)).?;
2936 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
2937 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2938 const mul = try ArithDialect.MulOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2939
2940 var recorder = BinaryRecorder{
2941 .allocator = std.testing.allocator,
2942 .lhs_value = lhs.getResult(),
2943 .lhs_slot = i32x4Slot(-8),
2944 .rhs_value = rhs.getResult(),
2945 .rhs_slot = i32x4Slot(-32),
2946 .result_value = mul.getResult(),
2947 .result_slot = i32x4Slot(-56),
2948 };
2949 defer recorder.bytes.deinit(std.testing.allocator);
2950
2951 try emitBinaryOp(&recorder, mul.op, .mul);
2952
2953 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
2954 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
2955 try std.testing.expectEqualSlices(u8, &.{
2956 0x0F, 0x10, 0x45, 0xEC,
2957 0x0F, 0x10, 0x4D, 0xD4,
2958 0x0F, 0x10, 0xD0, 0x66,
2959 0x0F, 0xF4, 0xC1, 0x66,
2960 0x0F, 0x73, 0xD2, 0x20,
2961 0x66, 0x0F, 0x73, 0xD1,
2962 0x20, 0x66, 0x0F, 0xF4,
2963 0xD1, 0x66, 0x0F, 0x76,
2964 0xDB, 0x66, 0x0F, 0x73,
2965 0xD3, 0x20, 0x66, 0x0F,
2966 0xDB, 0xC3, 0x66, 0x0F,
2967 0xDB, 0xD3, 0x66, 0x0F,
2968 0x73, 0xF2, 0x20, 0x66,
2969 0x0F, 0xEB, 0xC2, 0x0F,
2970 0x11, 0x45, 0xBC,
2971 }, recorder.bytes.items);
2972 }
2973
2974 test "x86_64 vector binary i32x4 div uses signed fallback" {
2975 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2976 defer ctx.deinit(std.testing.allocator);
2977 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
2978
2979 const loc = ir.Location.getUnknown();
2980 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.int32)).?;
2981 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -8);
2982 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
2983 const div = try ArithDialect.DivOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
2984
2985 var recorder = BinaryRecorder{
2986 .allocator = std.testing.allocator,
2987 .lhs_value = lhs.getResult(),
2988 .lhs_slot = i32x4Slot(-8),
2989 .rhs_value = rhs.getResult(),
2990 .rhs_slot = i32x4Slot(-32),
2991 .result_value = div.getResult(),
2992 .result_slot = i32x4Slot(-56),
2993 };
2994 defer recorder.bytes.deinit(std.testing.allocator);
2995
2996 try emitBinaryOp(&recorder, div.op, .div);
2997
2998 try std.testing.expectEqual(@as(usize, 8), recorder.scalar_loads);
2999 try std.testing.expectEqual(@as(usize, 4), recorder.scalar_stores);
3000 try std.testing.expectEqualSlices(u8, &.{
3001 0x99, 0xF7, 0xF9,
3002 0x99, 0xF7, 0xF9,
3003 0x99, 0xF7, 0xF9,
3004 0x99, 0xF7, 0xF9,
3005 }, recorder.bytes.items);
3006 }
3007
3008 test "x86_64 vector binary i64x2 add emits packed SSE2" {
3009 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3010 defer ctx.deinit(std.testing.allocator);
3011 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3012
3013 const loc = ir.Location.getUnknown();
3014 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
3015 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3016 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3017 const add = try ArithDialect.AddOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3018
3019 var recorder = BinaryRecorder{
3020 .allocator = std.testing.allocator,
3021 .lhs_value = lhs.getResult(),
3022 .lhs_slot = i64x2Slot(-8),
3023 .rhs_value = rhs.getResult(),
3024 .rhs_slot = i64x2Slot(-24),
3025 .result_value = add.getResult(),
3026 .result_slot = i64x2Slot(-40),
3027 };
3028 defer recorder.bytes.deinit(std.testing.allocator);
3029
3030 try emitBinaryOp(&recorder, add.op, .add);
3031
3032 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3033 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3034 try std.testing.expectEqualSlices(u8, &.{
3035 0x66, 0x0F, 0x10, 0x45, 0xF0,
3036 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3037 0x66, 0x0F, 0xD4, 0xC1, 0x66,
3038 0x0F, 0x11, 0x45, 0xD0,
3039 }, recorder.bytes.items);
3040 }
3041
3042 test "x86_64 vector binary u64x2 sub emits packed SSE2" {
3043 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3044 defer ctx.deinit(std.testing.allocator);
3045 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3046
3047 const loc = ir.Location.getUnknown();
3048 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3049 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
3050 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3051 const sub = try ArithDialect.SubOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3052
3053 var recorder = BinaryRecorder{
3054 .allocator = std.testing.allocator,
3055 .lhs_value = lhs.getResult(),
3056 .lhs_slot = u64x2Slot(-8),
3057 .rhs_value = rhs.getResult(),
3058 .rhs_slot = u64x2Slot(-24),
3059 .result_value = sub.getResult(),
3060 .result_slot = u64x2Slot(-40),
3061 };
3062 defer recorder.bytes.deinit(std.testing.allocator);
3063
3064 try emitBinaryOp(&recorder, sub.op, .sub);
3065
3066 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3067 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3068 try std.testing.expectEqualSlices(u8, &.{
3069 0x66, 0x0F, 0x10, 0x45, 0xF0,
3070 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3071 0x66, 0x0F, 0xFB, 0xC1, 0x66,
3072 0x0F, 0x11, 0x45, 0xD0,
3073 }, recorder.bytes.items);
3074 }
3075
3076 test "x86_64 vector bitwise u64x2 xor emits packed SSE2" {
3077 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3078 defer ctx.deinit(std.testing.allocator);
3079 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3080
3081 const loc = ir.Location.getUnknown();
3082 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3083 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3084 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3085 const bitwise = try ArithDialect.XorOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3086
3087 var recorder = BinaryRecorder{
3088 .allocator = std.testing.allocator,
3089 .lhs_value = lhs.getResult(),
3090 .lhs_slot = u64x2Slot(-8),
3091 .rhs_value = rhs.getResult(),
3092 .rhs_slot = u64x2Slot(-24),
3093 .result_value = bitwise.getResult(),
3094 .result_slot = u64x2Slot(-40),
3095 };
3096 defer recorder.bytes.deinit(std.testing.allocator);
3097
3098 try emitBitwiseBinary(&recorder, bitwise.op, .bxor);
3099
3100 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3101 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3102 try std.testing.expectEqualSlices(u8, &.{
3103 0x66, 0x0F, 0x10, 0x45, 0xF0,
3104 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3105 0x66, 0x0F, 0xEF, 0xC1, 0x66,
3106 0x0F, 0x11, 0x45, 0xD0,
3107 }, recorder.bytes.items);
3108 }
3109
3110 test "x86_64 vector shift u32x4 shl splat count emits packed SSE2" {
3111 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3112 defer ctx.deinit(std.testing.allocator);
3113 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3114
3115 const loc = ir.Location.getUnknown();
3116 const scalar_type = try ArithDialect.getScalarType(&ctx, .u32);
3117 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
3118 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3119 var scalar_count = try ArithDialect.ConstantOp.createInt(&ctx, loc, scalar_type, 3);
3120 var count = try ArithDialect.SplatOp.create(&ctx, loc, scalar_count.getResult(), vec_type);
3121 const shifted = try ArithDialect.ShlOp.create(&ctx, loc, lhs.getResult(), count.getResult());
3122
3123 var recorder = BinaryRecorder{
3124 .allocator = std.testing.allocator,
3125 .lhs_value = lhs.getResult(),
3126 .lhs_slot = u32x4Slot(-8),
3127 .rhs_value = count.getResult(),
3128 .rhs_slot = u32x4Slot(-32),
3129 .result_value = shifted.getResult(),
3130 .result_slot = u32x4Slot(-56),
3131 };
3132 defer recorder.bytes.deinit(std.testing.allocator);
3133
3134 try emitShift(&recorder, shifted.op, .shl);
3135
3136 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3137 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3138 try std.testing.expectEqualSlices(u8, &.{
3139 0x0F, 0x10, 0x45, 0xEC,
3140 0x66, 0x0F, 0x72, 0xF0,
3141 0x03, 0x0F, 0x11, 0x45,
3142 0xBC,
3143 }, recorder.bytes.items);
3144 }
3145
3146 test "x86_64 vector shift i32x4 shr constant count emits packed SSE2" {
3147 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3148 defer ctx.deinit(std.testing.allocator);
3149 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3150
3151 const loc = ir.Location.getUnknown();
3152 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.int32)).?;
3153 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -8);
3154 var count = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 3);
3155 const shifted = try ArithDialect.ShrOp.create(&ctx, loc, lhs.getResult(), count.getResult());
3156
3157 var recorder = BinaryRecorder{
3158 .allocator = std.testing.allocator,
3159 .lhs_value = lhs.getResult(),
3160 .lhs_slot = i32x4Slot(-8),
3161 .rhs_value = count.getResult(),
3162 .rhs_slot = i32x4Slot(-32),
3163 .result_value = shifted.getResult(),
3164 .result_slot = i32x4Slot(-56),
3165 };
3166 defer recorder.bytes.deinit(std.testing.allocator);
3167
3168 try emitShift(&recorder, shifted.op, .shr);
3169
3170 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3171 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3172 try std.testing.expectEqualSlices(u8, &.{
3173 0x0F, 0x10, 0x45, 0xEC,
3174 0x66, 0x0F, 0x72, 0xE0,
3175 0x03, 0x0F, 0x11, 0x45,
3176 0xBC,
3177 }, recorder.bytes.items);
3178 }
3179
3180 test "x86_64 vector shift u64x2 ushr constant count emits packed SSE2" {
3181 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3182 defer ctx.deinit(std.testing.allocator);
3183 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3184
3185 const loc = ir.Location.getUnknown();
3186 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3187 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -1);
3188 var count = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 4);
3189 const shifted = try ArithDialect.UshrOp.create(&ctx, loc, lhs.getResult(), count.getResult());
3190
3191 var recorder = BinaryRecorder{
3192 .allocator = std.testing.allocator,
3193 .lhs_value = lhs.getResult(),
3194 .lhs_slot = u64x2Slot(-8),
3195 .rhs_value = count.getResult(),
3196 .rhs_slot = u64x2Slot(-24),
3197 .result_value = shifted.getResult(),
3198 .result_slot = u64x2Slot(-40),
3199 };
3200 defer recorder.bytes.deinit(std.testing.allocator);
3201
3202 try emitShift(&recorder, shifted.op, .ushr);
3203
3204 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3205 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3206 try std.testing.expectEqualSlices(u8, &.{
3207 0x66, 0x0F, 0x10, 0x45, 0xF0,
3208 0x66, 0x0F, 0x73, 0xD0, 0x04,
3209 0x66, 0x0F, 0x11, 0x45, 0xD0,
3210 }, recorder.bytes.items);
3211 }
3212
3213 test "x86_64 vector shift i64x2 shr uses signed fallback" {
3214 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3215 defer ctx.deinit(std.testing.allocator);
3216 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3217
3218 const loc = ir.Location.getUnknown();
3219 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
3220 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -8);
3221 var count = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3222 const shifted = try ArithDialect.ShrOp.create(&ctx, loc, lhs.getResult(), count.getResult());
3223
3224 var recorder = BinaryRecorder{
3225 .allocator = std.testing.allocator,
3226 .lhs_value = lhs.getResult(),
3227 .lhs_slot = i64x2Slot(-8),
3228 .rhs_value = count.getResult(),
3229 .rhs_slot = i64x2Slot(-24),
3230 .result_value = shifted.getResult(),
3231 .result_slot = i64x2Slot(-40),
3232 };
3233 defer recorder.bytes.deinit(std.testing.allocator);
3234
3235 try emitShift(&recorder, shifted.op, .shr);
3236
3237 try std.testing.expectEqual(@as(usize, 4), recorder.scalar_loads);
3238 try std.testing.expectEqual(@as(usize, 2), recorder.scalar_stores);
3239 try std.testing.expectEqualSlices(u8, &.{
3240 0x48, 0xD3, 0xF8,
3241 0x48, 0xD3, 0xF8,
3242 }, recorder.bytes.items);
3243 }
3244
3245 test "x86_64 vector binary i64x2 mul emits packed SSE2" {
3246 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3247 defer ctx.deinit(std.testing.allocator);
3248 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3249
3250 const loc = ir.Location.getUnknown();
3251 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
3252 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
3253 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3254 const mul = try ArithDialect.MulOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3255
3256 var recorder = BinaryRecorder{
3257 .allocator = std.testing.allocator,
3258 .lhs_value = lhs.getResult(),
3259 .lhs_slot = i64x2Slot(-8),
3260 .rhs_value = rhs.getResult(),
3261 .rhs_slot = i64x2Slot(-24),
3262 .result_value = mul.getResult(),
3263 .result_slot = i64x2Slot(-40),
3264 };
3265 defer recorder.bytes.deinit(std.testing.allocator);
3266
3267 try emitBinaryOp(&recorder, mul.op, .mul);
3268
3269 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3270 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3271 try std.testing.expectEqualSlices(u8, &.{
3272 0x66, 0x0F, 0x10, 0x45, 0xF0,
3273 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3274 0x66, 0x0F, 0x10, 0xD0, 0x66,
3275 0x0F, 0x73, 0xD2, 0x20, 0x66,
3276 0x0F, 0x10, 0xD9, 0x66, 0x0F,
3277 0x73, 0xD3, 0x20, 0x66, 0x0F,
3278 0x10, 0xE0, 0x66, 0x0F, 0xF4,
3279 0xD1, 0x66, 0x0F, 0xF4, 0xE3,
3280 0x66, 0x0F, 0xD4, 0xD4, 0x66,
3281 0x0F, 0x73, 0xF2, 0x20, 0x66,
3282 0x0F, 0xF4, 0xC1, 0x66, 0x0F,
3283 0xD4, 0xC2, 0x66, 0x0F, 0x11,
3284 0x45, 0xD0,
3285 }, recorder.bytes.items);
3286 }
3287
3288 test "x86_64 vector binary i64x2 div uses signed fallback" {
3289 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3290 defer ctx.deinit(std.testing.allocator);
3291 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3292
3293 const loc = ir.Location.getUnknown();
3294 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
3295 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, -8);
3296 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3297 const div = try ArithDialect.DivOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3298
3299 var recorder = BinaryRecorder{
3300 .allocator = std.testing.allocator,
3301 .lhs_value = lhs.getResult(),
3302 .lhs_slot = i64x2Slot(-8),
3303 .rhs_value = rhs.getResult(),
3304 .rhs_slot = i64x2Slot(-24),
3305 .result_value = div.getResult(),
3306 .result_slot = i64x2Slot(-40),
3307 };
3308 defer recorder.bytes.deinit(std.testing.allocator);
3309
3310 try emitBinaryOp(&recorder, div.op, .div);
3311
3312 try std.testing.expectEqual(@as(usize, 4), recorder.scalar_loads);
3313 try std.testing.expectEqual(@as(usize, 2), recorder.scalar_stores);
3314 try std.testing.expectEqualSlices(u8, &.{
3315 0x48, 0x99, 0x48, 0xF7, 0xF9,
3316 0x48, 0x99, 0x48, 0xF7, 0xF9,
3317 }, recorder.bytes.items);
3318 }
3319
3320 test "x86_64 vector binary u64x2 mul emits packed SSE2" {
3321 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3322 defer ctx.deinit(std.testing.allocator);
3323 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3324
3325 const loc = ir.Location.getUnknown();
3326 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3327 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 7);
3328 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3329 const mul = try ArithDialect.MulOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3330
3331 var recorder = BinaryRecorder{
3332 .allocator = std.testing.allocator,
3333 .lhs_value = lhs.getResult(),
3334 .lhs_slot = u64x2Slot(-8),
3335 .rhs_value = rhs.getResult(),
3336 .rhs_slot = u64x2Slot(-24),
3337 .result_value = mul.getResult(),
3338 .result_slot = u64x2Slot(-40),
3339 };
3340 defer recorder.bytes.deinit(std.testing.allocator);
3341
3342 try emitBinaryOp(&recorder, mul.op, .mul);
3343
3344 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3345 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3346 try std.testing.expectEqualSlices(u8, &.{
3347 0x66, 0x0F, 0x10, 0x45, 0xF0,
3348 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3349 0x66, 0x0F, 0x10, 0xD0, 0x66,
3350 0x0F, 0x73, 0xD2, 0x20, 0x66,
3351 0x0F, 0x10, 0xD9, 0x66, 0x0F,
3352 0x73, 0xD3, 0x20, 0x66, 0x0F,
3353 0x10, 0xE0, 0x66, 0x0F, 0xF4,
3354 0xD1, 0x66, 0x0F, 0xF4, 0xE3,
3355 0x66, 0x0F, 0xD4, 0xD4, 0x66,
3356 0x0F, 0x73, 0xF2, 0x20, 0x66,
3357 0x0F, 0xF4, 0xC1, 0x66, 0x0F,
3358 0xD4, 0xC2, 0x66, 0x0F, 0x11,
3359 0x45, 0xD0,
3360 }, recorder.bytes.items);
3361 }
3362
3363 test "x86_64 vector binary u64x2 div uses unsigned fallback" {
3364 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3365 defer ctx.deinit(std.testing.allocator);
3366 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3367
3368 const loc = ir.Location.getUnknown();
3369 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3370 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 8);
3371 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3372 const div = try ArithDialect.DivOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3373
3374 var recorder = BinaryRecorder{
3375 .allocator = std.testing.allocator,
3376 .lhs_value = lhs.getResult(),
3377 .lhs_slot = u64x2Slot(-8),
3378 .rhs_value = rhs.getResult(),
3379 .rhs_slot = u64x2Slot(-24),
3380 .result_value = div.getResult(),
3381 .result_slot = u64x2Slot(-40),
3382 };
3383 defer recorder.bytes.deinit(std.testing.allocator);
3384
3385 try emitBinaryOp(&recorder, div.op, .div);
3386
3387 try std.testing.expectEqual(@as(usize, 4), recorder.scalar_loads);
3388 try std.testing.expectEqual(@as(usize, 2), recorder.scalar_stores);
3389 try std.testing.expectEqualSlices(u8, &.{
3390 0x48, 0x31, 0xD2, 0x48, 0xF7, 0xF1,
3391 0x48, 0x31, 0xD2, 0x48, 0xF7, 0xF1,
3392 }, recorder.bytes.items);
3393 }
3394
3395 test "x86_64 vector cmp f32x4 lt emits packed SSE" {
3396 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3397 defer ctx.deinit(std.testing.allocator);
3398 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3399
3400 const loc = ir.Location.getUnknown();
3401 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
3402 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
3403 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.0);
3404 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .lt, lhs.getResult(), rhs.getResult(), vec_type);
3405
3406 var recorder = BinaryRecorder{
3407 .allocator = std.testing.allocator,
3408 .lhs_value = lhs.getResult(),
3409 .lhs_slot = f32x4Slot(-8),
3410 .rhs_value = rhs.getResult(),
3411 .rhs_slot = f32x4Slot(-32),
3412 .result_value = cmp.getResult(),
3413 .result_slot = f32x4Slot(-56),
3414 };
3415 defer recorder.bytes.deinit(std.testing.allocator);
3416
3417 try emitCmp(&recorder, cmp.op);
3418
3419 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3420 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3421 try std.testing.expectEqualSlices(u8, &.{
3422 0x0F, 0x10, 0x45, 0xEC,
3423 0x0F, 0x10, 0x4D, 0xD4,
3424 0x0F, 0xC2, 0xC1, 0x01,
3425 0x66, 0x0F, 0x72, 0xD0,
3426 0x1F, 0x0F, 0x11, 0x45,
3427 0xBC,
3428 }, recorder.bytes.items);
3429 }
3430
3431 test "x86_64 vector cmp f64x2 ge emits packed SSE2" {
3432 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3433 defer ctx.deinit(std.testing.allocator);
3434 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3435
3436 const loc = ir.Location.getUnknown();
3437 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
3438 var lhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
3439 var rhs = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 2.0);
3440 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .ge, lhs.getResult(), rhs.getResult(), vec_type);
3441
3442 var recorder = BinaryRecorder{
3443 .allocator = std.testing.allocator,
3444 .lhs_value = lhs.getResult(),
3445 .lhs_slot = f64x2Slot(-8),
3446 .rhs_value = rhs.getResult(),
3447 .rhs_slot = f64x2Slot(-24),
3448 .result_value = cmp.getResult(),
3449 .result_slot = f64x2Slot(-40),
3450 };
3451 defer recorder.bytes.deinit(std.testing.allocator);
3452
3453 try emitCmp(&recorder, cmp.op);
3454
3455 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3456 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3457 try std.testing.expectEqualSlices(u8, &.{
3458 0x66, 0x0F, 0x10, 0x45, 0xE0,
3459 0x66, 0x0F, 0x10, 0x4D, 0xF0,
3460 0x66, 0x0F, 0xC2, 0xC1, 0x02,
3461 0x66, 0x0F, 0x73, 0xD0, 0x3F,
3462 0x66, 0x0F, 0x11, 0x45, 0xD0,
3463 }, recorder.bytes.items);
3464 }
3465
3466 test "x86_64 vector cmp i32x4 slt emits packed SSE2" {
3467 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3468 defer ctx.deinit(std.testing.allocator);
3469 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3470
3471 const loc = ir.Location.getUnknown();
3472 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.int32)).?;
3473 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3474 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3475 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .slt, lhs.getResult(), rhs.getResult(), vec_type);
3476
3477 var recorder = BinaryRecorder{
3478 .allocator = std.testing.allocator,
3479 .lhs_value = lhs.getResult(),
3480 .lhs_slot = i32x4Slot(-8),
3481 .rhs_value = rhs.getResult(),
3482 .rhs_slot = i32x4Slot(-32),
3483 .result_value = cmp.getResult(),
3484 .result_slot = i32x4Slot(-56),
3485 };
3486 defer recorder.bytes.deinit(std.testing.allocator);
3487
3488 try emitCmp(&recorder, cmp.op);
3489
3490 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3491 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3492 try std.testing.expectEqualSlices(u8, &.{
3493 0x0F, 0x10, 0x45, 0xD4,
3494 0x0F, 0x10, 0x4D, 0xEC,
3495 0x66, 0x0F, 0x66, 0xC1,
3496 0x66, 0x0F, 0x72, 0xD0,
3497 0x1F, 0x0F, 0x11, 0x45,
3498 0xBC,
3499 }, recorder.bytes.items);
3500 }
3501
3502 test "x86_64 vector cmp u32x4 uge emits packed SSE2" {
3503 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3504 defer ctx.deinit(std.testing.allocator);
3505 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3506
3507 const loc = ir.Location.getUnknown();
3508 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
3509 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3510 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3511 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .uge, lhs.getResult(), rhs.getResult(), vec_type);
3512
3513 var recorder = BinaryRecorder{
3514 .allocator = std.testing.allocator,
3515 .lhs_value = lhs.getResult(),
3516 .lhs_slot = u32x4Slot(-8),
3517 .rhs_value = rhs.getResult(),
3518 .rhs_slot = u32x4Slot(-32),
3519 .result_value = cmp.getResult(),
3520 .result_slot = u32x4Slot(-56),
3521 };
3522 defer recorder.bytes.deinit(std.testing.allocator);
3523
3524 try emitCmp(&recorder, cmp.op);
3525
3526 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3527 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3528 try std.testing.expectEqualSlices(u8, &.{
3529 0x0F, 0x10, 0x45, 0xD4,
3530 0x0F, 0x10, 0x4D, 0xEC,
3531 0x66, 0x0F, 0x76, 0xD2,
3532 0x66, 0x0F, 0x72, 0xF2,
3533 0x1F, 0x66, 0x0F, 0xEF,
3534 0xC2, 0x66, 0x0F, 0xEF,
3535 0xCA, 0x66, 0x0F, 0x66,
3536 0xC1, 0x66, 0x0F, 0x72,
3537 0xD0, 0x1F, 0x66, 0x0F,
3538 0x76, 0xC9, 0x66, 0x0F,
3539 0x72, 0xD1, 0x1F, 0x66,
3540 0x0F, 0xEF, 0xC1, 0x0F,
3541 0x11, 0x45, 0xBC,
3542 }, recorder.bytes.items);
3543 }
3544
3545 test "x86_64 vector cmp i64x2 eq emits packed SSE2" {
3546 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3547 defer ctx.deinit(std.testing.allocator);
3548 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3549
3550 const loc = ir.Location.getUnknown();
3551 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
3552 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3553 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3554 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .eq, lhs.getResult(), rhs.getResult(), vec_type);
3555
3556 var recorder = BinaryRecorder{
3557 .allocator = std.testing.allocator,
3558 .lhs_value = lhs.getResult(),
3559 .lhs_slot = i64x2Slot(-8),
3560 .rhs_value = rhs.getResult(),
3561 .rhs_slot = i64x2Slot(-24),
3562 .result_value = cmp.getResult(),
3563 .result_slot = i64x2Slot(-40),
3564 };
3565 defer recorder.bytes.deinit(std.testing.allocator);
3566
3567 try emitCmp(&recorder, cmp.op);
3568
3569 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3570 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3571 try std.testing.expectEqualSlices(u8, &.{
3572 0x66, 0x0F, 0x10, 0x45, 0xF0,
3573 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3574 0x66, 0x0F, 0x76, 0xC1, 0x66,
3575 0x0F, 0x70, 0xC8, 0xB1, 0x66,
3576 0x0F, 0xDB, 0xC1, 0x66, 0x0F,
3577 0x73, 0xD0, 0x3F, 0x66, 0x0F,
3578 0x11, 0x45, 0xD0,
3579 }, recorder.bytes.items);
3580 }
3581
3582 test "x86_64 vector cmp u64x2 ne emits packed SSE2" {
3583 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3584 defer ctx.deinit(std.testing.allocator);
3585 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3586
3587 const loc = ir.Location.getUnknown();
3588 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3589 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3590 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3591 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .ne, lhs.getResult(), rhs.getResult(), vec_type);
3592
3593 var recorder = BinaryRecorder{
3594 .allocator = std.testing.allocator,
3595 .lhs_value = lhs.getResult(),
3596 .lhs_slot = u64x2Slot(-8),
3597 .rhs_value = rhs.getResult(),
3598 .rhs_slot = u64x2Slot(-24),
3599 .result_value = cmp.getResult(),
3600 .result_slot = u64x2Slot(-40),
3601 };
3602 defer recorder.bytes.deinit(std.testing.allocator);
3603
3604 try emitCmp(&recorder, cmp.op);
3605
3606 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3607 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3608 try std.testing.expectEqualSlices(u8, &.{
3609 0x66, 0x0F, 0x10, 0x45, 0xF0,
3610 0x66, 0x0F, 0x10, 0x4D, 0xE0,
3611 0x66, 0x0F, 0x76, 0xC1, 0x66,
3612 0x0F, 0x70, 0xC8, 0xB1, 0x66,
3613 0x0F, 0xDB, 0xC1, 0x66, 0x0F,
3614 0x73, 0xD0, 0x3F, 0x66, 0x0F,
3615 0x76, 0xC9, 0x66, 0x0F, 0x73,
3616 0xD1, 0x3F, 0x66, 0x0F, 0xEF,
3617 0xC1, 0x66, 0x0F, 0x11, 0x45,
3618 0xD0,
3619 }, recorder.bytes.items);
3620 }
3621
3622 test "x86_64 vector cmp i64x2 slt emits packed SSE2" {
3623 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3624 defer ctx.deinit(std.testing.allocator);
3625 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3626
3627 const loc = ir.Location.getUnknown();
3628 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.int64)).?;
3629 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3630 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3631 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .slt, lhs.getResult(), rhs.getResult(), vec_type);
3632
3633 var recorder = BinaryRecorder{
3634 .allocator = std.testing.allocator,
3635 .lhs_value = lhs.getResult(),
3636 .lhs_slot = i64x2Slot(-8),
3637 .rhs_value = rhs.getResult(),
3638 .rhs_slot = i64x2Slot(-24),
3639 .result_value = cmp.getResult(),
3640 .result_slot = i64x2Slot(-40),
3641 };
3642 defer recorder.bytes.deinit(std.testing.allocator);
3643
3644 try emitCmp(&recorder, cmp.op);
3645
3646 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3647 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3648 try std.testing.expectEqualSlices(u8, &.{
3649 0x66, 0x0F, 0x10, 0x45, 0xE0,
3650 0x66, 0x0F, 0x10, 0x4D, 0xF0,
3651 0x66, 0x0F, 0x10, 0xD0, 0x66,
3652 0x0F, 0x10, 0xD9, 0x66, 0x0F,
3653 0x66, 0xD3, 0x66, 0x0F, 0x10,
3654 0xD8, 0x66, 0x0F, 0x76, 0xD9,
3655 0x66, 0x0F, 0x76, 0xE4, 0x66,
3656 0x0F, 0x72, 0xF4, 0x1F, 0x66,
3657 0x0F, 0xEF, 0xC4, 0x66, 0x0F,
3658 0xEF, 0xCC, 0x66, 0x0F, 0x66,
3659 0xC1, 0x66, 0x0F, 0x73, 0xF0,
3660 0x20, 0x66, 0x0F, 0xDB, 0xC3,
3661 0x66, 0x0F, 0xEB, 0xD0, 0x66,
3662 0x0F, 0x10, 0xC2, 0x66, 0x0F,
3663 0x73, 0xD0, 0x3F, 0x66, 0x0F,
3664 0x11, 0x45, 0xD0,
3665 }, recorder.bytes.items);
3666 }
3667
3668 test "x86_64 vector cmp u64x2 uge emits packed SSE2" {
3669 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3670 defer ctx.deinit(std.testing.allocator);
3671 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3672
3673 const loc = ir.Location.getUnknown();
3674 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3675 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3676 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3677 const cmp = try ArithDialect.VecCmpOp.create(&ctx, loc, .uge, lhs.getResult(), rhs.getResult(), vec_type);
3678
3679 var recorder = BinaryRecorder{
3680 .allocator = std.testing.allocator,
3681 .lhs_value = lhs.getResult(),
3682 .lhs_slot = u64x2Slot(-8),
3683 .rhs_value = rhs.getResult(),
3684 .rhs_slot = u64x2Slot(-24),
3685 .result_value = cmp.getResult(),
3686 .result_slot = u64x2Slot(-40),
3687 };
3688 defer recorder.bytes.deinit(std.testing.allocator);
3689
3690 try emitCmp(&recorder, cmp.op);
3691
3692 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3693 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3694 try std.testing.expectEqualSlices(u8, &.{
3695 0x66, 0x0F, 0x10, 0x45, 0xE0,
3696 0x66, 0x0F, 0x10, 0x4D, 0xF0,
3697 0x66, 0x0F, 0x10, 0xD0, 0x66,
3698 0x0F, 0x10, 0xD9, 0x66, 0x0F,
3699 0x76, 0xE4, 0x66, 0x0F, 0x72,
3700 0xF4, 0x1F, 0x66, 0x0F, 0xEF,
3701 0xD4, 0x66, 0x0F, 0xEF, 0xDC,
3702 0x66, 0x0F, 0x66, 0xD3, 0x66,
3703 0x0F, 0x10, 0xD8, 0x66, 0x0F,
3704 0x76, 0xD9, 0x66, 0x0F, 0x76,
3705 0xE4, 0x66, 0x0F, 0x72, 0xF4,
3706 0x1F, 0x66, 0x0F, 0xEF, 0xC4,
3707 0x66, 0x0F, 0xEF, 0xCC, 0x66,
3708 0x0F, 0x66, 0xC1, 0x66, 0x0F,
3709 0x73, 0xF0, 0x20, 0x66, 0x0F,
3710 0xDB, 0xC3, 0x66, 0x0F, 0xEB,
3711 0xD0, 0x66, 0x0F, 0x10, 0xC2,
3712 0x66, 0x0F, 0x73, 0xD0, 0x3F,
3713 0x66, 0x0F, 0x76, 0xC9, 0x66,
3714 0x0F, 0x73, 0xD1, 0x3F, 0x66,
3715 0x0F, 0xEF, 0xC1, 0x66, 0x0F,
3716 0x11, 0x45, 0xD0,
3717 }, recorder.bytes.items);
3718 }
3719
3720 test "x86_64 vector max u32x4 emits packed SSE2" {
3721 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3722 defer ctx.deinit(std.testing.allocator);
3723 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3724
3725 const loc = ir.Location.getUnknown();
3726 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
3727 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3728 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3729 const maximum = try ArithDialect.MaxOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3730
3731 var recorder = BinaryRecorder{
3732 .allocator = std.testing.allocator,
3733 .lhs_value = lhs.getResult(),
3734 .lhs_slot = u32x4Slot(-8),
3735 .rhs_value = rhs.getResult(),
3736 .rhs_slot = u32x4Slot(-32),
3737 .result_value = maximum.getResult(),
3738 .result_slot = u32x4Slot(-56),
3739 };
3740 defer recorder.bytes.deinit(std.testing.allocator);
3741
3742 try emitMinMax(&recorder, maximum.op, .max);
3743
3744 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3745 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3746 try std.testing.expect(recorder.bytes.items.len > 0);
3747 }
3748
3749 test "x86_64 vector min u64x2 emits packed SSE2" {
3750 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3751 defer ctx.deinit(std.testing.allocator);
3752 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3753
3754 const loc = ir.Location.getUnknown();
3755 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3756 var lhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3757 var rhs = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 2);
3758 const minimum = try ArithDialect.MinOp.create(&ctx, loc, lhs.getResult(), rhs.getResult());
3759
3760 var recorder = BinaryRecorder{
3761 .allocator = std.testing.allocator,
3762 .lhs_value = lhs.getResult(),
3763 .lhs_slot = u64x2Slot(-8),
3764 .rhs_value = rhs.getResult(),
3765 .rhs_slot = u64x2Slot(-24),
3766 .result_value = minimum.getResult(),
3767 .result_slot = u64x2Slot(-40),
3768 };
3769 defer recorder.bytes.deinit(std.testing.allocator);
3770
3771 try emitMinMax(&recorder, minimum.op, .min);
3772
3773 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3774 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3775 try std.testing.expect(recorder.bytes.items.len > 0);
3776 }
3777
3778 test "x86_64 vector shuffle f64x2 emits packed SSE" {
3779 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3780 defer ctx.deinit(std.testing.allocator);
3781 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3782
3783 const loc = ir.Location.getUnknown();
3784 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
3785 var input = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
3786 const indices = [_]i64{ 1, 0 };
3787 const shuffle = try ArithDialect.VecShuffleOp.create(&ctx, loc, input.getResult(), vec_type, indices[0..]);
3788
3789 var recorder = BinaryRecorder{
3790 .allocator = std.testing.allocator,
3791 .lhs_value = input.getResult(),
3792 .lhs_slot = f64x2Slot(-8),
3793 .rhs_value = input.getResult(),
3794 .rhs_slot = f64x2Slot(-8),
3795 .result_value = shuffle.getResult(),
3796 .result_slot = f64x2Slot(-24),
3797 };
3798 defer recorder.bytes.deinit(std.testing.allocator);
3799
3800 try emitShuffle(&recorder, shuffle.op);
3801
3802 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3803 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3804 try std.testing.expectEqualSlices(u8, &.{
3805 0x66, 0x0F, 0x10, 0x45, 0xF0,
3806 0x66, 0x0F, 0xC6, 0xC0, 0x01,
3807 0x66, 0x0F, 0x11, 0x45, 0xE0,
3808 }, recorder.bytes.items);
3809 }
3810
3811 test "x86_64 vector shuffle f32x4 emits packed SSE" {
3812 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3813 defer ctx.deinit(std.testing.allocator);
3814 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3815
3816 const loc = ir.Location.getUnknown();
3817 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
3818 var input = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
3819 const indices = [_]i64{ 3, 2, 1, 0 };
3820 const shuffle = try ArithDialect.VecShuffleOp.create(&ctx, loc, input.getResult(), vec_type, indices[0..]);
3821
3822 var recorder = BinaryRecorder{
3823 .allocator = std.testing.allocator,
3824 .lhs_value = input.getResult(),
3825 .lhs_slot = f32x4Slot(-8),
3826 .rhs_value = input.getResult(),
3827 .rhs_slot = f32x4Slot(-8),
3828 .result_value = shuffle.getResult(),
3829 .result_slot = f32x4Slot(-32),
3830 };
3831 defer recorder.bytes.deinit(std.testing.allocator);
3832
3833 try emitShuffle(&recorder, shuffle.op);
3834
3835 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3836 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3837 try std.testing.expectEqualSlices(u8, &.{
3838 0x0F, 0x10, 0x45, 0xEC,
3839 0x0F, 0xC6, 0xC0, 0x1B,
3840 0x0F, 0x11, 0x45, 0xD4,
3841 }, recorder.bytes.items);
3842 }
3843
3844 test "x86_64 vector shuffle u32x4 emits packed SSE2" {
3845 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3846 defer ctx.deinit(std.testing.allocator);
3847 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3848
3849 const loc = ir.Location.getUnknown();
3850 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
3851 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3852 const indices = [_]i64{ 3, 2, 1, 0 };
3853 const shuffle = try ArithDialect.VecShuffleOp.create(&ctx, loc, input.getResult(), vec_type, indices[0..]);
3854
3855 var recorder = BinaryRecorder{
3856 .allocator = std.testing.allocator,
3857 .lhs_value = input.getResult(),
3858 .lhs_slot = u32x4Slot(-8),
3859 .rhs_value = input.getResult(),
3860 .rhs_slot = u32x4Slot(-8),
3861 .result_value = shuffle.getResult(),
3862 .result_slot = u32x4Slot(-32),
3863 };
3864 defer recorder.bytes.deinit(std.testing.allocator);
3865
3866 try emitShuffle(&recorder, shuffle.op);
3867
3868 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3869 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3870 try std.testing.expectEqualSlices(u8, &.{
3871 0x0F, 0x10, 0x45, 0xEC,
3872 0x66, 0x0F, 0x70, 0xC0,
3873 0x1B, 0x0F, 0x11, 0x45,
3874 0xD4,
3875 }, recorder.bytes.items);
3876 }
3877
3878 test "x86_64 vector shuffle u64x2 emits packed SSE2" {
3879 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3880 defer ctx.deinit(std.testing.allocator);
3881 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3882
3883 const loc = ir.Location.getUnknown();
3884 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
3885 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3886 const indices = [_]i64{ 1, 0 };
3887 const shuffle = try ArithDialect.VecShuffleOp.create(&ctx, loc, input.getResult(), vec_type, indices[0..]);
3888
3889 var recorder = BinaryRecorder{
3890 .allocator = std.testing.allocator,
3891 .lhs_value = input.getResult(),
3892 .lhs_slot = u64x2Slot(-8),
3893 .rhs_value = input.getResult(),
3894 .rhs_slot = u64x2Slot(-8),
3895 .result_value = shuffle.getResult(),
3896 .result_slot = u64x2Slot(-24),
3897 };
3898 defer recorder.bytes.deinit(std.testing.allocator);
3899
3900 try emitShuffle(&recorder, shuffle.op);
3901
3902 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3903 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3904 try std.testing.expectEqualSlices(u8, &.{
3905 0x66, 0x0F, 0x10, 0x45, 0xF0,
3906 0x66, 0x0F, 0xC6, 0xC0, 0x01,
3907 0x66, 0x0F, 0x11, 0x45, 0xE0,
3908 }, recorder.bytes.items);
3909 }
3910
3911 test "x86_64 vector neg f64x2 emits packed SSE" {
3912 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3913 defer ctx.deinit(std.testing.allocator);
3914 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3915
3916 const loc = ir.Location.getUnknown();
3917 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.float64)).?;
3918 var input = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
3919 const neg = try ArithDialect.NegOp.create(&ctx, loc, input.getResult());
3920
3921 var recorder = BinaryRecorder{
3922 .allocator = std.testing.allocator,
3923 .lhs_value = input.getResult(),
3924 .lhs_slot = f64x2Slot(-8),
3925 .rhs_value = input.getResult(),
3926 .rhs_slot = f64x2Slot(-8),
3927 .result_value = neg.getResult(),
3928 .result_slot = f64x2Slot(-24),
3929 };
3930 defer recorder.bytes.deinit(std.testing.allocator);
3931
3932 try emitNeg(&recorder, neg.op);
3933
3934 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3935 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3936 try std.testing.expectEqualSlices(u8, &.{
3937 0x66, 0x0F, 0x10, 0x45, 0xF0,
3938 0x66, 0x0F, 0x76, 0xC9, 0x66,
3939 0x0F, 0x73, 0xF1, 0x3F, 0x66,
3940 0x0F, 0x57, 0xC1, 0x66, 0x0F,
3941 0x11, 0x45, 0xE0,
3942 }, recorder.bytes.items);
3943 }
3944
3945 test "x86_64 vector neg f32x4 emits packed SSE" {
3946 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3947 defer ctx.deinit(std.testing.allocator);
3948 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3949
3950 const loc = ir.Location.getUnknown();
3951 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.float32)).?;
3952 var input = try ArithDialect.VecConstantOp.createFloat(&ctx, loc, vec_type, 1.5);
3953 const neg = try ArithDialect.NegOp.create(&ctx, loc, input.getResult());
3954
3955 var recorder = BinaryRecorder{
3956 .allocator = std.testing.allocator,
3957 .lhs_value = input.getResult(),
3958 .lhs_slot = f32x4Slot(-8),
3959 .rhs_value = input.getResult(),
3960 .rhs_slot = f32x4Slot(-8),
3961 .result_value = neg.getResult(),
3962 .result_slot = f32x4Slot(-32),
3963 };
3964 defer recorder.bytes.deinit(std.testing.allocator);
3965
3966 try emitNeg(&recorder, neg.op);
3967
3968 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
3969 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
3970 try std.testing.expectEqualSlices(u8, &.{
3971 0x0F, 0x10, 0x45, 0xEC,
3972 0x66, 0x0F, 0x76, 0xC9,
3973 0x66, 0x0F, 0x72, 0xF1,
3974 0x1F, 0x0F, 0x57, 0xC1,
3975 0x0F, 0x11, 0x45, 0xD4,
3976 }, recorder.bytes.items);
3977 }
3978
3979 test "x86_64 vector neg u32x4 emits packed SSE2" {
3980 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
3981 defer ctx.deinit(std.testing.allocator);
3982 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
3983
3984 const loc = ir.Location.getUnknown();
3985 const vec_type = (try ArithDialect.getVecType(&ctx, 4, dialects.arith.type_names.uint32)).?;
3986 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
3987 const neg = try ArithDialect.NegOp.create(&ctx, loc, input.getResult());
3988
3989 var recorder = BinaryRecorder{
3990 .allocator = std.testing.allocator,
3991 .lhs_value = input.getResult(),
3992 .lhs_slot = u32x4Slot(-8),
3993 .rhs_value = input.getResult(),
3994 .rhs_slot = u32x4Slot(-8),
3995 .result_value = neg.getResult(),
3996 .result_slot = u32x4Slot(-32),
3997 };
3998 defer recorder.bytes.deinit(std.testing.allocator);
3999
4000 try emitNeg(&recorder, neg.op);
4001
4002 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
4003 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
4004 try std.testing.expectEqualSlices(u8, &.{
4005 0x66, 0x0F, 0xEF, 0xC0,
4006 0x0F, 0x10, 0x4D, 0xEC,
4007 0x66, 0x0F, 0xFA, 0xC1,
4008 0x0F, 0x11, 0x45, 0xD4,
4009 }, recorder.bytes.items);
4010 }
4011
4012 test "x86_64 vector neg u64x2 emits packed SSE2" {
4013 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
4014 defer ctx.deinit(std.testing.allocator);
4015 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
4016
4017 const loc = ir.Location.getUnknown();
4018 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
4019 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
4020 const neg = try ArithDialect.NegOp.create(&ctx, loc, input.getResult());
4021
4022 var recorder = BinaryRecorder{
4023 .allocator = std.testing.allocator,
4024 .lhs_value = input.getResult(),
4025 .lhs_slot = u64x2Slot(-8),
4026 .rhs_value = input.getResult(),
4027 .rhs_slot = u64x2Slot(-8),
4028 .result_value = neg.getResult(),
4029 .result_slot = u64x2Slot(-24),
4030 };
4031 defer recorder.bytes.deinit(std.testing.allocator);
4032
4033 try emitNeg(&recorder, neg.op);
4034
4035 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
4036 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
4037 try std.testing.expectEqualSlices(u8, &.{
4038 0x66, 0x0F, 0xEF, 0xC0,
4039 0x66, 0x0F, 0x10, 0x4D,
4040 0xF0, 0x66, 0x0F, 0xFB,
4041 0xC1, 0x66, 0x0F, 0x11,
4042 0x45, 0xE0,
4043 }, recorder.bytes.items);
4044 }
4045
4046 test "x86_64 vector not u64x2 emits packed SSE2" {
4047 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
4048 defer ctx.deinit(std.testing.allocator);
4049 try @import("../../dialects/root.zig").registerAllDialects(&ctx);
4050
4051 const loc = ir.Location.getUnknown();
4052 const vec_type = (try ArithDialect.getVecType(&ctx, 2, dialects.arith.type_names.uint64)).?;
4053 var input = try ArithDialect.VecConstantOp.createInt(&ctx, loc, vec_type, 1);
4054 const bitwise = try ArithDialect.NotOp.create(&ctx, loc, input.getResult());
4055
4056 var recorder = BinaryRecorder{
4057 .allocator = std.testing.allocator,
4058 .lhs_value = input.getResult(),
4059 .lhs_slot = u64x2Slot(-8),
4060 .rhs_value = input.getResult(),
4061 .rhs_slot = u64x2Slot(-8),
4062 .result_value = bitwise.getResult(),
4063 .result_slot = u64x2Slot(-24),
4064 };
4065 defer recorder.bytes.deinit(std.testing.allocator);
4066
4067 try emitNot(&recorder, bitwise.op);
4068
4069 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_loads);
4070 try std.testing.expectEqual(@as(usize, 0), recorder.scalar_stores);
4071 try std.testing.expectEqualSlices(u8, &.{
4072 0x66, 0x0F, 0x10, 0x45, 0xF0,
4073 0x66, 0x0F, 0x76, 0xC9, 0x66,
4074 0x0F, 0xEF, 0xC1, 0x66, 0x0F,
4075 0x11, 0x45, 0xE0,
4076 }, recorder.bytes.items);
4077 }