lib/choir/src/backends/x64/memory.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const ir = @import("../../core/root.zig");
3 const dialects = @import("../../dialects/root.zig");
4 const call_plan = @import("calls.zig");
5 const data = @import("data.zig");
6 const encoding = @import("encoding.zig");
7 const registers = @import("registers/root.zig");
8 const simd = @import("vector.zig");
9 const slot_layout = @import("slots.zig");
10
11 const memref = dialects.memref;
12 const ArithDialect = dialects.arith.ArithDialect;
13 const MemrefDialect = dialects.memref.MemrefDialect;
14 const Mem = encoding.Mem;
15
16 /// The registers an element access may use as scratch. They are the ones
17 /// `regalloc.recordScalarScratchClobbers` declares clobbered at a `memref.load` and a
18 /// `memref.store`, so the allocator keeps nothing live across the access in them and choosing
19 /// among them can overwrite nothing it planned.
20 const load_scratch = [_]registers.GPR{ .rax, .rcx };
21 const store_scratch = [_]registers.GPR{ .rax, .rcx, .rdx };
22
23 /// A scratch register that is none of the registers this access already holds.
24 ///
25 /// A register the allocator does not keep live across the access can still hold an operand OF the
26 /// access, because an interval ending at the access is allowed to sit in a register clobbered
27 /// there. So a fixed choice can destroy the very value it is about to read: an index rematerialised
28 /// into rax, then a base loaded into rax, addresses `base + base * 8`. The registers in hand are
29 /// passed in and the choice avoids every one of them.
30 fn scratchAvoiding(taken: []const ?registers.GPR, candidates: []const registers.GPR) !registers.GPR {
31 candidate: for (candidates) |candidate| {
32 for (taken) |maybe_reg| {
33 if (maybe_reg) |reg| {
34 if (reg == candidate) continue :candidate;
35 }
36 }
37 return candidate;
38 }
39 return error.NoScratchRegister;
40 }
41
42 /// One element's address, with the registers it reads it from, so a caller placing a third value
43 /// (the stored one) can avoid them too.
44 const ElementAddress = struct {
45 mem: Mem,
46 base: registers.GPR,
47 index: registers.GPR,
48 };
49
50 /// `reserved` names registers holding values this access still needs and that it does not address
51 /// through, which is the stored value of a `memref.store`. They are avoided for the same reason the
52 /// index and the base avoid each other.
53 fn elementAddress(
54 self: anytype,
55 memref_val: *ir.Value,
56 index_val: *ir.Value,
57 scale: encoding.Scale,
58 candidates: []const registers.GPR,
59 reserved: ?registers.GPR,
60 ) !ElementAddress {
61 const index_home = self.registerHome(index_val);
62 const base_reg = self.registerHome(memref_val) orelse base: {
63 const scratch = try scratchAvoiding(&.{ index_home, reserved }, candidates);
64 try self.loadSlot(try self.slotFor(memref_val), scratch);
65 break :base scratch;
66 };
67 const index_reg = index_home orelse index: {
68 const scratch = try scratchAvoiding(&.{ base_reg, reserved }, candidates);
69 try self.loadSlot(try self.slotFor(index_val), scratch);
70 break :index scratch;
71 };
72 return .{
73 .mem = Mem.baseIndex(base_reg, index_reg, scale, 0),
74 .base = base_reg,
75 .index = index_reg,
76 };
77 }
78
79 pub fn emitLoad(self: anytype, op: *ir.Operation) !void {
80 const load_op = MemrefDialect.LoadOp{ .op = op };
81 const result = load_op.getResult();
82 const memref_val = load_op.getMemref();
83 const index_val = load_op.getIndex();
84
85 const layout = try parseParams(self, memref_val.type);
86 try requireHost(self, layout);
87 const elem_size = try elementByteSize(self, layout.element_type_name);
88 const scale = try scaleForSize(elem_size);
89
90 const address = try elementAddress(self, memref_val, index_val, scale, &load_scratch, null);
91 const mem = address.mem;
92 if (try vectorSlotForOptional(self, result)) |result_vector| {
93 const kind = try packedMemrefKind(layout, result_vector);
94 if (self.xmmHomeForPhase(result, .definition)) |home| {
95 try simd.emitPackedLoadFromMem(self, kind, mem, home);
96 } else {
97 try simd.emitPackedLoadFromMem(self, kind, mem, .xmm0);
98 try simd.emitPackedStore(self, result_vector, .xmm0);
99 }
100 return;
101 }
102
103 const result_slot = try self.slotFor(result);
104 const result_home = self.registerHomeForPhase(result, .definition);
105 const dest = result_home orelse registers.GPR.rax;
106 if (elem_size == 1) {
107 if (!std.mem.eql(u8, layout.element_type_name, "arith.i8") and
108 !std.mem.eql(u8, layout.element_type_name, "arith.u8") and
109 !std.mem.eql(u8, layout.element_type_name, "arith.bool")) return error.UnsupportedType;
110 try self.emitEncoding(encoding.movzxRegMem8(dest, mem));
111 if (result_home == null) try self.storeSlot(result_slot, dest);
112 } else if (elem_size == 2) {
113 if (!std.mem.eql(u8, layout.element_type_name, "arith.i16") and
114 !std.mem.eql(u8, layout.element_type_name, "arith.u16")) return error.UnsupportedType;
115 try self.emitEncoding(encoding.movzxRegMem16(dest, mem));
116 if (result_home == null) try self.storeSlot(result_slot, dest);
117 } else if (elem_size == 4) {
118 if (result_slot.width == 32 and std.mem.eql(u8, layout.element_type_name, "arith.f32")) {
119 if (self.xmmHomeForPhase(result, .definition)) |home| {
120 try self.emitEncoding(encoding.movss(home, .{ .mem = mem }));
121 } else {
122 try self.emitEncoding(encoding.movss(.xmm0, .{ .mem = mem }));
123 try self.storeSlotXmm(result_slot, .xmm0);
124 }
125 } else {
126 try self.emitEncoding(encoding.movRegMem32(dest, mem));
127 if (result_home == null) try self.storeSlot(result_slot, dest);
128 }
129 } else if (elem_size == 8) {
130 if (result_slot.width == 64 and std.mem.eql(u8, layout.element_type_name, "arith.f64")) {
131 if (self.xmmHomeForPhase(result, .definition)) |home| {
132 try self.emitEncoding(encoding.movsd(home, .{ .mem = mem }));
133 } else {
134 try self.emitEncoding(encoding.movsd(.xmm0, .{ .mem = mem }));
135 try self.storeSlotXmm(result_slot, .xmm0);
136 }
137 } else {
138 try self.emitEncoding(encoding.movRegMem(dest, mem));
139 if (result_home == null) try self.storeSlot(result_slot, dest);
140 }
141 } else {
142 return error.UnsupportedType;
143 }
144 }
145
146 pub fn emitStore(self: anytype, op: *ir.Operation) !void {
147 const store_op = MemrefDialect.StoreOp{ .op = op };
148 const value = store_op.getValue();
149 const memref_val = store_op.getMemref();
150 const index_val = store_op.getIndex();
151
152 const layout = try parseParams(self, memref_val.type);
153 try requireHost(self, layout);
154 const elem_size = try elementByteSize(self, layout.element_type_name);
155 const scale = try scaleForSize(elem_size);
156
157 const value_home = self.registerHome(value);
158 const address = try elementAddress(self, memref_val, index_val, scale, &store_scratch, value_home);
159 const mem = address.mem;
160 const value_scratch = try scratchAvoiding(&.{ address.base, address.index }, &store_scratch);
161 if (try vectorSlotForOptional(self, value)) |value_vector| {
162 const kind = try packedMemrefKind(layout, value_vector);
163 if (self.xmmHome(value)) |home| {
164 try simd.emitPackedStoreToMem(self, kind, mem, home);
165 } else {
166 try simd.emitPackedLoad(self, value_vector, .xmm0);
167 try simd.emitPackedStoreToMem(self, kind, mem, .xmm0);
168 }
169 return;
170 }
171
172 const value_slot = try self.slotFor(value);
173 if (elem_size == 1) {
174 if (!std.mem.eql(u8, layout.element_type_name, "arith.i8") and
175 !std.mem.eql(u8, layout.element_type_name, "arith.u8") and
176 !std.mem.eql(u8, layout.element_type_name, "arith.bool")) return error.UnsupportedType;
177 try self.loadInto(value, value_scratch);
178 try self.emitEncoding(encoding.movMemReg8(mem, value_scratch));
179 } else if (elem_size == 2) {
180 if (!std.mem.eql(u8, layout.element_type_name, "arith.i16") and
181 !std.mem.eql(u8, layout.element_type_name, "arith.u16")) return error.UnsupportedType;
182 try self.loadInto(value, value_scratch);
183 try self.emitEncoding(encoding.movMemReg16(mem, value_scratch));
184 } else if (elem_size == 4) {
185 if (value_slot.width == 32 and std.mem.eql(u8, layout.element_type_name, "arith.f32")) {
186 if (self.xmmHome(value)) |home| {
187 try self.emitEncoding(encoding.movssStore(mem, home));
188 } else {
189 try self.loadSlotXmm(value_slot, .xmm0);
190 try self.emitEncoding(encoding.movssStore(mem, .xmm0));
191 }
192 } else {
193 const value_reg = self.registerHome(value) orelse reg: {
194 try self.loadSlot(value_slot, value_scratch);
195 break :reg value_scratch;
196 };
197 try self.emitEncoding(encoding.movMemReg32(mem, value_reg));
198 }
199 } else if (elem_size == 8) {
200 if (value_slot.width == 64 and std.mem.eql(u8, layout.element_type_name, "arith.f64")) {
201 if (self.xmmHome(value)) |home| {
202 try self.emitEncoding(encoding.movsdStore(mem, home));
203 } else {
204 try self.loadSlotXmm(value_slot, .xmm0);
205 try self.emitEncoding(encoding.movsdStore(mem, .xmm0));
206 }
207 } else {
208 const value_reg = self.registerHome(value) orelse reg: {
209 try self.loadSlot(value_slot, value_scratch);
210 break :reg value_scratch;
211 };
212 try self.emitEncoding(encoding.movMemReg(mem, value_reg));
213 }
214 } else {
215 return error.UnsupportedType;
216 }
217 }
218
219 pub fn emitAlloc(self: anytype, op: *ir.Operation) !void {
220 const alloc_op = MemrefDialect.AllocOp{ .op = op };
221 const result = alloc_op.getResult();
222 const layout = try parseParams(self, result.type);
223 try requireHost(self, layout);
224 const elem_size = try elementByteSize(self, layout.element_type_name);
225
226 const result_slot = try self.slotFor(result);
227 const args = [_]call_plan.ExternArg{
228 .{ .type_name = "arith.index", .source = .{ .gpr = .rdi } },
229 };
230
231 if (alloc_op.getDynamicSize()) |dyn_size| {
232 const size_slot = try self.slotFor(dyn_size);
233 try self.loadSlot(size_slot, .rax);
234 if (elem_size != 1) {
235 try self.emitEncoding(encoding.movRegImm64(.rcx, elem_size));
236 try self.emitEncoding(encoding.imulRegReg(.rax, .rcx));
237 }
238 try self.emitEncoding(encoding.movRegReg(.rdi, .rax));
239 } else {
240 const bytes = try staticByteSize(self, layout);
241 try self.emitEncoding(encoding.movRegImm64(.rdi, bytes));
242 }
243
244 const allocation = [_]call_plan.ExternResult{
245 .{ .type_name = MemrefDialect.name, .slot = result_slot },
246 };
247 try call_plan.emitExtern(self, "malloc", &args, &allocation);
248 try self.allocation_kinds.put(self.allocator, result, .heap);
249 }
250
251 /// Materializes the address of a global as a memref value.
252 ///
253 /// The address is a 64-bit immediate a relocation fills, which is the mechanism the read only
254 /// data path already uses. Registering the symbol here is what puts a global the code actually
255 /// reaches into the image, so a JIT compiling one function needs no module walk.
256 pub fn emitGetGlobal(self: anytype, op: *ir.Operation) !void {
257 const get_op = MemrefDialect.GetGlobalOp{ .op = op };
258 const name = get_op.getSymName() orelse return error.InvalidDataSymbol;
259 const global = memref.findGlobal(op, name) orelse return error.InvalidDataSymbol;
260 const symbol = try data.symbolOfGlobal(global);
261 try data.emitAddress(self, get_op.getResult(), symbol);
262 }
263
264 /// Adds an unscaled byte offset to a byte addressed base.
265 ///
266 /// One `add`, because the offset is already in bytes. The x86 addressing mode cannot help here:
267 /// its scale field only multiplies by 1, 2, 4 or 8 and `elementAddress` fixes the displacement
268 /// at zero, so the sum is formed in a register and the result is an ordinary memref value.
269 pub fn emitView(self: anytype, op: *ir.Operation) !void {
270 const view_op = MemrefDialect.ViewOp{ .op = op };
271 const base = view_op.getBase();
272 try requireHost(self, try parseParams(self, base.type));
273 try requireHost(self, try parseParams(self, view_op.getResult().type));
274 try self.loadInto(base, .rax);
275 try self.loadInto(view_op.getByteOffset(), .rcx);
276 try self.emitEncoding(encoding.addRegReg(.rax, .rcx));
277 try self.storeFrom(view_op.getResult(), .rax);
278 }
279
280 pub fn emitAlloca(self: anytype, op: *ir.Operation) !void {
281 const alloca_op = MemrefDialect.AllocaOp{ .op = op };
282 const result = alloca_op.getResult();
283 const result_slot = try self.slotFor(result);
284 const payload_slot = self.alloca_payload_slots.get(result) orelse return error.MissingSlot;
285
286 try self.emitEncoding(encoding.movRegReg(.rax, .rbp));
287 if (payload_slot.offset != 0) {
288 try self.emitEncoding(encoding.addRegImm(.rax, payload_slot.offset));
289 }
290 try self.storeSlot(result_slot, .rax);
291 try self.allocation_kinds.put(self.allocator, result, .stack_emulated);
292 }
293
294 pub fn emitDealloc(self: anytype, op: *ir.Operation) !void {
295 const dealloc_op = MemrefDialect.DeallocOp{ .op = op };
296 const memref_val = dealloc_op.getMemref();
297 const allocation_kind = self.allocation_kinds.get(memref_val) orelse return error.UnsupportedDeallocTarget;
298 if (allocation_kind == .stack_emulated) return;
299
300 const memref_slot = try self.slotFor(memref_val);
301 try self.loadSlot(memref_slot, .rdi);
302 const args = [_]call_plan.ExternArg{
303 .{ .type_name = MemrefDialect.name, .source = .{ .gpr = .rdi } },
304 };
305 try call_plan.emitExtern(self, "free", &args, &.{});
306 }
307
308 /// Atomic accesses address memory through fixed scratch registers: the base in rcx, the index in
309 /// rdx, and the operand in r8, leaving rax free for the value `cmpxchg` compares and answers.
310 const atomic_base = registers.GPR.rcx;
311 const atomic_index = registers.GPR.rdx;
312 const atomic_operand = registers.GPR.r8;
313
314 /// The two widths one x86 instruction reads or writes atomically when aligned.
315 const AtomicWidth = enum { word, half };
316
317 fn atomicWidth(self: anytype, layout: MemrefDialect.MemrefParams) !AtomicWidth {
318 try requireHost(self, layout);
319 const name = layout.element_type_name;
320 if (std.mem.eql(u8, name, "arith.i64") or
321 std.mem.eql(u8, name, "arith.u64") or
322 std.mem.eql(u8, name, "arith.index")) return .word;
323 if (std.mem.eql(u8, name, "arith.i32") or
324 std.mem.eql(u8, name, "arith.u32")) return .half;
325 return error.UnsupportedType;
326 }
327
328 /// A function holding an ordered access runs every value from its frame slot, so the fixed scratch
329 /// registers above clobber nothing live. `regalloc.holdsOrderedEffect` is what keeps that true: it
330 /// refuses such a function by its effect record rather than by the absence of a mnemonic from a
331 /// list, so an ordered operation added later inherits the refusal.
332 ///
333 /// This is the check that the premise still holds when an atomic is emitted. It REFUSES rather than
334 /// asserts, because an assertion is nothing in ReleaseFast and what follows a violation there is a
335 /// scratch register written over a live value, which is a wrong answer and not a crash.
336 fn requireFrameSlotsOnly(self: anytype) !void {
337 if (self.value_locations.count() != 0 or
338 self.value_location_ranges.items.len != 0 or
339 self.xmm_locations.count() != 0 or
340 self.xmm_location_ranges.items.len != 0)
341 {
342 return error.OrderedAccessWithAllocatedValues;
343 }
344 }
345
346 fn atomicAddress(self: anytype, memref_val: *ir.Value, index_val: *ir.Value, width: AtomicWidth) !Mem {
347 try requireFrameSlotsOnly(self);
348 try self.loadSlot(try self.slotFor(memref_val), atomic_base);
349 try self.loadSlot(try self.slotFor(index_val), atomic_index);
350 const scale: encoding.Scale = switch (width) {
351 .word => .eight,
352 .half => .four,
353 };
354 return Mem.baseIndex(atomic_base, atomic_index, scale, 0);
355 }
356
357 /// An aligned `mov` is already an acquire load under x86 TSO, and a sequentially consistent load
358 /// needs nothing more because every sequentially consistent store is an `xchg`.
359 pub fn emitAtomicLoad(self: anytype, op: *ir.Operation) !void {
360 const load_op = MemrefDialect.AtomicLoadOp{ .op = op };
361 _ = load_op.getOrdering() orelse return error.MissingOrdering;
362 const layout = try parseParams(self, load_op.getMemref().type);
363 const width = try atomicWidth(self, layout);
364 const mem = try atomicAddress(self, load_op.getMemref(), load_op.getIndex(), width);
365 switch (width) {
366 .word => try self.emitEncoding(encoding.movRegMem(.rax, mem)),
367 .half => try self.emitEncoding(encoding.movRegMem32(.rax, mem)),
368 }
369 try self.storeSlot(try self.slotFor(load_op.getResult()), .rax);
370 }
371
372 /// A release store is a plain `mov` under x86 TSO. A sequentially consistent store is an `xchg`,
373 /// whose implicit lock orders it before every later load.
374 pub fn emitAtomicStore(self: anytype, op: *ir.Operation) !void {
375 const store_op = MemrefDialect.AtomicStoreOp{ .op = op };
376 const ordering = store_op.getOrdering() orelse return error.MissingOrdering;
377 const layout = try parseParams(self, store_op.getMemref().type);
378 const width = try atomicWidth(self, layout);
379 const mem = try atomicAddress(self, store_op.getMemref(), store_op.getIndex(), width);
380 try self.loadSlot(try self.slotFor(store_op.getValue()), atomic_operand);
381 const sequential = ordering == .seq_cst;
382 switch (width) {
383 .word => try self.emitEncoding(if (sequential)
384 encoding.xchgMemReg(mem, atomic_operand)
385 else
386 encoding.movMemReg(mem, atomic_operand)),
387 .half => try self.emitEncoding(if (sequential)
388 encoding.xchgMemReg32(mem, atomic_operand)
389 else
390 encoding.movMemReg32(mem, atomic_operand)),
391 }
392 }
393
394 /// Every ordering lowers to `lock cmpxchg`, which is a full barrier on x86. The instruction
395 /// leaves the observed word in rax whether or not the exchange happened, and that word is the
396 /// result: the exchange succeeded exactly when it equals the expected value.
397 pub fn emitAtomicCas(self: anytype, op: *ir.Operation) !void {
398 const cas_op = MemrefDialect.AtomicCasOp{ .op = op };
399 const layout = try parseParams(self, cas_op.getMemref().type);
400 const width = try atomicWidth(self, layout);
401 const mem = try atomicAddress(self, cas_op.getMemref(), cas_op.getIndex(), width);
402 try self.loadSlot(try self.slotFor(cas_op.getExpected()), .rax);
403 try self.loadSlot(try self.slotFor(cas_op.getDesired()), atomic_operand);
404 switch (width) {
405 .word => try self.emitEncoding(encoding.lockCmpxchgMemReg(mem, atomic_operand)),
406 .half => try self.emitEncoding(encoding.lockCmpxchgMemReg32(mem, atomic_operand)),
407 }
408 try self.storeSlot(try self.slotFor(cas_op.getResult()), .rax);
409 }
410
411 /// Only a sequentially consistent fence emits code on x86, an `mfence` that keeps earlier stores
412 /// ahead of later loads. TSO already gives acquire, release, and acquire-release fences. Every
413 /// scope is treated as the whole system, the strongest reading of each.
414 pub fn emitFence(self: anytype, op: *ir.Operation) !void {
415 const fence_op = MemrefDialect.FenceOp{ .op = op };
416 _ = fence_op.getScope() orelse return error.MissingOrdering;
417 const ordering = fence_op.getOrdering() orelse return error.MissingOrdering;
418 if (ordering == .seq_cst) try self.emitEncoding(encoding.mfence());
419 }
420
421 fn parseParams(self: anytype, memref_type: ir.Type) !MemrefDialect.MemrefParams {
422 _ = self;
423 const param_key = memref_type.getDialectParamKey() orelse return error.MemrefTypeMissingParams;
424 return MemrefDialect.parseMemrefParams(param_key) orelse return error.InvalidMemrefType;
425 }
426
427 fn elementByteSize(self: anytype, element_type_name: []const u8) !u64 {
428 _ = self;
429 if (std.mem.eql(u8, element_type_name, "arith.i32") or
430 std.mem.eql(u8, element_type_name, "arith.u32") or
431 std.mem.eql(u8, element_type_name, "arith.f32")) return 4;
432 if (std.mem.eql(u8, element_type_name, "arith.i64") or
433 std.mem.eql(u8, element_type_name, "arith.u64") or
434 std.mem.eql(u8, element_type_name, "arith.f64") or
435 std.mem.eql(u8, element_type_name, "arith.index"))
436 {
437 return 8;
438 }
439 if (std.mem.eql(u8, element_type_name, "arith.i8") or
440 std.mem.eql(u8, element_type_name, "arith.u8") or
441 std.mem.eql(u8, element_type_name, "arith.bool")) return 1;
442 if (std.mem.eql(u8, element_type_name, "arith.i16") or
443 std.mem.eql(u8, element_type_name, "arith.u16") or
444 std.mem.eql(u8, element_type_name, "arith.f16") or
445 std.mem.eql(u8, element_type_name, "arith.bf16")) return 2;
446 return error.UnsupportedType;
447 }
448
449 fn staticByteSize(self: anytype, layout: MemrefDialect.MemrefParams) !u64 {
450 const size = layout.size orelse return error.DynamicMemref;
451 const elem_bytes = try elementByteSize(self, layout.element_type_name);
452 return size * elem_bytes;
453 }
454
455 pub fn allocaByteSize(self: anytype, alloca_op: MemrefDialect.AllocaOp) !u64 {
456 const result = alloca_op.getResult();
457 const layout = try parseParams(self, result.type);
458 try requireHost(self, layout);
459 if (layout.size) |_| return staticByteSize(self, layout);
460
461 const dyn_size = alloca_op.getDynamicSize() orelse return error.DynamicMemref;
462 const count = try constantIndexValue(self, dyn_size);
463 const elem_bytes = try elementByteSize(self, layout.element_type_name);
464 if (count > std.math.maxInt(u64) / elem_bytes) return error.DynamicMemref;
465 return count * elem_bytes;
466 }
467
468 pub fn allocaAlignment(self: anytype, alloca_op: MemrefDialect.AllocaOp) !u64 {
469 const result = alloca_op.getResult();
470 const layout = try parseParams(self, result.type);
471 try requireHost(self, layout);
472 const alignment = layout.alignment orelse abiDefaultAlignment(try elementByteSize(self, layout.element_type_name));
473 if (alignment == 0) return error.InvalidMemrefType;
474 return alignment;
475 }
476
477 fn abiDefaultAlignment(size: u64) u64 {
478 if (size <= 1) return 1;
479 if (size <= 2) return 2;
480 if (size <= 4) return 4;
481 return 8;
482 }
483
484 fn constantIndexValue(self: anytype, value: *ir.Value) !u64 {
485 _ = self;
486 const defining_op_opaque = value.getDefiningOp() orelse return error.DynamicMemref;
487 const defining_op: *ir.Operation = @ptrCast(@alignCast(defining_op_opaque));
488 if (!std.mem.eql(u8, defining_op.name.name, ArithDialect.ConstantOp.operation_name)) {
489 return error.DynamicMemref;
490 }
491 if (data.hasAttributes(defining_op)) return error.DynamicMemref;
492 const int_value = (ArithDialect.ConstantOp{ .op = defining_op }).getIntValue() orelse return error.InvalidConstant;
493 if (int_value < 0) return error.DynamicMemref;
494 return @intCast(int_value);
495 }
496
497 fn requireHost(self: anytype, layout: MemrefDialect.MemrefParams) !void {
498 _ = self;
499 if (layout.addr_space != .host) return error.UnsupportedMemrefAddressSpace;
500 }
501
502 fn scaleForSize(size: u64) !encoding.Scale {
503 return switch (size) {
504 1 => .one,
505 2 => .two,
506 4 => .four,
507 8 => .eight,
508 else => error.UnsupportedType,
509 };
510 }
511
512 fn vectorSlotForOptional(self: anytype, value: *ir.Value) !?slot_layout.VectorSlot {
513 return self.vectorSlotFor(value) catch |err| switch (err) {
514 error.MissingSlot => null,
515 else => return err,
516 };
517 }
518
519 fn packedMemrefKind(layout: MemrefDialect.MemrefParams, vector_slot: slot_layout.VectorSlot) !simd.PackedKind {
520 if (!std.mem.eql(u8, layout.element_type_name, vector_slot.elem_type_name)) return error.UnsupportedType;
521 return simd.packedKind(vector_slot) orelse error.UnsupportedType;
522 }
523
524 test "x86_64 memory owner classifies element sizes and scales" {
525 try std.testing.expectEqual(@as(u64, 4), try elementByteSize({}, "arith.f32"));
526 try std.testing.expectEqual(@as(u64, 8), try elementByteSize({}, "arith.index"));
527 try std.testing.expectEqual(encoding.Scale.four, try scaleForSize(4));
528 try std.testing.expectError(error.UnsupportedType, scaleForSize(3));
529 }
530
531 test "an ordered access refuses a function whose values are register homed" {
532 const Homes = struct {
533 count_value: usize,
534
535 fn count(self: @This()) usize {
536 return self.count_value;
537 }
538 };
539 const Ranges = struct { items: []const u8 };
540 const Emitter = struct {
541 value_locations: Homes,
542 value_location_ranges: Ranges,
543 xmm_locations: Homes,
544 xmm_location_ranges: Ranges,
545 };
546
547 const frame_slots_only = Emitter{
548 .value_locations = .{ .count_value = 0 },
549 .value_location_ranges = .{ .items = &.{} },
550 .xmm_locations = .{ .count_value = 0 },
551 .xmm_location_ranges = .{ .items = &.{} },
552 };
553 try requireFrameSlotsOnly(frame_slots_only);
554
555 var one_value_homed = frame_slots_only;
556 one_value_homed.value_locations.count_value = 1;
557 try std.testing.expectError(
558 error.OrderedAccessWithAllocatedValues,
559 requireFrameSlotsOnly(one_value_homed),
560 );
561
562 var one_range_planned = frame_slots_only;
563 one_range_planned.value_location_ranges.items = &.{0};
564 try std.testing.expectError(
565 error.OrderedAccessWithAllocatedValues,
566 requireFrameSlotsOnly(one_range_planned),
567 );
568
569 var one_float_homed = frame_slots_only;
570 one_float_homed.xmm_locations.count_value = 1;
571 try std.testing.expectError(
572 error.OrderedAccessWithAllocatedValues,
573 requireFrameSlotsOnly(one_float_homed),
574 );
575
576 var one_float_range = frame_slots_only;
577 one_float_range.xmm_location_ranges.items = &.{0};
578 try std.testing.expectError(
579 error.OrderedAccessWithAllocatedValues,
580 requireFrameSlotsOnly(one_float_range),
581 );
582 }