lib/choir/src/dialects/memref.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const effects = ir.interfaces.effects;
3 const alloc_arena = @import("alloc_arena");
4 const alloc_observe = @import("alloc_observe");
5 const ir = @import("../core/root.zig");
6 const interfaces = @import("../core/root.zig").interfaces;
7 const arith = @import("arith/root.zig");
8
9 /// Refusals the memref verifiers name. Each one is a fact about the operation that was written,
10 /// never an assertion about the compiler, so user input reaches a named error rather than a trap.
11 pub const MemrefVerifyError = error{
12 GlobalMissingName,
13 GlobalMissingType,
14 GlobalTypeNotMemref,
15 GlobalTypeNotStatic,
16 GlobalMissingAlignment,
17 GlobalInvalidAlignment,
18 GlobalMissingConstant,
19 GlobalConstantWithoutInitial,
20 GlobalInitialLengthMismatch,
21 GetGlobalMissingName,
22 GetGlobalResultNotMemref,
23 ViewBaseNotMemref,
24 ViewBaseNotBytes,
25 ViewResultNotMemref,
26 ViewResultNotStatic,
27 AtomicOperandNotMemref,
28 AtomicElementNotWordOrHalfWord,
29 AtomicTypeMismatch,
30 AtomicOrderingMissing,
31 AtomicOrderingInvalid,
32 };
33
34 /// Where a global's storage comes from, which follows from its attributes rather than being
35 /// spelled separately.
36 pub const GlobalPlacement = enum {
37 /// Declared constant and carrying bytes: the bytes are in the image and never written.
38 read_only,
39 /// Carrying bytes that code may write.
40 writable,
41 /// Carrying no bytes at all: the loader supplies zeroes for the whole extent.
42 zeroed,
43 };
44
45 pub const AddressSpace = enum(u8) {
46 host = 0,
47 device = 1,
48 constant = 2,
49 shared = 3,
50 unified = 4,
51 local = 5,
52
53 pub fn toString(self: AddressSpace) []const u8 {
54 return switch (self) {
55 .host => "host",
56 .device => "device",
57 .constant => "constant",
58 .shared => "shared",
59 .unified => "unified",
60 .local => "local",
61 };
62 }
63
64 pub fn fromString(s: []const u8) ?AddressSpace {
65 if (std.mem.eql(u8, s, "host")) return .host;
66 if (std.mem.eql(u8, s, "device")) return .device;
67 if (std.mem.eql(u8, s, "constant")) return .constant;
68 if (std.mem.eql(u8, s, "shared")) return .shared;
69 if (std.mem.eql(u8, s, "unified")) return .unified;
70 if (std.mem.eql(u8, s, "local")) return .local;
71 return null;
72 }
73 };
74
75 pub const Indexing = enum(u8) {
76 i32,
77 i64,
78
79 pub fn toString(self: Indexing) []const u8 {
80 return @tagName(self);
81 }
82
83 pub fn fromString(s: []const u8) ?Indexing {
84 if (std.mem.eql(u8, s, "i32")) return .i32;
85 if (std.mem.eql(u8, s, "i64")) return .i64;
86 return null;
87 }
88 };
89
90 pub const CacheOperation = enum(u8) {
91 always,
92 global,
93 streaming,
94 last_use,
95 volatile_,
96 write_back,
97 write_through,
98 workgroup,
99
100 pub fn toString(self: CacheOperation) []const u8 {
101 return switch (self) {
102 .volatile_ => "volatile",
103 else => @tagName(self),
104 };
105 }
106
107 pub fn fromString(s: []const u8) ?CacheOperation {
108 if (std.mem.eql(u8, s, "always")) return .always;
109 if (std.mem.eql(u8, s, "global")) return .global;
110 if (std.mem.eql(u8, s, "streaming")) return .streaming;
111 if (std.mem.eql(u8, s, "last_use")) return .last_use;
112 if (std.mem.eql(u8, s, "volatile")) return .volatile_;
113 if (std.mem.eql(u8, s, "write_back")) return .write_back;
114 if (std.mem.eql(u8, s, "write_through")) return .write_through;
115 if (std.mem.eql(u8, s, "workgroup")) return .workgroup;
116 return null;
117 }
118 };
119
120 pub const CacheEviction = enum(u8) {
121 normal,
122 first,
123 last,
124 no_allocate,
125
126 pub fn toString(self: CacheEviction) []const u8 {
127 return @tagName(self);
128 }
129
130 pub fn fromString(s: []const u8) ?CacheEviction {
131 inline for (@typeInfo(CacheEviction).@"enum".field_names, std.meta.tags(CacheEviction)) |field_name, tag| {
132 if (std.mem.eql(u8, s, field_name)) {
133 return tag;
134 }
135 }
136 return null;
137 }
138 };
139
140 pub const FenceScope = enum(u8) {
141 system,
142 device,
143 workgroup,
144
145 pub fn toString(self: FenceScope) []const u8 {
146 return @tagName(self);
147 }
148
149 pub fn fromString(s: []const u8) ?FenceScope {
150 inline for (@typeInfo(FenceScope).@"enum".field_names, std.meta.tags(FenceScope)) |field_name, tag| {
151 if (std.mem.eql(u8, s, field_name)) {
152 return tag;
153 }
154 }
155 return null;
156 }
157 };
158
159 pub const FenceOrdering = enum(u8) {
160 acquire,
161 release,
162 acq_rel,
163 seq_cst,
164
165 pub fn toString(self: FenceOrdering) []const u8 {
166 return @tagName(self);
167 }
168
169 pub fn fromString(s: []const u8) ?FenceOrdering {
170 inline for (@typeInfo(FenceOrdering).@"enum".field_names, std.meta.tags(FenceOrdering)) |field_name, tag| {
171 if (std.mem.eql(u8, s, field_name)) {
172 return tag;
173 }
174 }
175 return null;
176 }
177 };
178
179 pub const AtomicRmwKind = enum(u8) {
180 add,
181 min,
182 max,
183 bit_and,
184 bit_or,
185 bit_xor,
186 exchange,
187
188 pub fn toString(self: AtomicRmwKind) []const u8 {
189 return @tagName(self);
190 }
191
192 pub fn fromString(s: []const u8) ?AtomicRmwKind {
193 inline for (@typeInfo(AtomicRmwKind).@"enum".field_names, std.meta.tags(AtomicRmwKind)) |field_name, tag| {
194 if (std.mem.eql(u8, s, field_name)) {
195 return tag;
196 }
197 }
198 return null;
199 }
200 };
201
202 pub const MemrefDialect = struct {
203 pub const name = "memref";
204 const op_specs = ir.dialects.opSpec.dialect(@This());
205 const op_templates = ir.dialects.operationTemplate.dialect(@This());
206 pub const spec = ir.dialects.dialectSpec(@This(), .{
207 .types = &.{ir.dialects.typeName(name)},
208 .type_interface_fallbacks = &.{
209 .{ .id = interfaces.TypeParamInterface.id, .fallback = typeParamFallback },
210 .{ .id = interfaces.ShapedTypeInterface.id, .fallback = shapedTypeFallback },
211 },
212 });
213
214 pub const MemrefTypePayload = struct {
215 size: ?u64,
216 element_type_name: []const u8,
217 element_type: ?ir.Type,
218 addr_space: AddressSpace,
219 alignment: ?u64,
220 exclusive: ?bool,
221 indexing: ?Indexing,
222 shape_storage: [1]u64 = [_]u64{0},
223 shape: ?[]const u64 = null,
224 };
225
226 const type_param_vtable = interfaces.TypeParamInterface.VTable{
227 .parse = parseTypeParams,
228 };
229
230 const shaped_type_vtable = interfaces.ShapedTypeInterface.VTable{
231 .getRank = shapedGetRank,
232 .getShape = shapedGetShape,
233 .getElementType = shapedGetElementType,
234 .getAddressSpaceTag = shapedGetAddressSpaceTag,
235 };
236
237 pub const MemrefTypeAttrs = struct {
238 alignment: ?u64 = null,
239 exclusive: ?bool = null,
240 indexing: ?Indexing = null,
241 };
242
243 pub const MemrefParams = struct {
244 size: ?u64,
245 element_type_name: []const u8,
246 addr_space: AddressSpace,
247 alignment: ?u64 = null,
248 exclusive: ?bool = null,
249 indexing: ?Indexing = null,
250 };
251
252 pub const LayoutAttrs = struct {
253 offset: ?u64 = null,
254 shape: ?[]const u64 = null,
255 stride: ?[]const u64 = null,
256 };
257
258 pub const AllocOp = struct {
259 op: *ir.Operation,
260
261 const leaf = op_templates.explicitLeaf(@This(), .{
262 .mnemonic = "alloc",
263 .interfaces = &.{allocationEffects("heap")},
264 .operands = ir.dialects.shape.atMost(1),
265 .operand_names = .{"dynamic_size"},
266 .results = .{"memref"},
267 });
268 pub const operation_spec = leaf.operation_spec;
269 pub const operation_name = leaf.operation_name;
270 pub const createLeaf = leaf.createLeaf;
271 pub const getOptionalOperand = leaf.getOptionalOperand;
272
273 pub fn createStatic(
274 ctx: *ir.Context,
275 loc: ir.Location,
276 result_type: ir.Type,
277 ) !AllocOp {
278 return @This().createLeaf(ctx, loc, &.{}, &.{result_type});
279 }
280
281 pub fn createDynamic(
282 ctx: *ir.Context,
283 loc: ir.Location,
284 size: *ir.Value,
285 result_type: ir.Type,
286 ) !AllocOp {
287 return @This().createLeaf(ctx, loc, &.{size}, &.{result_type});
288 }
289
290 pub fn getResult(self: *const AllocOp) *ir.Value {
291 return leaf.getResult(self.*);
292 }
293
294 pub fn getDynamicSize(self: AllocOp) ?*ir.Value {
295 return self.getOptionalOperand("dynamic_size");
296 }
297 };
298
299 /// Allocates per-invocation storage. Its contents are undefined until stored;
300 /// loading an element before storing it has no defined result on any backend.
301 /// The CPU twin maps a local alloca to host stack storage without zeroing it.
302 pub const AllocaOp = struct {
303 op: *ir.Operation,
304
305 const leaf = op_templates.explicitLeaf(@This(), .{
306 .mnemonic = "alloca",
307 .interfaces = &.{allocationEffects("stack")},
308 .operands = ir.dialects.shape.atMost(1),
309 .operand_names = .{"dynamic_size"},
310 .results = .{"memref"},
311 });
312 pub const operation_spec = leaf.operation_spec;
313 pub const operation_name = leaf.operation_name;
314 pub const createLeaf = leaf.createLeaf;
315 pub const getOptionalOperand = leaf.getOptionalOperand;
316
317 pub fn createStatic(
318 ctx: *ir.Context,
319 loc: ir.Location,
320 result_type: ir.Type,
321 ) !AllocaOp {
322 return @This().createLeaf(ctx, loc, &.{}, &.{result_type});
323 }
324
325 pub fn createDynamic(
326 ctx: *ir.Context,
327 loc: ir.Location,
328 size: *ir.Value,
329 result_type: ir.Type,
330 ) !AllocaOp {
331 return @This().createLeaf(ctx, loc, &.{size}, &.{result_type});
332 }
333
334 pub fn getResult(self: *const AllocaOp) *ir.Value {
335 return leaf.getResult(self.*);
336 }
337
338 pub fn getDynamicSize(self: AllocaOp) ?*ir.Value {
339 return self.getOptionalOperand("dynamic_size");
340 }
341 };
342
343 /// A module level declaration of storage that code addresses by name.
344 ///
345 /// The placement follows from two attributes rather than being spelled a third time.
346 /// `constant` with `initial` bytes is storage nothing writes, `initial` bytes without
347 /// `constant` is storage code may write, and neither is an extent the loader fills with
348 /// zeroes and the image carries no bytes for. `constant` with nothing to be constant about
349 /// is refused, because the only thing it could mean is a read only run of zeroes that no
350 /// one can ever have written.
351 ///
352 /// The type travels as a one element type list because the builtin attributes carry a list
353 /// of types and not a single one, and this operation has no result to carry it on.
354 pub const GlobalOp = struct {
355 op: *ir.Operation,
356
357 pub const attr_names = struct {
358 pub const sym_name = "sym_name";
359 pub const memref_type = "type";
360 pub const alignment = "alignment";
361 pub const constant = "constant";
362 pub const initial = "initial";
363 };
364
365 const leaf = op_templates.explicitLeaf(@This(), .{
366 .mnemonic = "global",
367 .operands = 0,
368 .results = 0,
369 .required_attrs = .{
370 ir.dialects.attribute.string(attr_names.sym_name),
371 ir.dialects.attribute.any(attr_names.memref_type),
372 ir.dialects.attribute.integer(attr_names.alignment),
373 ir.dialects.attribute.boolean(attr_names.constant),
374 },
375 .attrs = .{ir.dialects.attribute.string(attr_names.initial)},
376 .interfaces = &.{effects.EffectOpInterface.entryFor(.{ .complete = true })},
377 });
378 pub const operation_spec = leaf.operation_spec;
379 pub const operation_name = leaf.operation_name;
380 pub const createLeaf = leaf.createLeaf;
381 pub const verify = verifyGlobalOp;
382
383 pub const Declaration = struct {
384 sym_name: []const u8,
385 memref_type: ir.Type,
386 alignment: u64 = 1,
387 constant: bool = false,
388 /// Bytes the image carries. Absent declares an extent of zeroes instead.
389 initial: ?[]const u8 = null,
390 };
391
392 pub fn create(
393 ctx: *ir.Context,
394 loc: ir.Location,
395 declaration: Declaration,
396 ) !GlobalOp {
397 try loadSpec(ctx);
398 if (declaration.alignment > std.math.maxInt(i64)) {
399 return MemrefVerifyError.GlobalInvalidAlignment;
400 }
401 const self = try @This().createLeaf(ctx, loc, &.{}, &.{});
402 errdefer self.op.erase();
403 const names = attr_names;
404 try self.op.setAttr(names.sym_name, try ctx.getStringAttr(declaration.sym_name));
405 try self.op.setAttr(
406 names.memref_type,
407 try ctx.getTypeListAttr(&.{declaration.memref_type}),
408 );
409 try self.op.setAttr(
410 names.alignment,
411 try ctx.getI64Attr(@intCast(declaration.alignment)),
412 );
413 try self.op.setAttr(names.constant, try ctx.getBoolAttr(declaration.constant));
414 if (declaration.initial) |bytes| {
415 try self.op.setAttr(names.initial, try ctx.getStringAttr(bytes));
416 }
417 try verifyGlobal(self.op);
418 return self;
419 }
420
421 pub fn getSymName(self: GlobalOp) ?[]const u8 {
422 const attr = self.op.getAttrAs(ir.Attribute.StringAttr, attr_names.sym_name) orelse
423 return null;
424 return attr.getValue();
425 }
426
427 pub fn getType(self: GlobalOp) ?ir.Type {
428 const attr = self.op.getAttrAs(
429 ir.Attribute.TypeListAttr,
430 attr_names.memref_type,
431 ) orelse return null;
432 const values = attr.getValues();
433 if (values.len != 1) return null;
434 return values[0];
435 }
436
437 pub fn getAlignment(self: GlobalOp) ?u64 {
438 const attr = self.op.getAttrAs(ir.Attribute.IntegerAttr, attr_names.alignment) orelse
439 return null;
440 const value = attr.getValue();
441 if (value < 0) return null;
442 return @intCast(value);
443 }
444
445 pub fn isConstant(self: GlobalOp) ?bool {
446 const attr = self.op.getAttrAs(ir.Attribute.BoolAttr, attr_names.constant) orelse
447 return null;
448 return attr.getValue();
449 }
450
451 pub fn getInitial(self: GlobalOp) ?[]const u8 {
452 const attr = self.op.getAttrAs(ir.Attribute.StringAttr, attr_names.initial) orelse
453 return null;
454 return attr.getValue();
455 }
456
457 /// Where this global's storage comes from, or null when the attributes do not decide it.
458 pub fn getPlacement(self: GlobalOp) ?GlobalPlacement {
459 const constant = self.isConstant() orelse return null;
460 if (self.getInitial() == null) {
461 if (constant) return null;
462 return .zeroed;
463 }
464 return if (constant) .read_only else .writable;
465 }
466 };
467
468 /// The address of a global, as a memref value of that global's declared type.
469 ///
470 /// This computes an address and touches nothing, so it declares one result and no event.
471 /// Reading or writing through the result is what `memref.load` and `memref.store` declare.
472 pub const GetGlobalOp = struct {
473 op: *ir.Operation,
474
475 pub const attr_names = struct {
476 pub const sym_name = "sym_name";
477 };
478
479 const leaf = op_templates.explicitLeaf(@This(), .{
480 .mnemonic = "get_global",
481 .operands = 0,
482 .results = .{"memref"},
483 .required_attrs = .{ir.dialects.attribute.string(attr_names.sym_name)},
484 .interfaces = &.{effects.EffectOpInterface.entryFor(.{
485 .complete = true,
486 .facts = &.{.{ .result = .{ .index = 0, .ownership = .none } }},
487 })},
488 });
489 pub const operation_spec = leaf.operation_spec;
490 pub const operation_name = leaf.operation_name;
491 pub const createLeaf = leaf.createLeaf;
492 pub const verify = verifyGetGlobalOp;
493
494 pub fn create(
495 ctx: *ir.Context,
496 loc: ir.Location,
497 sym_name: []const u8,
498 result_type: ir.Type,
499 ) !GetGlobalOp {
500 try loadSpec(ctx);
501 const self = try @This().createLeaf(ctx, loc, &.{}, &.{result_type});
502 errdefer self.op.erase();
503 try self.op.setAttr(attr_names.sym_name, try ctx.getStringAttr(sym_name));
504 try verifyGetGlobal(self.op);
505 return self;
506 }
507
508 pub fn getResult(self: *const GetGlobalOp) *ir.Value {
509 return self.op.getResult(0).?;
510 }
511
512 pub fn getSymName(self: GetGlobalOp) ?[]const u8 {
513 const attr = self.op.getAttrAs(ir.Attribute.StringAttr, attr_names.sym_name) orelse
514 return null;
515 return attr.getValue();
516 }
517 };
518
519 pub const DeallocOp = struct {
520 op: *ir.Operation,
521
522 pub const operation_spec = op_specs.leaf(.{
523 .mnemonic = "dealloc",
524 .interfaces = &.{effects.EffectOpInterface.entryFor(.{ .facts = &.{
525 .{ .requirement = .{ .kind = .live, .subject = .{ .operand = 0 } } },
526 .{ .event = .{ .kind = .free, .resource = .{ .subject = .{ .operand = 0 } } } },
527 } })},
528 .operands = 1,
529 .results = 0,
530 });
531 pub const operation_name = operation_spec.name;
532
533 pub fn create(
534 ctx: *ir.Context,
535 loc: ir.Location,
536 memref: *ir.Value,
537 ) !DeallocOp {
538 var builder = ir.OperationBuilder.init(ctx);
539 var state = op_specs.state(@This(), loc);
540 state.addOperands(&.{memref});
541
542 const op = try builder.create(state);
543 return .{ .op = op };
544 }
545
546 pub fn getMemref(self: DeallocOp) *ir.Value {
547 return self.op.operands.items[0].value;
548 }
549 };
550
551 pub const FenceOp = struct {
552 op: *ir.Operation,
553
554 pub const operation_spec = op_specs.leaf(.{
555 .mnemonic = "fence",
556 .interfaces = &.{effects.EffectOpInterface.entryFor(.{ .facts = &.{.{
557 .event = .{ .kind = .synchronize, .ordered = true },
558 }} })},
559 .operands = 0,
560 .results = 0,
561 .attrs = &.{ "scope", "ordering" },
562 .traits = ir.OperationTraits{},
563 });
564 pub const operation_name = operation_spec.name;
565
566 pub fn create(
567 ctx: *ir.Context,
568 loc: ir.Location,
569 scope: FenceScope,
570 ordering: FenceOrdering,
571 ) !FenceOp {
572 var builder = ir.OperationBuilder.init(ctx);
573 const op = try builder.create(op_specs.state(@This(), loc));
574 errdefer op.erase();
575 try setFenceScopeAttr(op, ctx, scope);
576 try setFenceOrderingAttr(op, ctx, ordering);
577 return .{ .op = op };
578 }
579
580 pub fn getScope(self: FenceOp) ?FenceScope {
581 return getFenceScopeAttr(self.op);
582 }
583
584 pub fn getOrdering(self: FenceOp) ?FenceOrdering {
585 return getFenceOrderingAttr(self.op);
586 }
587 };
588
589 pub const LoadOp = struct {
590 op: *ir.Operation,
591
592 pub const operation_spec = op_specs.leaf(.{
593 .mnemonic = "load",
594 .interfaces = &.{accessEffects(0, 1, true, false, false)},
595 .operands = 2,
596 .results = 1,
597 .attrs = &.{ "cache", "eviction" },
598 .traits = ir.OperationTraits{},
599 });
600 pub const operation_name = operation_spec.name;
601
602 pub fn create(
603 ctx: *ir.Context,
604 loc: ir.Location,
605 memref: *ir.Value,
606 index: *ir.Value,
607 result_type: ir.Type,
608 ) !LoadOp {
609 var builder = ir.OperationBuilder.init(ctx);
610 var state = op_specs.state(@This(), loc);
611 state.addOperands(&.{ memref, index });
612 state.addTypes(&.{result_type});
613
614 const op = try builder.create(state);
615 return .{ .op = op };
616 }
617
618 pub fn createWithCache(
619 ctx: *ir.Context,
620 loc: ir.Location,
621 memref: *ir.Value,
622 index: *ir.Value,
623 result_type: ir.Type,
624 cache: ?CacheOperation,
625 eviction: ?CacheEviction,
626 ) !LoadOp {
627 const load = try create(ctx, loc, memref, index, result_type);
628 errdefer load.op.erase();
629 if (cache) |hint| {
630 try setCacheOperationAttr(load.op, ctx, hint);
631 }
632 if (eviction) |hint| {
633 try setCacheEvictionAttr(load.op, ctx, hint);
634 }
635 return load;
636 }
637
638 pub fn getResult(self: *const LoadOp) *ir.Value {
639 return self.op.getResult(0).?;
640 }
641
642 pub fn getMemref(self: LoadOp) *ir.Value {
643 return self.op.operands.items[0].value;
644 }
645
646 pub fn getIndex(self: LoadOp) *ir.Value {
647 return self.op.operands.items[1].value;
648 }
649
650 pub fn getCacheOperation(self: LoadOp) ?CacheOperation {
651 return getCacheOperationAttr(self.op);
652 }
653
654 pub fn getCacheEviction(self: LoadOp) ?CacheEviction {
655 return getCacheEvictionAttr(self.op);
656 }
657 };
658
659 pub const StoreOp = struct {
660 op: *ir.Operation,
661
662 pub const operation_spec = op_specs.leaf(.{
663 .mnemonic = "store",
664 .interfaces = &.{accessEffects(1, 2, false, true, false)},
665 .operands = 3,
666 .results = 0,
667 .attrs = &.{ "cache", "eviction" },
668 .traits = ir.OperationTraits{},
669 });
670 pub const operation_name = operation_spec.name;
671
672 pub fn create(
673 ctx: *ir.Context,
674 loc: ir.Location,
675 value: *ir.Value,
676 memref: *ir.Value,
677 index: *ir.Value,
678 ) !StoreOp {
679 var builder = ir.OperationBuilder.init(ctx);
680 var state = op_specs.state(@This(), loc);
681 state.addOperands(&.{ value, memref, index });
682
683 const op = try builder.create(state);
684 return .{ .op = op };
685 }
686
687 pub fn createWithCache(
688 ctx: *ir.Context,
689 loc: ir.Location,
690 value: *ir.Value,
691 memref: *ir.Value,
692 index: *ir.Value,
693 cache: ?CacheOperation,
694 eviction: ?CacheEviction,
695 ) !StoreOp {
696 const store = try create(ctx, loc, value, memref, index);
697 errdefer store.op.erase();
698 if (cache) |hint| {
699 try setCacheOperationAttr(store.op, ctx, hint);
700 }
701 if (eviction) |hint| {
702 try setCacheEvictionAttr(store.op, ctx, hint);
703 }
704 return store;
705 }
706
707 pub fn getValue(self: StoreOp) *ir.Value {
708 return self.op.operands.items[0].value;
709 }
710
711 pub fn getMemref(self: StoreOp) *ir.Value {
712 return self.op.operands.items[1].value;
713 }
714
715 pub fn getIndex(self: StoreOp) *ir.Value {
716 return self.op.operands.items[2].value;
717 }
718
719 pub fn getCacheOperation(self: StoreOp) ?CacheOperation {
720 return getCacheOperationAttr(self.op);
721 }
722
723 pub fn getCacheEviction(self: StoreOp) ?CacheEviction {
724 return getCacheEvictionAttr(self.op);
725 }
726 };
727
728 pub const AtomicRmwOp = struct {
729 op: *ir.Operation,
730
731 pub const operation_spec = op_specs.leaf(.{
732 .mnemonic = "atomic_rmw",
733 .interfaces = &.{accessEffects(1, 2, true, true, true)},
734 .operands = 3,
735 .results = 1,
736 .attrs = &.{"kind"},
737 .traits = ir.OperationTraits{},
738 });
739 pub const operation_name = operation_spec.name;
740
741 pub fn create(
742 ctx: *ir.Context,
743 loc: ir.Location,
744 kind: AtomicRmwKind,
745 value: *ir.Value,
746 memref: *ir.Value,
747 index: *ir.Value,
748 result_type: ir.Type,
749 ) !AtomicRmwOp {
750 var builder = ir.OperationBuilder.init(ctx);
751 var state = op_specs.state(@This(), loc);
752 state.addOperands(&.{ value, memref, index });
753 state.addTypes(&.{result_type});
754
755 const op = try builder.create(state);
756 errdefer op.erase();
757 const kind_attr = try ctx.getDialectAttr("memref.atomic_kind", kind.toString());
758 try op.setAttr("kind", kind_attr);
759 return .{ .op = op };
760 }
761
762 pub fn getKind(self: AtomicRmwOp) ?AtomicRmwKind {
763 const dialect_attr = self.op.getAttrAs(ir.Attribute.DialectAttr, "kind") orelse return null;
764 return AtomicRmwKind.fromString(dialect_attr.payload);
765 }
766
767 pub fn getValue(self: AtomicRmwOp) *ir.Value {
768 return self.op.operands.items[0].value;
769 }
770
771 pub fn getMemref(self: AtomicRmwOp) *ir.Value {
772 return self.op.operands.items[1].value;
773 }
774
775 pub fn getIndex(self: AtomicRmwOp) *ir.Value {
776 return self.op.operands.items[2].value;
777 }
778
779 pub fn getResult(self: *const AtomicRmwOp) *ir.Value {
780 return self.op.getResult(0).?;
781 }
782 };
783
784 /// Reads one element atomically: no other thread observes a torn word, and the read is
785 /// ordered by `ordering`, which is `acquire` or `seq_cst` because a load cannot release.
786 pub const AtomicLoadOp = struct {
787 op: *ir.Operation,
788
789 pub const operation_spec = op_specs.leaf(.{
790 .mnemonic = "atomic_load",
791 .interfaces = &.{accessEffects(0, 1, true, false, true)},
792 .operands = 2,
793 .results = 1,
794 .required_attrs = .{ir.dialects.attribute.dialect("ordering", "memref.fence_ordering")},
795 .traits = ir.OperationTraits{},
796 });
797 pub const operation_name = operation_spec.name;
798 pub const verify = verifyAtomicLoadOp;
799
800 pub fn create(
801 ctx: *ir.Context,
802 loc: ir.Location,
803 memref: *ir.Value,
804 index: *ir.Value,
805 result_type: ir.Type,
806 ordering: FenceOrdering,
807 ) !AtomicLoadOp {
808 var builder = ir.OperationBuilder.init(ctx);
809 var state = op_specs.state(@This(), loc);
810 state.addOperands(&.{ memref, index });
811 state.addTypes(&.{result_type});
812
813 const op = try builder.create(state);
814 errdefer op.erase();
815 try setFenceOrderingAttr(op, ctx, ordering);
816 try verifyAtomicLoad(op);
817 return .{ .op = op };
818 }
819
820 pub fn getMemref(self: AtomicLoadOp) *ir.Value {
821 return self.op.operands.items[0].value;
822 }
823
824 pub fn getIndex(self: AtomicLoadOp) *ir.Value {
825 return self.op.operands.items[1].value;
826 }
827
828 pub fn getOrdering(self: AtomicLoadOp) ?FenceOrdering {
829 return getFenceOrderingAttr(self.op);
830 }
831
832 pub fn getResult(self: *const AtomicLoadOp) *ir.Value {
833 return self.op.getResult(0).?;
834 }
835 };
836
837 /// Writes one element atomically, ordered by `ordering`, which is `release` or `seq_cst`
838 /// because a store cannot acquire.
839 pub const AtomicStoreOp = struct {
840 op: *ir.Operation,
841
842 pub const operation_spec = op_specs.leaf(.{
843 .mnemonic = "atomic_store",
844 .interfaces = &.{accessEffects(1, 2, false, true, true)},
845 .operands = 3,
846 .results = 0,
847 .required_attrs = .{ir.dialects.attribute.dialect("ordering", "memref.fence_ordering")},
848 .traits = ir.OperationTraits{},
849 });
850 pub const operation_name = operation_spec.name;
851 pub const verify = verifyAtomicStoreOp;
852
853 pub fn create(
854 ctx: *ir.Context,
855 loc: ir.Location,
856 value: *ir.Value,
857 memref: *ir.Value,
858 index: *ir.Value,
859 ordering: FenceOrdering,
860 ) !AtomicStoreOp {
861 var builder = ir.OperationBuilder.init(ctx);
862 var state = op_specs.state(@This(), loc);
863 state.addOperands(&.{ value, memref, index });
864
865 const op = try builder.create(state);
866 errdefer op.erase();
867 try setFenceOrderingAttr(op, ctx, ordering);
868 try verifyAtomicStore(op);
869 return .{ .op = op };
870 }
871
872 pub fn getValue(self: AtomicStoreOp) *ir.Value {
873 return self.op.operands.items[0].value;
874 }
875
876 pub fn getMemref(self: AtomicStoreOp) *ir.Value {
877 return self.op.operands.items[1].value;
878 }
879
880 pub fn getIndex(self: AtomicStoreOp) *ir.Value {
881 return self.op.operands.items[2].value;
882 }
883
884 pub fn getOrdering(self: AtomicStoreOp) ?FenceOrdering {
885 return getFenceOrderingAttr(self.op);
886 }
887 };
888
889 pub const AtomicCasOp = struct {
890 op: *ir.Operation,
891
892 pub const operation_spec = op_specs.leaf(.{
893 .mnemonic = "atomic_cas",
894 .interfaces = &.{accessEffects(2, 3, true, true, true)},
895 .operands = 4,
896 .results = 1,
897 .attrs = .{ir.dialects.attribute.dialect("ordering", "memref.fence_ordering")},
898 .traits = ir.OperationTraits{},
899 });
900 pub const operation_name = operation_spec.name;
901 pub const verify = verifyAtomicCasOp;
902
903 pub fn create(
904 ctx: *ir.Context,
905 loc: ir.Location,
906 expected: *ir.Value,
907 desired: *ir.Value,
908 memref: *ir.Value,
909 index: *ir.Value,
910 result_type: ir.Type,
911 ) !AtomicCasOp {
912 var builder = ir.OperationBuilder.init(ctx);
913 var state = op_specs.state(@This(), loc);
914 state.addOperands(&.{ expected, desired, memref, index });
915 state.addTypes(&.{result_type});
916
917 const op = try builder.create(state);
918 return .{ .op = op };
919 }
920
921 pub fn getExpected(self: AtomicCasOp) *ir.Value {
922 return self.op.operands.items[0].value;
923 }
924
925 pub fn getDesired(self: AtomicCasOp) *ir.Value {
926 return self.op.operands.items[1].value;
927 }
928
929 pub fn getMemref(self: AtomicCasOp) *ir.Value {
930 return self.op.operands.items[2].value;
931 }
932
933 pub fn getIndex(self: AtomicCasOp) *ir.Value {
934 return self.op.operands.items[3].value;
935 }
936
937 /// The same exchange with its ordering spelled. Every ordering is legal on a compare
938 /// and swap, which both reads and writes.
939 pub fn createOrdered(
940 ctx: *ir.Context,
941 loc: ir.Location,
942 expected: *ir.Value,
943 desired: *ir.Value,
944 memref: *ir.Value,
945 index: *ir.Value,
946 result_type: ir.Type,
947 ordering: FenceOrdering,
948 ) !AtomicCasOp {
949 const cas = try create(ctx, loc, expected, desired, memref, index, result_type);
950 errdefer cas.op.erase();
951 try setFenceOrderingAttr(cas.op, ctx, ordering);
952 return cas;
953 }
954
955 /// The ordering the exchange carries, `seq_cst` when none is spelled.
956 pub fn getOrdering(self: AtomicCasOp) FenceOrdering {
957 return getFenceOrderingAttr(self.op) orelse .seq_cst;
958 }
959
960 pub fn getResult(self: *const AtomicCasOp) *ir.Value {
961 return self.op.getResult(0).?;
962 }
963 };
964
965 pub const CopyOp = struct {
966 op: *ir.Operation,
967
968 pub const operation_spec = op_specs.leaf(.{
969 .mnemonic = "copy",
970 .interfaces = &.{effects.EffectOpInterface.entryFor(.{ .facts = &.{
971 .{ .event = .{ .kind = .read, .resource = .{ .subject = .{ .operand = 0 } } } },
972 .{ .event = .{ .kind = .write, .resource = .{ .subject = .{ .operand = 1 } } } },
973 } })},
974 .operands = 2,
975 .results = 0,
976 });
977 pub const operation_name = operation_spec.name;
978
979 pub fn create(
980 ctx: *ir.Context,
981 loc: ir.Location,
982 src: *ir.Value,
983 dst: *ir.Value,
984 ) !CopyOp {
985 var builder = ir.OperationBuilder.init(ctx);
986 var state = op_specs.state(@This(), loc);
987 state.addOperands(&.{ src, dst });
988
989 const op = try builder.create(state);
990 return .{ .op = op };
991 }
992
993 pub fn getSrc(self: CopyOp) *ir.Value {
994 return self.op.operands.items[0].value;
995 }
996
997 pub fn getDst(self: CopyOp) *ir.Value {
998 return self.op.operands.items[1].value;
999 }
1000 };
1001
1002 pub const SubviewOp = struct {
1003 op: *ir.Operation,
1004
1005 pub const operation_spec = op_specs.leaf(.{
1006 .mnemonic = "subview",
1007 .interfaces = &.{aliasEffects()},
1008 .operands = 1,
1009 .results = 1,
1010 .attrs = &.{ "offset", "shape", "stride" },
1011 });
1012 pub const operation_name = operation_spec.name;
1013
1014 pub fn create(
1015 ctx: *ir.Context,
1016 loc: ir.Location,
1017 source: *ir.Value,
1018 result_type: ir.Type,
1019 ) !SubviewOp {
1020 var builder = ir.OperationBuilder.init(ctx);
1021 var state = op_specs.state(@This(), loc);
1022 state.addOperands(&.{source});
1023 state.addTypes(&.{result_type});
1024
1025 const op = try builder.create(state);
1026 return .{ .op = op };
1027 }
1028
1029 pub fn getResult(self: *const SubviewOp) *ir.Value {
1030 return self.op.getResult(0).?;
1031 }
1032
1033 pub fn getSource(self: SubviewOp) *ir.Value {
1034 return self.op.operands.items[0].value;
1035 }
1036
1037 pub fn getShapePayload(self: SubviewOp) ?[]const u8 {
1038 return getLayoutPayload(self.op, "shape");
1039 }
1040
1041 pub fn getStridePayload(self: SubviewOp) ?[]const u8 {
1042 return getLayoutPayload(self.op, "stride");
1043 }
1044
1045 pub fn getOffsetPayload(self: SubviewOp) ?[]const u8 {
1046 return getLayoutPayload(self.op, "offset");
1047 }
1048 };
1049
1050 /// A memref over a byte offset into a byte addressed base, with the offset unscaled.
1051 ///
1052 /// This is the one spelling for reaching a value inside an arena. `memref.load` and
1053 /// `memref.store` scale their index by the element size of the memref they are given, so a
1054 /// byte offset cannot be expressed as an index into a base of wider elements, and
1055 /// `memref.subview` carries a static offset attribute rather than a value. Here the base is
1056 /// 8 bit elements, the offset is one `index` operand added to the base unscaled, and loads
1057 /// and stores through the result scale by the RESULT's element size.
1058 ///
1059 /// The result type is the result's own type rather than a repeated attribute, so there is
1060 /// one statement of it that cannot disagree with itself.
1061 ///
1062 /// The arithmetic is one add and touches no memory, but the VALUE is a borrow: the result
1063 /// points inside the base's storage. So this declares the base borrowed and the result an
1064 /// alias of it, the same as `memref.subview` and `memref.transpose`. Declaring an
1065 /// independent result instead would tell a consumer that a write through the view cannot
1066 /// reach the base, which is the one thing that is never true here.
1067 ///
1068 /// Aligning the offset is the producer's duty. This operation does not check it, at
1069 /// verification or at run time, because the offset is a value and the alignment a value
1070 /// must satisfy is a property of what the producer intends to store there.
1071 pub const ViewOp = struct {
1072 op: *ir.Operation,
1073
1074 pub const operation_spec = op_specs.leaf(.{
1075 .mnemonic = "view",
1076 .interfaces = &.{aliasEffects()},
1077 .operands = 2,
1078 .results = 1,
1079 });
1080 pub const operation_name = operation_spec.name;
1081 pub const verify = verifyViewOp;
1082
1083 pub fn create(
1084 ctx: *ir.Context,
1085 loc: ir.Location,
1086 base: *ir.Value,
1087 byte_offset: *ir.Value,
1088 result_type: ir.Type,
1089 ) !ViewOp {
1090 var builder = ir.OperationBuilder.init(ctx);
1091 var state = op_specs.state(@This(), loc);
1092 state.addOperands(&.{ base, byte_offset });
1093 state.addTypes(&.{result_type});
1094
1095 const op = try builder.create(state);
1096 errdefer op.erase();
1097 try verifyView(op);
1098 return .{ .op = op };
1099 }
1100
1101 pub fn getResult(self: *const ViewOp) *ir.Value {
1102 return self.op.getResult(0).?;
1103 }
1104
1105 pub fn getBase(self: ViewOp) *ir.Value {
1106 return self.op.operands.items[0].value;
1107 }
1108
1109 pub fn getByteOffset(self: ViewOp) *ir.Value {
1110 return self.op.operands.items[1].value;
1111 }
1112 };
1113
1114 pub const TransposeOp = struct {
1115 op: *ir.Operation,
1116
1117 pub const operation_spec = op_specs.leaf(.{
1118 .mnemonic = "transpose",
1119 .interfaces = &.{aliasEffects()},
1120 .operands = 1,
1121 .results = 1,
1122 .attrs = &.{ "shape", "stride" },
1123 });
1124 pub const operation_name = operation_spec.name;
1125
1126 pub fn create(
1127 ctx: *ir.Context,
1128 loc: ir.Location,
1129 source: *ir.Value,
1130 result_type: ir.Type,
1131 ) !TransposeOp {
1132 var builder = ir.OperationBuilder.init(ctx);
1133 var state = op_specs.state(@This(), loc);
1134 state.addOperands(&.{source});
1135 state.addTypes(&.{result_type});
1136
1137 const op = try builder.create(state);
1138 return .{ .op = op };
1139 }
1140
1141 pub fn getResult(self: *const TransposeOp) *ir.Value {
1142 return self.op.getResult(0).?;
1143 }
1144
1145 pub fn getSource(self: TransposeOp) *ir.Value {
1146 return self.op.operands.items[0].value;
1147 }
1148
1149 pub fn getShapePayload(self: TransposeOp) ?[]const u8 {
1150 return getLayoutPayload(self.op, "shape");
1151 }
1152
1153 pub fn getStridePayload(self: TransposeOp) ?[]const u8 {
1154 return getLayoutPayload(self.op, "stride");
1155 }
1156 };
1157
1158 fn deinitPayload(allocator: std.mem.Allocator, ptr: *anyopaque) void {
1159 const payload: *MemrefTypePayload = @ptrCast(@alignCast(ptr));
1160 allocator.destroy(payload);
1161 }
1162
1163 fn typeParamFallback(ctx: *const ir.Context, typ: ir.Type) ?*const anyopaque {
1164 _ = ctx;
1165 const type_name = typ.getDialectTypeName() orelse return null;
1166 if (!std.mem.eql(u8, type_name, name)) return null;
1167 return &type_param_vtable;
1168 }
1169
1170 fn shapedTypeFallback(ctx: *const ir.Context, typ: ir.Type) ?*const anyopaque {
1171 _ = ctx;
1172 const type_name = typ.getDialectTypeName() orelse return null;
1173 if (!std.mem.eql(u8, type_name, name)) return null;
1174 return &shaped_type_vtable;
1175 }
1176
1177 fn loadSpec(ctx: *ir.Context) !void {
1178 ir.dialects.loadDialectSpec(ctx, spec) catch |err| switch (err) {
1179 error.ContextFrozen => {},
1180 else => return err,
1181 };
1182 }
1183
1184 fn payloadFromTypePtr(ctx: *ir.Context, type_ptr: *const anyopaque) ?*const MemrefTypePayload {
1185 loadSpec(ctx) catch return null;
1186
1187 const storage: *const ir.Type.DialectTypeStorage = @ptrCast(@alignCast(type_ptr));
1188 const typ = ir.Type{
1189 .type_id = .dialect_type,
1190 .impl = storage,
1191 };
1192 return ctx.getTypeParamPayload(typ, MemrefTypePayload) catch null;
1193 }
1194
1195 fn parseTypeParams(type_ptr: *const anyopaque, ctx_opaque: *const interfaces.ContextOpaque) anyerror!?interfaces.TypeParamPayload {
1196 const ctx = interfaces.castContext(ir.Context, ctx_opaque);
1197 const storage: *const ir.Type.DialectTypeStorage = @ptrCast(@alignCast(type_ptr));
1198 if (storage.param_key.len == 0) return null;
1199
1200 const params = parseMemrefParams(storage.param_key) orelse return null;
1201
1202 const payload = try ir.context.typePayloadAllocator(ctx).create(MemrefTypePayload);
1203 payload.* = .{
1204 .size = params.size,
1205 .element_type_name = params.element_type_name,
1206 .element_type = ctx.getDialectTypeFromName(params.element_type_name) catch null,
1207 .addr_space = params.addr_space,
1208 .alignment = params.alignment,
1209 .exclusive = params.exclusive,
1210 .indexing = params.indexing,
1211 };
1212
1213 if (params.size) |size| {
1214 payload.shape_storage[0] = size;
1215 payload.shape = payload.shape_storage[0..1];
1216 } else {
1217 payload.shape = null;
1218 }
1219
1220 return .{ .ptr = payload, .deinit = deinitPayload };
1221 }
1222
1223 fn shapedGetRank(type_ptr: *const anyopaque, ctx_opaque: *const interfaces.ContextOpaque) ?usize {
1224 const ctx = interfaces.castContext(ir.Context, ctx_opaque);
1225 const payload = payloadFromTypePtr(ctx, type_ptr) orelse return null;
1226 _ = payload;
1227 return 1;
1228 }
1229
1230 fn shapedGetShape(type_ptr: *const anyopaque, ctx_opaque: *const interfaces.ContextOpaque) ?[]const u64 {
1231 const ctx = interfaces.castContext(ir.Context, ctx_opaque);
1232 const payload = payloadFromTypePtr(ctx, type_ptr) orelse return null;
1233 return payload.shape;
1234 }
1235
1236 fn shapedGetElementType(type_ptr: *const anyopaque, ctx_opaque: *const interfaces.ContextOpaque) ?ir.Type {
1237 const ctx = interfaces.castContext(ir.Context, ctx_opaque);
1238 const payload = payloadFromTypePtr(ctx, type_ptr) orelse return null;
1239 return payload.element_type;
1240 }
1241
1242 fn shapedGetAddressSpaceTag(type_ptr: *const anyopaque, ctx_opaque: *const interfaces.ContextOpaque) ?u8 {
1243 const ctx = interfaces.castContext(ir.Context, ctx_opaque);
1244 const payload = payloadFromTypePtr(ctx, type_ptr) orelse return null;
1245 return @backingInt(payload.addr_space);
1246 }
1247
1248 pub fn getMemrefType1D(
1249 ctx: *ir.Context,
1250 size: u64,
1251 element_type: ir.Type,
1252 addr_space: AddressSpace,
1253 ) !ir.Type {
1254 return getMemrefType1DWithAttrs(ctx, size, element_type, addr_space, .{});
1255 }
1256
1257 pub fn getMemrefTypeDynamic(
1258 ctx: *ir.Context,
1259 element_type: ir.Type,
1260 addr_space: AddressSpace,
1261 ) !ir.Type {
1262 return getMemrefTypeDynamicWithAttrs(ctx, element_type, addr_space, .{});
1263 }
1264
1265 pub fn getMemrefType1DWithAttrs(
1266 ctx: *ir.Context,
1267 size: u64,
1268 element_type: ir.Type,
1269 addr_space: AddressSpace,
1270 attrs: MemrefTypeAttrs,
1271 ) !ir.Type {
1272 try loadSpec(ctx);
1273 var buf: [512]u8 = undefined;
1274 const elem_name = element_type.getDialectTypeName() orelse "unknown";
1275 var pos: usize = 0;
1276 pos = try ir.format.appendFmt(buf[0..], pos, "{d},{s},{s}", .{
1277 size,
1278 elem_name,
1279 addr_space.toString(),
1280 });
1281 pos = try appendTypeAttrs(buf[0..], pos, attrs);
1282 return ctx.getDialectTypeFromNameWithKey("memref", buf[0..pos]);
1283 }
1284
1285 pub fn getMemrefTypeDynamicWithAttrs(
1286 ctx: *ir.Context,
1287 element_type: ir.Type,
1288 addr_space: AddressSpace,
1289 attrs: MemrefTypeAttrs,
1290 ) !ir.Type {
1291 try loadSpec(ctx);
1292 var buf: [512]u8 = undefined;
1293 const elem_name = element_type.getDialectTypeName() orelse "unknown";
1294 var pos: usize = 0;
1295 pos = try ir.format.appendFmt(buf[0..], pos, "?,{s},{s}", .{
1296 elem_name,
1297 addr_space.toString(),
1298 });
1299 pos = try appendTypeAttrs(buf[0..], pos, attrs);
1300 return ctx.getDialectTypeFromNameWithKey("memref", buf[0..pos]);
1301 }
1302
1303 pub fn parseMemrefParams(param_key: []const u8) ?MemrefParams {
1304 var section_iter = std.mem.splitScalar(u8, param_key, ';');
1305 const base = section_iter.next() orelse return null;
1306 var iter = std.mem.splitScalar(u8, base, ',');
1307
1308 const size_str = iter.next() orelse return null;
1309 const size: ?u64 = if (std.mem.eql(u8, size_str, "?"))
1310 null
1311 else
1312 std.fmt.parseInt(u64, size_str, 10) catch return null;
1313
1314 const elem_type = iter.next() orelse return null;
1315 const addr_space_str = iter.next() orelse return null;
1316 const addr_space = AddressSpace.fromString(addr_space_str) orelse return null;
1317
1318 var alignment: ?u64 = null;
1319 var exclusive: ?bool = null;
1320 var indexing: ?Indexing = null;
1321
1322 while (section_iter.next()) |section| {
1323 if (section.len == 0) continue;
1324 var kv_iter = std.mem.splitScalar(u8, section, '=');
1325 const key = kv_iter.next() orelse continue;
1326 const value = kv_iter.next() orelse continue;
1327 if (kv_iter.next() != null) return null;
1328
1329 if (std.mem.eql(u8, key, "alignment") or std.mem.eql(u8, key, "align")) {
1330 alignment = std.fmt.parseInt(u64, value, 10) catch return null;
1331 continue;
1332 }
1333
1334 if (std.mem.eql(u8, key, "exclusive")) {
1335 if (std.mem.eql(u8, value, "true")) {
1336 exclusive = true;
1337 } else if (std.mem.eql(u8, value, "false")) {
1338 exclusive = false;
1339 } else {
1340 return null;
1341 }
1342 continue;
1343 }
1344
1345 if (std.mem.eql(u8, key, "indexing")) {
1346 indexing = Indexing.fromString(value) orelse return null;
1347 continue;
1348 }
1349 }
1350
1351 return .{
1352 .size = size,
1353 .element_type_name = elem_type,
1354 .addr_space = addr_space,
1355 .alignment = alignment,
1356 .exclusive = exclusive,
1357 .indexing = indexing,
1358 };
1359 }
1360
1361 fn formatDims(buf: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, dims: []const u64) !void {
1362 if (dims.len == 0) return;
1363 for (dims, 0..) |dim, i| {
1364 if (i > 0) try buf.append(allocator, ',');
1365 var tmp: [32]u8 = undefined;
1366 const text = try std.fmt.bufPrint(&tmp, "{d}", .{dim});
1367 try buf.appendSlice(allocator, text);
1368 }
1369 }
1370
1371 fn setLayoutDimsAttr(op: *ir.Operation, ctx: *ir.Context, attr_name: []const u8, full_name: []const u8, dims: []const u64) !void {
1372 var buf: std.ArrayListUnmanaged(u8) = .empty;
1373 const allocator = ir.context.transientAllocator(ctx);
1374 defer buf.deinit(allocator);
1375 try formatDims(&buf, allocator, dims);
1376 const attr = try ctx.getDialectAttr(full_name, buf.items);
1377 try op.setAttr(attr_name, attr);
1378 }
1379
1380 fn setLayoutOffsetAttr(op: *ir.Operation, ctx: *ir.Context, offset: u64) !void {
1381 var buf: [32]u8 = undefined;
1382 const payload = try std.fmt.bufPrint(&buf, "{d}", .{offset});
1383 const attr = try ctx.getDialectAttr("memref.offset", payload);
1384 try op.setAttr("offset", attr);
1385 }
1386
1387 fn getLayoutPayload(op: *const ir.Operation, attr_name: []const u8) ?[]const u8 {
1388 const dialect_attr = op.getAttrAs(ir.Attribute.DialectAttr, attr_name) orelse return null;
1389 return dialect_attr.payload;
1390 }
1391
1392 pub fn setLayoutAttrs(op: *ir.Operation, ctx: *ir.Context, attrs: LayoutAttrs) !void {
1393 if (attrs.offset) |offset| {
1394 try setLayoutOffsetAttr(op, ctx, offset);
1395 }
1396 if (attrs.shape) |shape| {
1397 try setLayoutDimsAttr(op, ctx, "shape", "memref.shape", shape);
1398 }
1399 if (attrs.stride) |stride| {
1400 try setLayoutDimsAttr(op, ctx, "stride", "memref.stride", stride);
1401 }
1402 }
1403
1404 fn setCacheOperationAttr(op: *ir.Operation, ctx: *ir.Context, cache: CacheOperation) !void {
1405 const cache_attr = try ctx.getDialectAttr("memref.cache", cache.toString());
1406 try op.setAttr("cache", cache_attr);
1407 }
1408
1409 fn getCacheOperationAttr(op: *const ir.Operation) ?CacheOperation {
1410 const dialect_attr = op.getAttrAs(ir.Attribute.DialectAttr, "cache") orelse return null;
1411 return CacheOperation.fromString(dialect_attr.payload);
1412 }
1413
1414 fn setCacheEvictionAttr(op: *ir.Operation, ctx: *ir.Context, eviction: CacheEviction) !void {
1415 const eviction_attr = try ctx.getDialectAttr("memref.eviction", eviction.toString());
1416 try op.setAttr("eviction", eviction_attr);
1417 }
1418
1419 fn getCacheEvictionAttr(op: *const ir.Operation) ?CacheEviction {
1420 const dialect_attr = op.getAttrAs(ir.Attribute.DialectAttr, "eviction") orelse return null;
1421 return CacheEviction.fromString(dialect_attr.payload);
1422 }
1423
1424 fn setFenceScopeAttr(op: *ir.Operation, ctx: *ir.Context, scope: FenceScope) !void {
1425 const attr = try ctx.getDialectAttr("memref.fence_scope", scope.toString());
1426 try op.setAttr("scope", attr);
1427 }
1428
1429 fn getFenceScopeAttr(op: *const ir.Operation) ?FenceScope {
1430 const dialect_attr = op.getAttrAs(ir.Attribute.DialectAttr, "scope") orelse return null;
1431 return FenceScope.fromString(dialect_attr.payload);
1432 }
1433
1434 fn setFenceOrderingAttr(op: *ir.Operation, ctx: *ir.Context, ordering: FenceOrdering) !void {
1435 const attr = try ctx.getDialectAttr("memref.fence_ordering", ordering.toString());
1436 try op.setAttr("ordering", attr);
1437 }
1438
1439 fn getFenceOrderingAttr(op: *const ir.Operation) ?FenceOrdering {
1440 const dialect_attr = op.getAttrAs(ir.Attribute.DialectAttr, "ordering") orelse return null;
1441 return FenceOrdering.fromString(dialect_attr.payload);
1442 }
1443
1444 fn appendTypeAttrs(buf: []u8, start: usize, attrs: MemrefTypeAttrs) !usize {
1445 var pos = start;
1446 if (attrs.alignment) |alignment| {
1447 pos = try ir.format.appendFmt(buf, pos, ";alignment={d}", .{alignment});
1448 }
1449 if (attrs.exclusive) |exclusive| {
1450 pos = try ir.format.appendFmt(
1451 buf,
1452 pos,
1453 ";exclusive={s}",
1454 .{if (exclusive) "true" else "false"},
1455 );
1456 }
1457 if (attrs.indexing) |indexing| {
1458 pos = try ir.format.appendFmt(buf, pos, ";indexing={s}", .{indexing.toString()});
1459 }
1460 return pos;
1461 }
1462 };
1463
1464 const ConstructorResourceCounts = struct {
1465 operations: usize,
1466
1467 fn capture(ctx: *const ir.Context) ConstructorResourceCounts {
1468 return .{
1469 .operations = ctx.operationCount(),
1470 };
1471 }
1472
1473 fn expectEqual(self: ConstructorResourceCounts, ctx: *const ir.Context) !void {
1474 try std.testing.expectEqual(self.operations, ctx.operationCount());
1475 }
1476 };
1477
1478 fn expectConstructorCleanup(baseline: ConstructorResourceCounts, ctx: *ir.Context, constructed: anytype) !void {
1479 const value = constructed catch |err| {
1480 try baseline.expectEqual(ctx);
1481 return err;
1482 };
1483 value.op.erase();
1484 try baseline.expectEqual(ctx);
1485 }
1486
1487 fn checkMemrefConstructorAllocationFailures(allocator: std.mem.Allocator) !void {
1488 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1489 defer ctx.deinit(allocator);
1490 const loc = ir.Location.getUnknown();
1491 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1492 const index_type = try arith.ArithDialect.getIndexType(&ctx);
1493 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 16, f32_type, .device);
1494 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, memref_type);
1495 var index = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 0);
1496 var value = try arith.ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 1.0);
1497 const baseline = ConstructorResourceCounts.capture(&ctx);
1498
1499 try expectConstructorCleanup(baseline, &ctx, MemrefDialect.FenceOp.create(&ctx, loc, .device, .acq_rel));
1500 try expectConstructorCleanup(baseline, &ctx, MemrefDialect.LoadOp.createWithCache(
1501 &ctx,
1502 loc,
1503 alloc.getResult(),
1504 index.getResult(),
1505 f32_type,
1506 .streaming,
1507 .first,
1508 ));
1509 try expectConstructorCleanup(baseline, &ctx, MemrefDialect.StoreOp.createWithCache(
1510 &ctx,
1511 loc,
1512 value.getResult(),
1513 alloc.getResult(),
1514 index.getResult(),
1515 .write_through,
1516 .no_allocate,
1517 ));
1518 try expectConstructorCleanup(baseline, &ctx, MemrefDialect.AtomicRmwOp.create(
1519 &ctx,
1520 loc,
1521 .add,
1522 value.getResult(),
1523 alloc.getResult(),
1524 index.getResult(),
1525 f32_type,
1526 ));
1527 }
1528
1529 test "MemrefDialect constructors clean every allocation failure" {
1530 try std.testing.checkAllAllocationFailures(
1531 std.testing.allocator,
1532 checkMemrefConstructorAllocationFailures,
1533 .{},
1534 );
1535 }
1536
1537 test "MemrefDialect type construction" {
1538 const testing = std.testing;
1539 var arena = alloc_arena.Arena.init(std.testing.allocator);
1540 defer arena.deinit();
1541 const allocator = arena.allocator();
1542
1543 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1544 defer ctx.deinit(allocator);
1545
1546 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1547 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 1024, f32_type, .host);
1548
1549 const params = MemrefDialect.parseMemrefParams(memref_type.getDialectParamKey().?).?;
1550 try testing.expectEqual(@as(u64, 1024), params.size.?);
1551 try testing.expectEqual(AddressSpace.host, params.addr_space);
1552 try testing.expect(params.alignment == null);
1553 try testing.expect(params.exclusive == null);
1554 try testing.expect(params.indexing == null);
1555 }
1556
1557 test "MemrefDialect type attributes" {
1558 const testing = std.testing;
1559 var arena = alloc_arena.Arena.init(std.testing.allocator);
1560 defer arena.deinit();
1561 const allocator = arena.allocator();
1562
1563 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1564 defer ctx.deinit(allocator);
1565
1566 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1567 const attrs = MemrefDialect.MemrefTypeAttrs{
1568 .alignment = 16,
1569 .exclusive = true,
1570 .indexing = .i32,
1571 };
1572 const memref_type = try MemrefDialect.getMemrefType1DWithAttrs(&ctx, 64, f32_type, .device, attrs);
1573
1574 const params = MemrefDialect.parseMemrefParams(memref_type.getDialectParamKey().?).?;
1575 try testing.expectEqual(@as(u64, 64), params.size.?);
1576 try testing.expectEqual(AddressSpace.device, params.addr_space);
1577 try testing.expectEqual(@as(u64, 16), params.alignment.?);
1578 try testing.expectEqual(true, params.exclusive.?);
1579 try testing.expectEqual(Indexing.i32, params.indexing.?);
1580 }
1581
1582 test "MemrefDialect structured payload and shaped interface" {
1583 const testing = std.testing;
1584 var gpa = alloc_observe.debug.Allocator(.{}).init(testing.allocator);
1585 defer {
1586 const status = gpa.deinit();
1587 testing.expect(status == .ok) catch @panic("memref payload leaked allocations");
1588 }
1589
1590 var ctx = try ir.Context.init(gpa.allocator(), ir.Context.Limits.testing);
1591 defer ctx.deinit(gpa.allocator());
1592
1593 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1594 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 64, f32_type, .device);
1595
1596 const payload1 = (try ctx.getTypeParamPayload(memref_type, MemrefDialect.MemrefTypePayload)).?;
1597 const payload2 = (try ctx.getTypeParamPayload(memref_type, MemrefDialect.MemrefTypePayload)).?;
1598 try testing.expect(payload1 == payload2);
1599 try testing.expectEqual(@as(u64, 64), payload1.size.?);
1600 try testing.expectEqual(AddressSpace.device, payload1.addr_space);
1601
1602 const shaped = ctx.typeInterface(memref_type, interfaces.ShapedTypeInterface).?;
1603 const rank = shaped.call(.getRank, .{}).?;
1604 try testing.expectEqual(@as(usize, 1), rank);
1605
1606 const shape = shaped.call(.getShape, .{}).?;
1607 try testing.expectEqual(@as(usize, 1), shape.len);
1608 try testing.expectEqual(@as(u64, 64), shape[0]);
1609
1610 const elem = shaped.call(.getElementType, .{}).?;
1611 try testing.expect(elem.eql(f32_type));
1612
1613 const addr_tag = shaped.call(.getAddressSpaceTag, .{}).?;
1614 try testing.expectEqual(@as(u8, @backingInt(AddressSpace.device)), addr_tag);
1615 }
1616
1617 test "MemrefDialect spec owns type interface fallbacks" {
1618 const testing = std.testing;
1619
1620 var arena = alloc_arena.Arena.init(std.testing.allocator);
1621 defer arena.deinit();
1622 const allocator = arena.allocator();
1623
1624 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1625 defer ctx.deinit(allocator);
1626
1627 try ir.dialects.loadDialectSpec(&ctx, MemrefDialect.spec);
1628
1629 try testing.expect(ctx.getDialectTypeInterfaceFallback(MemrefDialect.name, interfaces.TypeParamInterface.id) != null);
1630 try testing.expect(ctx.getDialectTypeInterfaceFallback(MemrefDialect.name, interfaces.ShapedTypeInterface.id) != null);
1631
1632 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1633 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 32, f32_type, .shared);
1634
1635 try testing.expect((try ctx.getTypeParamPayload(memref_type, MemrefDialect.MemrefTypePayload)) != null);
1636 try testing.expect(ctx.typeInterface(memref_type, interfaces.ShapedTypeInterface) != null);
1637 }
1638
1639 test "MemrefDialect.AllocOp creates allocation" {
1640 const testing = std.testing;
1641 var arena = alloc_arena.Arena.init(std.testing.allocator);
1642 defer arena.deinit();
1643 const allocator = arena.allocator();
1644
1645 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1646 defer ctx.deinit(allocator);
1647
1648 const loc = ir.Location.getUnknown();
1649 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1650 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 1024, f32_type, .device);
1651
1652 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, memref_type);
1653
1654 try testing.expectEqualStrings("memref.alloc", alloc.op.name.name);
1655 try testing.expect(alloc.getDynamicSize() == null);
1656 }
1657
1658 test "MemrefDialect.AllocaOp creates local allocation" {
1659 const testing = std.testing;
1660 var arena = alloc_arena.Arena.init(std.testing.allocator);
1661 defer arena.deinit();
1662 const allocator = arena.allocator();
1663
1664 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1665 defer ctx.deinit(allocator);
1666
1667 const loc = ir.Location.getUnknown();
1668 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1669 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 32, f32_type, .host);
1670
1671 var alloca = try MemrefDialect.AllocaOp.createStatic(&ctx, loc, memref_type);
1672
1673 try testing.expectEqualStrings("memref.alloca", alloca.op.name.name);
1674 try testing.expect(alloca.getDynamicSize() == null);
1675 }
1676
1677 test "MemrefDialect.LoadOp and StoreOp" {
1678 const testing = std.testing;
1679 var arena = alloc_arena.Arena.init(std.testing.allocator);
1680 defer arena.deinit();
1681 const allocator = arena.allocator();
1682
1683 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1684 defer ctx.deinit(allocator);
1685
1686 const loc = ir.Location.getUnknown();
1687 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1688 const index_type = try arith.ArithDialect.getIndexType(&ctx);
1689 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 1024, f32_type, .host);
1690
1691 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, memref_type);
1692 var idx = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 0);
1693
1694 var load = try MemrefDialect.LoadOp.create(&ctx, loc, alloc.getResult(), idx.getResult(), f32_type);
1695 try testing.expectEqualStrings("memref.load", load.op.name.name);
1696 try testing.expect(load.getMemref() == alloc.getResult());
1697
1698 var val = try arith.ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 3.14);
1699 var store = try MemrefDialect.StoreOp.create(&ctx, loc, val.getResult(), alloc.getResult(), idx.getResult());
1700 try testing.expectEqualStrings("memref.store", store.op.name.name);
1701 try testing.expect(store.getMemref() == alloc.getResult());
1702 try testing.expect(store.getValue() == val.getResult());
1703 }
1704
1705 test "MemrefDialect cache attributes on load/store" {
1706 const testing = std.testing;
1707 var arena = alloc_arena.Arena.init(std.testing.allocator);
1708 defer arena.deinit();
1709 const allocator = arena.allocator();
1710
1711 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1712 defer ctx.deinit(allocator);
1713
1714 const loc = ir.Location.getUnknown();
1715 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1716 const index_type = try arith.ArithDialect.getIndexType(&ctx);
1717 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 16, f32_type, .device);
1718
1719 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, memref_type);
1720 var idx = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 0);
1721
1722 var load = try MemrefDialect.LoadOp.createWithCache(
1723 &ctx,
1724 loc,
1725 alloc.getResult(),
1726 idx.getResult(),
1727 f32_type,
1728 .streaming,
1729 .first,
1730 );
1731 try testing.expectEqual(CacheOperation.streaming, load.getCacheOperation().?);
1732 try testing.expectEqual(CacheEviction.first, load.getCacheEviction().?);
1733
1734 var val = try arith.ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 1.0);
1735 var store = try MemrefDialect.StoreOp.createWithCache(
1736 &ctx,
1737 loc,
1738 val.getResult(),
1739 alloc.getResult(),
1740 idx.getResult(),
1741 .write_through,
1742 .no_allocate,
1743 );
1744 try testing.expectEqual(CacheOperation.write_through, store.getCacheOperation().?);
1745 try testing.expectEqual(CacheEviction.no_allocate, store.getCacheEviction().?);
1746 }
1747
1748 test "MemrefDialect.FenceOp carries scope and ordering side effect" {
1749 const testing = std.testing;
1750 var arena = alloc_arena.Arena.init(std.testing.allocator);
1751 defer arena.deinit();
1752 const allocator = arena.allocator();
1753
1754 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1755 defer ctx.deinit(allocator);
1756 try ir.dialects.loadDialectSpec(&ctx, MemrefDialect.spec);
1757
1758 const loc = ir.Location.getUnknown();
1759 const fence = try MemrefDialect.FenceOp.create(&ctx, loc, .device, .acq_rel);
1760
1761 try testing.expectEqualStrings("memref.fence", fence.op.name.name);
1762 try testing.expectEqual(FenceScope.device, fence.getScope().?);
1763 try testing.expectEqual(FenceOrdering.acq_rel, fence.getOrdering().?);
1764 try testing.expectEqual(@as(?FenceScope, null), FenceScope.fromString("thread"));
1765 try testing.expectEqual(@as(?FenceOrdering, null), FenceOrdering.fromString("consume"));
1766 }
1767
1768 test "MemrefDialect atomic load and store refuse orderings, widths, and types they cannot carry" {
1769 const testing = std.testing;
1770 var arena = alloc_arena.Arena.init(std.testing.allocator);
1771 defer arena.deinit();
1772 const allocator = arena.allocator();
1773
1774 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1775 defer ctx.deinit(allocator);
1776 try ir.dialects.loadDialectSpec(&ctx, MemrefDialect.spec);
1777
1778 const loc = ir.Location.getUnknown();
1779 const i64_type = try arith.ArithDialect.getScalarType(&ctx, .i64);
1780 const i32_type = try arith.ArithDialect.getScalarType(&ctx, .i32);
1781 const i16_type = try arith.ArithDialect.getScalarType(&ctx, .i16);
1782 const f64_type = try arith.ArithDialect.getScalarType(&ctx, .f64);
1783 const index_type = try arith.ArithDialect.getIndexType(&ctx);
1784 const words = try MemrefDialect.AllocOp.createStatic(&ctx, loc, try MemrefDialect.getMemrefType1D(&ctx, 4, i64_type, .host));
1785 const shorts = try MemrefDialect.AllocOp.createStatic(
1786 &ctx,
1787 loc,
1788 try MemrefDialect.getMemrefType1D(&ctx, 4, i16_type, .host),
1789 );
1790 const doubles = try MemrefDialect.AllocOp.createStatic(
1791 &ctx,
1792 loc,
1793 try MemrefDialect.getMemrefType1D(&ctx, 4, f64_type, .host),
1794 );
1795 const slot = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 1);
1796 const word = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, i64_type, 9);
1797 const half = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, i32_type, 9);
1798 const baseline = ConstructorResourceCounts.capture(&ctx);
1799
1800 try testing.expectError(error.AtomicOrderingInvalid, expectConstructorCleanup(
1801 baseline,
1802 &ctx,
1803 MemrefDialect.AtomicLoadOp.create(
1804 &ctx,
1805 loc,
1806 words.getResult(),
1807 slot.getResult(),
1808 i64_type,
1809 .release,
1810 ),
1811 ));
1812 try testing.expectError(error.AtomicOrderingInvalid, expectConstructorCleanup(
1813 baseline,
1814 &ctx,
1815 MemrefDialect.AtomicLoadOp.create(
1816 &ctx,
1817 loc,
1818 words.getResult(),
1819 slot.getResult(),
1820 i64_type,
1821 .acq_rel,
1822 ),
1823 ));
1824 try testing.expectError(error.AtomicOrderingInvalid, expectConstructorCleanup(
1825 baseline,
1826 &ctx,
1827 MemrefDialect.AtomicStoreOp.create(
1828 &ctx,
1829 loc,
1830 word.getResult(),
1831 words.getResult(),
1832 slot.getResult(),
1833 .acquire,
1834 ),
1835 ));
1836 try testing.expectError(error.AtomicElementNotWordOrHalfWord, expectConstructorCleanup(
1837 baseline,
1838 &ctx,
1839 MemrefDialect.AtomicLoadOp.create(
1840 &ctx,
1841 loc,
1842 shorts.getResult(),
1843 slot.getResult(),
1844 i16_type,
1845 .acquire,
1846 ),
1847 ));
1848 try testing.expectError(error.AtomicElementNotWordOrHalfWord, expectConstructorCleanup(
1849 baseline,
1850 &ctx,
1851 MemrefDialect.AtomicLoadOp.create(
1852 &ctx,
1853 loc,
1854 doubles.getResult(),
1855 slot.getResult(),
1856 f64_type,
1857 .seq_cst,
1858 ),
1859 ));
1860 try testing.expectError(error.AtomicTypeMismatch, expectConstructorCleanup(
1861 baseline,
1862 &ctx,
1863 MemrefDialect.AtomicLoadOp.create(
1864 &ctx,
1865 loc,
1866 words.getResult(),
1867 slot.getResult(),
1868 i32_type,
1869 .acquire,
1870 ),
1871 ));
1872 try testing.expectError(error.AtomicTypeMismatch, expectConstructorCleanup(
1873 baseline,
1874 &ctx,
1875 MemrefDialect.AtomicStoreOp.create(
1876 &ctx,
1877 loc,
1878 half.getResult(),
1879 words.getResult(),
1880 slot.getResult(),
1881 .release,
1882 ),
1883 ));
1884 try testing.expectError(error.AtomicOperandNotMemref, expectConstructorCleanup(
1885 baseline,
1886 &ctx,
1887 MemrefDialect.AtomicStoreOp.create(
1888 &ctx,
1889 loc,
1890 word.getResult(),
1891 word.getResult(),
1892 slot.getResult(),
1893 .seq_cst,
1894 ),
1895 ));
1896
1897 const load = try MemrefDialect.AtomicLoadOp.create(
1898 &ctx,
1899 loc,
1900 words.getResult(),
1901 slot.getResult(),
1902 i64_type,
1903 .seq_cst,
1904 );
1905 const store = try MemrefDialect.AtomicStoreOp.create(
1906 &ctx,
1907 loc,
1908 word.getResult(),
1909 words.getResult(),
1910 slot.getResult(),
1911 .release,
1912 );
1913 const cas = try MemrefDialect.AtomicCasOp.create(
1914 &ctx,
1915 loc,
1916 word.getResult(),
1917 load.getResult(),
1918 words.getResult(),
1919 slot.getResult(),
1920 i64_type,
1921 );
1922 const ordered_cas = try MemrefDialect.AtomicCasOp.createOrdered(
1923 &ctx,
1924 loc,
1925 word.getResult(),
1926 load.getResult(),
1927 words.getResult(),
1928 slot.getResult(),
1929 i64_type,
1930 .acquire,
1931 );
1932 const fence = try MemrefDialect.FenceOp.create(&ctx, loc, .system, .release);
1933 try testing.expectEqual(FenceOrdering.seq_cst, load.getOrdering().?);
1934 try testing.expectEqual(FenceOrdering.release, store.getOrdering().?);
1935 try testing.expectEqual(FenceOrdering.seq_cst, cas.getOrdering());
1936 try testing.expectEqual(FenceOrdering.acquire, ordered_cas.getOrdering());
1937
1938 const operations = [_]*ir.Operation{ load.op, store.op, cas.op, ordered_cas.op, fence.op };
1939 for (operations) |op| {
1940 try ir.verifyOperation(op, .{});
1941 var declaration = try effects.inspect(std.testing.allocator, op);
1942 defer declaration.deinit(std.testing.allocator);
1943 try testing.expect(!effects.discard(declaration.facts));
1944 try testing.expect(
1945 !effects.duplicate(declaration.facts, .{
1946 .read_values = true,
1947 .execution_context = true,
1948 }),
1949 );
1950 }
1951 }
1952
1953 test "MemrefDialect subview/transpose attach layout attrs" {
1954 const testing = std.testing;
1955 var arena = alloc_arena.Arena.init(std.testing.allocator);
1956 defer arena.deinit();
1957 const allocator = arena.allocator();
1958
1959 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1960 defer ctx.deinit(allocator);
1961
1962 const loc = ir.Location.getUnknown();
1963 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
1964 const src_type = try MemrefDialect.getMemrefType1D(&ctx, 8, f32_type, .host);
1965 const view_type = try MemrefDialect.getMemrefType1D(&ctx, 4, f32_type, .host);
1966
1967 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, src_type);
1968 var subview = try MemrefDialect.SubviewOp.create(&ctx, loc, alloc.getResult(), view_type);
1969 try MemrefDialect.setLayoutAttrs(subview.op, &ctx, .{
1970 .offset = 1,
1971 .shape = &.{ 2, 2 },
1972 .stride = &.{ 2, 1 },
1973 });
1974
1975 try testing.expectEqualStrings("1", subview.getOffsetPayload().?);
1976 try testing.expectEqualStrings("2,2", subview.getShapePayload().?);
1977 try testing.expectEqualStrings("2,1", subview.getStridePayload().?);
1978
1979 var transpose = try MemrefDialect.TransposeOp.create(&ctx, loc, alloc.getResult(), view_type);
1980 try MemrefDialect.setLayoutAttrs(transpose.op, &ctx, .{
1981 .shape = &.{ 2, 2 },
1982 .stride = &.{ 1, 2 },
1983 });
1984
1985 try testing.expectEqualStrings("2,2", transpose.getShapePayload().?);
1986 try testing.expectEqualStrings("1,2", transpose.getStridePayload().?);
1987 }
1988
1989 test "MemrefDialect.AtomicRmwOp carries kind, operands, and old-value result" {
1990 const testing = std.testing;
1991 var arena = alloc_arena.Arena.init(std.testing.allocator);
1992 defer arena.deinit();
1993 const allocator = arena.allocator();
1994
1995 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
1996 defer ctx.deinit(allocator);
1997
1998 const loc = ir.Location.getUnknown();
1999 const f32_type = try arith.ArithDialect.getScalarType(&ctx, .f32);
2000 const i32_type = try arith.ArithDialect.getScalarType(&ctx, .i32);
2001 const index_type = try arith.ArithDialect.getIndexType(&ctx);
2002 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 64, f32_type, .device);
2003
2004 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, memref_type);
2005 var idx = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 7);
2006 var val = try arith.ArithDialect.ConstantOp.createFloat(&ctx, loc, f32_type, 2.5);
2007
2008 var atomic = try MemrefDialect.AtomicRmwOp.create(
2009 &ctx,
2010 loc,
2011 .add,
2012 val.getResult(),
2013 alloc.getResult(),
2014 idx.getResult(),
2015 f32_type,
2016 );
2017 try testing.expectEqualStrings("memref.atomic_rmw", atomic.op.name.name);
2018 try testing.expectEqual(AtomicRmwKind.add, atomic.getKind().?);
2019 try testing.expect(atomic.getValue() == val.getResult());
2020 try testing.expect(atomic.getMemref() == alloc.getResult());
2021 try testing.expect(atomic.getIndex() == idx.getResult());
2022 try testing.expect(atomic.getResult().type.eql(f32_type));
2023
2024 var ival = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, i32_type, 3);
2025 inline for (.{ AtomicRmwKind.min, AtomicRmwKind.max, AtomicRmwKind.bit_and, AtomicRmwKind.bit_or, AtomicRmwKind.bit_xor, AtomicRmwKind.exchange }) |kind| {
2026 var op = try MemrefDialect.AtomicRmwOp.create(
2027 &ctx,
2028 loc,
2029 kind,
2030 ival.getResult(),
2031 alloc.getResult(),
2032 idx.getResult(),
2033 i32_type,
2034 );
2035 try testing.expectEqual(kind, op.getKind().?);
2036 }
2037
2038 try testing.expectEqual(@as(?AtomicRmwKind, null), AtomicRmwKind.fromString("nand"));
2039 try testing.expectEqual(AtomicRmwKind.bit_xor, AtomicRmwKind.fromString("bit_xor").?);
2040 }
2041
2042 test "MemrefDialect.AtomicCasOp carries operands and old-value result" {
2043 const testing = std.testing;
2044 var arena = alloc_arena.Arena.init(std.testing.allocator);
2045 defer arena.deinit();
2046 const allocator = arena.allocator();
2047
2048 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2049 defer ctx.deinit(allocator);
2050
2051 const loc = ir.Location.getUnknown();
2052 const i32_type = try arith.ArithDialect.getScalarType(&ctx, .i32);
2053 const index_type = try arith.ArithDialect.getIndexType(&ctx);
2054 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 64, i32_type, .device);
2055
2056 var alloc = try MemrefDialect.AllocOp.createStatic(&ctx, loc, memref_type);
2057 var idx = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 7);
2058 var expected = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, i32_type, 3);
2059 var desired = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, i32_type, 5);
2060
2061 var atomic = try MemrefDialect.AtomicCasOp.create(
2062 &ctx,
2063 loc,
2064 expected.getResult(),
2065 desired.getResult(),
2066 alloc.getResult(),
2067 idx.getResult(),
2068 i32_type,
2069 );
2070 try testing.expectEqualStrings("memref.atomic_cas", atomic.op.name.name);
2071 try testing.expect(atomic.getExpected() == expected.getResult());
2072 try testing.expect(atomic.getDesired() == desired.getResult());
2073 try testing.expect(atomic.getMemref() == alloc.getResult());
2074 try testing.expect(atomic.getIndex() == idx.getResult());
2075 try testing.expect(atomic.getResult().type.eql(i32_type));
2076 }
2077
2078 /// THE LIST IS EXHAUSTIVE, WHICH IS WHAT `complete` STATES. An allocation
2079 /// allocates, may fail in its domain, and hands back one fresh identity. It
2080 /// reads nothing, writes nothing, and frees nothing, so there is no fact left
2081 /// unsaid. Completeness is a claim about the list and not a permission: an
2082 /// allocate event is not a read or a write and a fresh identity is not
2083 /// ownership none, so `total` still refuses, and every permission derived
2084 /// through it still refuses.
2085 fn allocationEffects(comptime domain: []const u8) ir.interfaces.InterfaceEntry {
2086 return effects.EffectOpInterface.entryFor(.{ .complete = true, .facts = &.{
2087 .{ .event = .{ .kind = .allocate, .resource = .{
2088 .subject = .{ .result = 0 },
2089 .allocator_domain = domain,
2090 } } },
2091 .{ .event = .{ .kind = .failure, .resource = .{
2092 .allocator_domain = domain,
2093 } } },
2094 .{ .result = .{ .index = 0, .fresh_identity = true, .ownership = .owned } },
2095 } });
2096 }
2097
2098 /// Largest alignment a global may request.
2099 ///
2100 /// A loader places a section on a page boundary, so an offset aligned within a section is also
2101 /// aligned in memory up to one page and no further. `lib/choir/src/backends/machine.zig` states
2102 /// the same bound for the symbols this becomes, and a test there pins the two together rather
2103 /// than this dialect depending on a backend.
2104 pub const max_global_alignment = 4096;
2105
2106 /// The memref parameters of `ty`, or null when `ty` is not a memref type.
2107 pub fn paramsOf(ty: ir.Type) ?MemrefDialect.MemrefParams {
2108 const name = ty.getDialectTypeName() orelse return null;
2109 if (!std.mem.eql(u8, name, MemrefDialect.name)) return null;
2110 const key = ty.getDialectParamKey() orelse return null;
2111 return MemrefDialect.parseMemrefParams(key);
2112 }
2113
2114 /// The number of bytes one value of `element_type_name` occupies.
2115 ///
2116 /// The width comes from `arith`, which owns the scalar types, rather than from a switch this
2117 /// dialect keeps beside it. An element whose width is not a whole number of bytes has no byte
2118 /// size here rather than a rounded one, so storage of such elements is refused by name instead
2119 /// of quietly taking more room than it asked for.
2120 pub fn elementByteSize(element_type_name: []const u8) ?u64 {
2121 const kind = arith.scalarKindFromTypeName(element_type_name) orelse return null;
2122 const bits = arith.scalarBitWidth(kind);
2123 if (bits == 0 or bits % 8 != 0) return null;
2124 return bits / 8;
2125 }
2126
2127 /// The number of bytes a statically shaped memref occupies once loaded.
2128 pub fn staticByteSize(memref_type: ir.Type) ?u64 {
2129 const params = paramsOf(memref_type) orelse return null;
2130 const count = params.size orelse return null;
2131 const width = elementByteSize(params.element_type_name) orelse return null;
2132 return std.math.mul(u64, count, width) catch null;
2133 }
2134
2135 /// The `memref.global` that `sym_name` names, searched outward from `from`.
2136 ///
2137 /// A global is a module level declaration and its uses sit inside functions, so resolution walks
2138 /// outward through enclosing operations rather than down from a root this dialect cannot name.
2139 /// The walk is bounded by nesting depth and reads each enclosing body once.
2140 pub fn findGlobal(from: *const ir.Operation, sym_name: []const u8) ?MemrefDialect.GlobalOp {
2141 var current = from.getParentOp();
2142 while (current) |ancestor| : (current = ancestor.getParentOp()) {
2143 if (globalInBody(ancestor, sym_name)) |found| return found;
2144 }
2145 return null;
2146 }
2147
2148 fn globalInBody(container: *ir.Operation, sym_name: []const u8) ?MemrefDialect.GlobalOp {
2149 const region = container.getRegion(0) orelse return null;
2150 const block = region.getEntryBlock() orelse return null;
2151 var cursor = block.operations.head;
2152 while (cursor) |node| {
2153 const op: *ir.Operation = @ptrCast(@alignCast(node));
2154 cursor = op.next_op;
2155 if (!std.mem.eql(u8, op.name.name, MemrefDialect.GlobalOp.operation_name)) continue;
2156 const candidate = MemrefDialect.GlobalOp{ .op = op };
2157 const name = candidate.getSymName() orelse continue;
2158 if (std.mem.eql(u8, name, sym_name)) return candidate;
2159 }
2160 return null;
2161 }
2162
2163 fn verifyGlobal(op: *ir.Operation) MemrefVerifyError!void {
2164 const self = MemrefDialect.GlobalOp{ .op = op };
2165 const name = self.getSymName() orelse return MemrefVerifyError.GlobalMissingName;
2166 if (name.len == 0) return MemrefVerifyError.GlobalMissingName;
2167 const memref_type = self.getType() orelse return MemrefVerifyError.GlobalMissingType;
2168 const params = paramsOf(memref_type) orelse return MemrefVerifyError.GlobalTypeNotMemref;
2169 if (params.size == null) return MemrefVerifyError.GlobalTypeNotStatic;
2170 const alignment = self.getAlignment() orelse return MemrefVerifyError.GlobalMissingAlignment;
2171 if (alignment == 0 or alignment > max_global_alignment) {
2172 return MemrefVerifyError.GlobalInvalidAlignment;
2173 }
2174 if (!std.math.isPowerOfTwo(alignment)) return MemrefVerifyError.GlobalInvalidAlignment;
2175 const constant = self.isConstant() orelse return MemrefVerifyError.GlobalMissingConstant;
2176 const size = staticByteSize(memref_type) orelse return MemrefVerifyError.GlobalTypeNotStatic;
2177 const initial = self.getInitial() orelse {
2178 if (constant) return MemrefVerifyError.GlobalConstantWithoutInitial;
2179 return;
2180 };
2181 if (initial.len != size) return MemrefVerifyError.GlobalInitialLengthMismatch;
2182 }
2183
2184 fn verifyGlobalOp(op_ptr: *const anyopaque) anyerror!void {
2185 const op: *ir.Operation = @ptrCast(@alignCast(@constCast(op_ptr)));
2186 try verifyGlobal(op);
2187 }
2188
2189 fn verifyGetGlobal(op: *ir.Operation) MemrefVerifyError!void {
2190 const self = MemrefDialect.GetGlobalOp{ .op = op };
2191 const name = self.getSymName() orelse return MemrefVerifyError.GetGlobalMissingName;
2192 if (name.len == 0) return MemrefVerifyError.GetGlobalMissingName;
2193 if (op.results.items.len != 1) return MemrefVerifyError.GetGlobalResultNotMemref;
2194 if (paramsOf(op.results.items[0].type) == null) {
2195 return MemrefVerifyError.GetGlobalResultNotMemref;
2196 }
2197 }
2198
2199 fn verifyGetGlobalOp(op_ptr: *const anyopaque) anyerror!void {
2200 const op: *ir.Operation = @ptrCast(@alignCast(@constCast(op_ptr)));
2201 try verifyGetGlobal(op);
2202 }
2203
2204 fn verifyView(op: *ir.Operation) MemrefVerifyError!void {
2205 if (op.operands.items.len != 2) return MemrefVerifyError.ViewBaseNotMemref;
2206 const base_params = paramsOf(op.operands.items[0].value.type) orelse
2207 return MemrefVerifyError.ViewBaseNotMemref;
2208 const width = elementByteSize(base_params.element_type_name) orelse
2209 return MemrefVerifyError.ViewBaseNotBytes;
2210 if (width != 1) return MemrefVerifyError.ViewBaseNotBytes;
2211 if (op.results.items.len != 1) return MemrefVerifyError.ViewResultNotMemref;
2212 const result_params = paramsOf(op.results.items[0].type) orelse
2213 return MemrefVerifyError.ViewResultNotMemref;
2214 if (result_params.size == null) return MemrefVerifyError.ViewResultNotStatic;
2215 }
2216
2217 fn verifyViewOp(op_ptr: *const anyopaque) anyerror!void {
2218 const op: *ir.Operation = @ptrCast(@alignCast(@constCast(op_ptr)));
2219 try verifyView(op);
2220 }
2221
2222 /// Whether `element_type_name` names an integer a single x86 instruction reads or writes
2223 /// atomically when aligned: a machine word or a 32-bit half of one.
2224 fn atomicElementSupported(element_type_name: []const u8) bool {
2225 const kind = arith.scalarKindFromTypeName(element_type_name) orelse return false;
2226 if (!arith.scalarKindIsInteger(kind)) return false;
2227 const bits = arith.scalarBitWidth(kind);
2228 return bits == 32 or bits == 64;
2229 }
2230
2231 fn verifyAtomicAccess(
2232 op: *ir.Operation,
2233 memref_index: usize,
2234 value_type: ir.Type,
2235 ) MemrefVerifyError!void {
2236 if (op.operands.items.len <= memref_index) return MemrefVerifyError.AtomicOperandNotMemref;
2237 const params = paramsOf(op.operands.items[memref_index].value.type) orelse
2238 return MemrefVerifyError.AtomicOperandNotMemref;
2239 if (!atomicElementSupported(params.element_type_name)) {
2240 return MemrefVerifyError.AtomicElementNotWordOrHalfWord;
2241 }
2242 const value_name = value_type.getDialectTypeName() orelse
2243 return MemrefVerifyError.AtomicTypeMismatch;
2244 if (!std.mem.eql(u8, value_name, params.element_type_name)) {
2245 return MemrefVerifyError.AtomicTypeMismatch;
2246 }
2247 }
2248
2249 fn verifyAtomicOrdering(
2250 op: *const ir.Operation,
2251 legal: []const FenceOrdering,
2252 ) MemrefVerifyError!void {
2253 if (op.getAttr("ordering") == null) return MemrefVerifyError.AtomicOrderingMissing;
2254 const ordering = MemrefDialect.getFenceOrderingAttr(op) orelse
2255 return MemrefVerifyError.AtomicOrderingInvalid;
2256 for (legal) |allowed| {
2257 if (ordering == allowed) return;
2258 }
2259 return MemrefVerifyError.AtomicOrderingInvalid;
2260 }
2261
2262 fn verifyAtomicLoad(op: *ir.Operation) MemrefVerifyError!void {
2263 try verifyAtomicOrdering(op, &.{ .acquire, .seq_cst });
2264 if (op.results.items.len != 1) return MemrefVerifyError.AtomicTypeMismatch;
2265 try verifyAtomicAccess(op, 0, op.results.items[0].type);
2266 }
2267
2268 fn verifyAtomicLoadOp(op_ptr: *const anyopaque) anyerror!void {
2269 const op: *ir.Operation = @ptrCast(@alignCast(@constCast(op_ptr)));
2270 try verifyAtomicLoad(op);
2271 }
2272
2273 fn verifyAtomicStore(op: *ir.Operation) MemrefVerifyError!void {
2274 try verifyAtomicOrdering(op, &.{ .release, .seq_cst });
2275 if (op.operands.items.len != 3) return MemrefVerifyError.AtomicOperandNotMemref;
2276 try verifyAtomicAccess(op, 1, op.operands.items[0].value.type);
2277 }
2278
2279 fn verifyAtomicStoreOp(op_ptr: *const anyopaque) anyerror!void {
2280 const op: *ir.Operation = @ptrCast(@alignCast(@constCast(op_ptr)));
2281 try verifyAtomicStore(op);
2282 }
2283
2284 fn verifyAtomicCasOp(op_ptr: *const anyopaque) anyerror!void {
2285 const op: *const ir.Operation = @ptrCast(@alignCast(op_ptr));
2286 if (op.getAttr("ordering") == null) return;
2287 try verifyAtomicOrdering(op, &.{ .acquire, .release, .acq_rel, .seq_cst });
2288 }
2289
2290 fn aliasEffects() ir.interfaces.InterfaceEntry {
2291 return effects.EffectOpInterface.entryFor(.{ .facts = &.{
2292 .{ .event = .{ .kind = .borrow, .resource = .{ .subject = .{ .operand = 0 } } } },
2293 .{ .result = .{ .index = 0, .alias = .{ .operand = 0 }, .ownership = .borrowed } },
2294 } });
2295 }
2296
2297 /// THE LIST IS EXHAUSTIVE. An access requires its base live and its index in
2298 /// bounds, and it reads or writes that base. Its results are values carrying
2299 /// no identity. Nothing else happens, so the enumeration states `complete`.
2300 /// The requirements are what still refuse every permission: `total` refuses
2301 /// any requirement, so a load or a store is no more discardable, duplicable
2302 /// or reorderable than it was before this line.
2303 /// Whether the index operand is a literal the declared extent already admits.
2304 ///
2305 /// A REQUIREMENT IS A PREMISE TO PROVE AT THE USE, SO A PREMISE THE OPERANDS
2306 /// THEMSELVES SETTLE IS NOT ONE. `arith` states the same thing about division
2307 /// by a literal that is not zero and about a literal shift count: the
2308 /// declaration omits the requirement rather than restating what the operand
2309 /// says. This omits `in_bounds` only when the extent is a static size and the
2310 /// index is a literal below it, which is decided from the two operands alone
2311 /// and needs nothing about the program around them.
2312 ///
2313 /// It says nothing about liveness. `live` stays declared for every access,
2314 /// literal index or not, because an extent cannot prove that the base is
2315 /// still there to be read.
2316 fn indexProvenInBounds(
2317 op: *const ir.Operation,
2318 comptime base_index: usize,
2319 comptime index_index: usize,
2320 ) bool {
2321 const key = op.operands.items[base_index].value.type.getDialectParamKey() orelse return false;
2322 const params = MemrefDialect.parseMemrefParams(key) orelse return false;
2323 const extent = params.size orelse return false;
2324 const index = literalIndex(op.operands.items[index_index].value) orelse return false;
2325 if (index < 0) return false;
2326 return @as(u128, @intCast(index)) < @as(u128, extent);
2327 }
2328
2329 /// The value a literal index carries, or null when the operand is not one.
2330 fn literalIndex(value: *ir.Value) ?i64 {
2331 const raw = value.getDefiningOp() orelse return null;
2332 const definition: *ir.Operation = @ptrCast(@alignCast(raw));
2333 if (!std.mem.eql(u8, definition.name.name, arith.ArithDialect.ConstantOp.operation_name)) {
2334 return null;
2335 }
2336 const attr = definition.getAttr("value") orelse return null;
2337 return (attr.cast(ir.Attribute.IntegerAttr) orelse return null).getValue();
2338 }
2339
2340 fn accessEffects(
2341 comptime base_index: usize,
2342 comptime index_index: usize,
2343 comptime reads: bool,
2344 comptime writes: bool,
2345 comptime ordered: bool,
2346 ) ir.interfaces.InterfaceEntry {
2347 const Declaration = struct {
2348 fn enumerate(op: *const ir.Operation, collector: *effects.Collector) void {
2349 collector.valueResults(op);
2350 if (op.getNumOperands() <= @max(base_index, index_index)) return;
2351 var resource = effects.Resource{ .subject = .{ .operand = base_index } };
2352 if (op.operands.items[base_index].value.type.getDialectParamKey()) |key| {
2353 if (MemrefDialect.parseMemrefParams(key)) |params| {
2354 resource.address_space = @intCast(@backingInt(params.addr_space));
2355 }
2356 }
2357 collector.append(.{ .requirement = .{ .kind = .live, .subject = resource.subject } });
2358 if (!indexProvenInBounds(op, base_index, index_index)) {
2359 collector.append(.{ .requirement = .{
2360 .kind = .in_bounds,
2361 .subject = resource.subject,
2362 .related = .{ .operand = index_index },
2363 } });
2364 }
2365 if (reads) collector.append(.{ .event = .{
2366 .kind = .read,
2367 .resource = resource,
2368 .ordered = ordered,
2369 } });
2370 if (writes) collector.append(.{ .event = .{
2371 .kind = .write,
2372 .resource = resource,
2373 .ordered = ordered,
2374 } });
2375 }
2376 };
2377 return effects.EffectOpInterface.entryFor(.{
2378 .complete = true,
2379 .capacity = .{ .entries = 4, .per_result = 1 },
2380 .enumerate = Declaration.enumerate,
2381 });
2382 }
2383
2384 test "memref effect declarations preserve checked accesses and allocation identity" {
2385 const arithmetic = arith.ArithDialect;
2386 var ctx = try ir.Context.init(std.testing.allocator, ir.Context.Limits.testing);
2387 defer ctx.deinit(std.testing.allocator);
2388 const typ = try arithmetic.getI32Type(&ctx);
2389 const memref_type = try MemrefDialect.getMemrefType1D(&ctx, 4, typ, .host);
2390 const allocation = try MemrefDialect.AllocOp.createStatic(&ctx, .unknown, memref_type);
2391 const index = try arithmetic.ConstantOp.createInt(&ctx, .unknown, typ, 0);
2392 const load = try MemrefDialect.LoadOp.create(
2393 &ctx,
2394 .unknown,
2395 allocation.getResult(),
2396 index.getResult(),
2397 typ,
2398 );
2399 var load_facts = try effects.inspect(std.testing.allocator, load.op);
2400 defer load_facts.deinit(std.testing.allocator);
2401 try std.testing.expect(load_facts.facts.complete);
2402 try std.testing.expect(!effects.total(load_facts.facts));
2403 try std.testing.expect(!effects.discard(load_facts.facts));
2404 try std.testing.expect(!effects.duplicate(load_facts.facts, .{
2405 .read_values = true,
2406 .execution_context = true,
2407 }));
2408 try std.testing.expect(!effects.reorder(load_facts.facts, load_facts.facts, .{
2409 .no_dependencies = true,
2410 .concurrency_exclusive = true,
2411 }));
2412 try std.testing.expectEqual(
2413 effects.RequirementKind.live,
2414 load_facts.facts.records[1].requirement.kind,
2415 );
2416 try std.testing.expectEqual(effects.EventKind.read, load_facts.facts.records[2].event.kind);
2417 try std.testing.expectEqual(
2418 @as(usize, 0),
2419 load_facts.facts.records[2].event.resource.subject.operand,
2420 );
2421 for (load_facts.facts.records) |record| {
2422 if (record != .requirement) continue;
2423 try std.testing.expect(record.requirement.kind != .in_bounds);
2424 }
2425
2426 const computed = try MemrefDialect.LoadOp.create(
2427 &ctx,
2428 .unknown,
2429 allocation.getResult(),
2430 load.getResult(),
2431 typ,
2432 );
2433 var computed_facts = try effects.inspect(std.testing.allocator, computed.op);
2434 defer computed_facts.deinit(std.testing.allocator);
2435 try std.testing.expect(computed_facts.facts.complete);
2436 try std.testing.expectEqual(
2437 effects.RequirementKind.in_bounds,
2438 computed_facts.facts.records[2].requirement.kind,
2439 );
2440
2441 const past = try arithmetic.ConstantOp.createInt(&ctx, .unknown, typ, 4);
2442 const outside = try MemrefDialect.LoadOp.create(
2443 &ctx,
2444 .unknown,
2445 allocation.getResult(),
2446 past.getResult(),
2447 typ,
2448 );
2449 var outside_facts = try effects.inspect(std.testing.allocator, outside.op);
2450 defer outside_facts.deinit(std.testing.allocator);
2451 try std.testing.expectEqual(
2452 effects.RequirementKind.in_bounds,
2453 outside_facts.facts.records[2].requirement.kind,
2454 );
2455 var allocation_facts = try effects.inspect(std.testing.allocator, allocation.op);
2456 defer allocation_facts.deinit(std.testing.allocator);
2457 try std.testing.expectEqual(
2458 effects.EventKind.allocate,
2459 allocation_facts.facts.records[0].event.kind,
2460 );
2461 try std.testing.expect(allocation_facts.facts.records[2].result.fresh_identity);
2462 try std.testing.expect(allocation_facts.facts.complete);
2463 try std.testing.expect(!effects.total(allocation_facts.facts));
2464 try std.testing.expect(!effects.discard(allocation_facts.facts));
2465 try std.testing.expect(!effects.duplicate(allocation_facts.facts, .{}));
2466 }
2467
2468 test "MemrefDialect.GlobalOp maps declarations onto the three placements" {
2469 const testing = std.testing;
2470 var arena = alloc_arena.Arena.init(std.testing.allocator);
2471 defer arena.deinit();
2472 const allocator = arena.allocator();
2473
2474 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2475 defer ctx.deinit(allocator);
2476
2477 const loc = ir.Location.getUnknown();
2478 const i64_type = try arith.ArithDialect.getScalarType(&ctx, .i64);
2479 const u8_type = try arith.ArithDialect.getScalarType(&ctx, .u8);
2480 const word_type = try MemrefDialect.getMemrefType1D(&ctx, 1, i64_type, .host);
2481 const arena_type = try MemrefDialect.getMemrefType1D(&ctx, 4096, u8_type, .host);
2482 const word: []const u8 = &.{ 1, 2, 3, 4, 5, 6, 7, 8 };
2483
2484 const zeroed = try MemrefDialect.GlobalOp.create(&ctx, loc, .{
2485 .sym_name = "arena",
2486 .memref_type = arena_type,
2487 .alignment = 16,
2488 });
2489 try testing.expectEqualStrings("memref.global", zeroed.op.name.name);
2490 try testing.expectEqualStrings("arena", zeroed.getSymName().?);
2491 try testing.expectEqual(@as(u64, 16), zeroed.getAlignment().?);
2492 try testing.expect(zeroed.getInitial() == null);
2493 try testing.expectEqual(GlobalPlacement.zeroed, zeroed.getPlacement().?);
2494
2495 const writable = try MemrefDialect.GlobalOp.create(&ctx, loc, .{
2496 .sym_name = "seed",
2497 .memref_type = word_type,
2498 .alignment = 8,
2499 .initial = word,
2500 });
2501 try testing.expectEqual(GlobalPlacement.writable, writable.getPlacement().?);
2502 try testing.expectEqualSlices(u8, word, writable.getInitial().?);
2503
2504 const read_only = try MemrefDialect.GlobalOp.create(&ctx, loc, .{
2505 .sym_name = "table",
2506 .memref_type = word_type,
2507 .alignment = 8,
2508 .constant = true,
2509 .initial = word,
2510 });
2511 try testing.expectEqual(GlobalPlacement.read_only, read_only.getPlacement().?);
2512 }
2513
2514 test "MemrefDialect.GlobalOp refuses a declaration no section can hold" {
2515 const testing = std.testing;
2516 var arena = alloc_arena.Arena.init(std.testing.allocator);
2517 defer arena.deinit();
2518 const allocator = arena.allocator();
2519
2520 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2521 defer ctx.deinit(allocator);
2522
2523 const loc = ir.Location.getUnknown();
2524 const i64_type = try arith.ArithDialect.getScalarType(&ctx, .i64);
2525 const word_type = try MemrefDialect.getMemrefType1D(&ctx, 1, i64_type, .host);
2526
2527 try testing.expectError(
2528 MemrefVerifyError.GlobalConstantWithoutInitial,
2529 MemrefDialect.GlobalOp.create(&ctx, loc, .{
2530 .sym_name = "table",
2531 .memref_type = word_type,
2532 .alignment = 8,
2533 .constant = true,
2534 }),
2535 );
2536 try testing.expectError(
2537 MemrefVerifyError.GlobalInitialLengthMismatch,
2538 MemrefDialect.GlobalOp.create(&ctx, loc, .{
2539 .sym_name = "table",
2540 .memref_type = word_type,
2541 .alignment = 8,
2542 .constant = true,
2543 .initial = &.{ 1, 2, 3 },
2544 }),
2545 );
2546 try testing.expectError(
2547 MemrefVerifyError.GlobalInvalidAlignment,
2548 MemrefDialect.GlobalOp.create(&ctx, loc, .{
2549 .sym_name = "table",
2550 .memref_type = word_type,
2551 .alignment = 3,
2552 }),
2553 );
2554 }
2555
2556 test "memref global declarations and address computations declare their effects" {
2557 const testing = std.testing;
2558 var ctx = try ir.Context.init(testing.allocator, ir.Context.Limits.testing);
2559 defer ctx.deinit(testing.allocator);
2560
2561 const loc = ir.Location.getUnknown();
2562 const u8_type = try arith.ArithDialect.getScalarType(&ctx, .u8);
2563 const arena_type = try MemrefDialect.getMemrefType1D(&ctx, 4096, u8_type, .host);
2564
2565 const global = try MemrefDialect.GlobalOp.create(&ctx, loc, .{
2566 .sym_name = "arena",
2567 .memref_type = arena_type,
2568 .alignment = 16,
2569 });
2570 var global_facts = try effects.inspect(testing.allocator, global.op);
2571 defer global_facts.deinit(testing.allocator);
2572 try testing.expectEqual(@as(usize, 0), global_facts.facts.records.len);
2573 try testing.expect(effects.memoryFree(global_facts.facts));
2574 try testing.expect(effects.total(global_facts.facts));
2575
2576 const address = try MemrefDialect.GetGlobalOp.create(&ctx, loc, "arena", arena_type);
2577 var address_facts = try effects.inspect(testing.allocator, address.op);
2578 defer address_facts.deinit(testing.allocator);
2579 try testing.expect(effects.memoryFree(address_facts.facts));
2580 try testing.expect(effects.total(address_facts.facts));
2581 try testing.expectEqual(
2582 effects.Ownership.none,
2583 address_facts.facts.records[0].result.ownership,
2584 );
2585 }
2586
2587 test "MemrefDialect.ViewOp offsets a byte base and refuses any other" {
2588 const testing = std.testing;
2589 var arena = alloc_arena.Arena.init(std.testing.allocator);
2590 defer arena.deinit();
2591 const allocator = arena.allocator();
2592
2593 var ctx = try ir.Context.init(allocator, ir.Context.Limits.testing);
2594 defer ctx.deinit(allocator);
2595
2596 const loc = ir.Location.getUnknown();
2597 const i64_type = try arith.ArithDialect.getScalarType(&ctx, .i64);
2598 const u8_type = try arith.ArithDialect.getScalarType(&ctx, .u8);
2599 const index_type = try arith.ArithDialect.getIndexType(&ctx);
2600 const word_type = try MemrefDialect.getMemrefType1D(&ctx, 1, i64_type, .host);
2601 const arena_type = try MemrefDialect.getMemrefType1D(&ctx, 4096, u8_type, .host);
2602 const words_type = try MemrefDialect.getMemrefType1D(&ctx, 4, i64_type, .host);
2603
2604 var offset = try arith.ArithDialect.ConstantOp.createInt(&ctx, loc, index_type, 8);
2605 var bytes = try MemrefDialect.AllocaOp.createStatic(&ctx, loc, arena_type);
2606 const view = try MemrefDialect.ViewOp.create(
2607 &ctx,
2608 loc,
2609 bytes.getResult(),
2610 offset.getResult(),
2611 word_type,
2612 );
2613 try testing.expectEqualStrings("memref.view", view.op.name.name);
2614 try testing.expect(view.getBase() == bytes.getResult());
2615 try testing.expect(view.getByteOffset() == offset.getResult());
2616 try testing.expect(view.getResult().type.eql(word_type));
2617
2618 var words = try MemrefDialect.AllocaOp.createStatic(&ctx, loc, words_type);
2619 try testing.expectError(MemrefVerifyError.ViewBaseNotBytes, MemrefDialect.ViewOp.create(
2620 &ctx,
2621 loc,
2622 words.getResult(),
2623 offset.getResult(),
2624 word_type,
2625 ));
2626
2627 var view_facts = try effects.inspect(testing.allocator, view.op);
2628 defer view_facts.deinit(testing.allocator);
2629 try testing.expect(!view_facts.facts.complete);
2630 try testing.expectEqual(effects.EventKind.borrow, view_facts.facts.records[0].event.kind);
2631 try testing.expectEqual(
2632 @as(usize, 0),
2633 view_facts.facts.records[0].event.resource.subject.operand,
2634 );
2635 try testing.expectEqual(
2636 effects.Ownership.borrowed,
2637 view_facts.facts.records[1].result.ownership,
2638 );
2639 try testing.expectEqual(@as(usize, 0), view_facts.facts.records[1].result.alias.?.operand);
2640 }