lib/choir/src/backends/x64/invoke.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 //! Checked calls into SysV machine code.
2 //! A call passes every argument as a 64-bit word through one fixed C function type, then reads
3 //! each result where the recorded signature returns it.
4
5 const std = @import("std");
6 const artifact = @import("../root.zig").artifact;
7 const dialects = @import("../../dialects/root.zig");
8 const abi = @import("abi.zig");
9 const registers = @import("registers/root.zig");
10 const slot_layout = @import("slots.zig");
11
12 const Signature = artifact.Signature;
13 const ScalarType = artifact.ScalarType;
14 const ValueType = artifact.ValueType;
15 const VectorType = artifact.VectorType;
16
17 pub const Error = error{SignatureMismatch};
18
19 /// One vector result. Each lane holds its element's bits, zero-extended to 64.
20 pub const Vector = struct {
21 element: ScalarType,
22 lanes: u8,
23 /// Entries past `lanes` hold zero.
24 bits: [VectorType.max_lanes]u64,
25 };
26
27 /// One argument or result of a checked call.
28 /// Each scalar variant holds the boundary scalar of the same name. `index` holds a 64-bit word,
29 /// and `memref` holds an address.
30 pub const Value = union(enum) {
31 i8: i8,
32 i16: i16,
33 i32: i32,
34 i64: i64,
35 u8: u8,
36 u16: u16,
37 u32: u32,
38 u64: u64,
39 index: u64,
40 bool: bool,
41 f32: f32,
42 f64: f64,
43 memref: usize,
44 vector: Vector,
45
46 /// Returns the boundary type that a call checks this value against.
47 pub fn valueType(self: Value) ValueType {
48 return switch (self) {
49 .memref => .memref,
50 .vector => |vector| .{
51 .vector = .{ .element = vector.element, .lanes = vector.lanes },
52 },
53 inline else => |_, tag| .{ .scalar = @field(ScalarType, @tagName(tag)) },
54 };
55 }
56 };
57
58 const int_words = abi.max_int_arg_regs;
59 const float_words = abi.max_fp_arg_regs;
60 const stack_words = 64;
61
62 comptime {
63 std.debug.assert(Signature.max_parameters + 1 - int_words <= stack_words);
64 }
65
66 const entry_parameter_types = blk: {
67 var types: [int_words + float_words + stack_words]type = @splat(u64);
68 for (types[int_words..][0..float_words]) |*parameter_type| parameter_type.* = f64;
69 break :blk types;
70 };
71
72 fn EntryFn(comptime Return: type) type {
73 return @Fn(&entry_parameter_types, &@splat(.{}), Return, .{ .@"callconv" = .c });
74 }
75
76 const EntryArguments = std.meta.ArgsTuple(EntryFn(void));
77
78 const Frame = struct {
79 ints: [int_words]u64 = @splat(0),
80 floats: [float_words]u64 = @splat(0),
81 stack: [stack_words]u64 = @splat(0),
82
83 fn place(self: *Frame, location: abi.ValueLocation, word: u64) void {
84 switch (location) {
85 .int_reg => |register| {
86 const index = std.mem.indexOfScalar(registers.GPR, &abi.int_arg_regs, register).?;
87 self.ints[index] = word;
88 },
89 .fp_reg => |register| {
90 const index = std.mem.indexOfScalar(registers.XMM, &abi.fp_arg_regs, register).?;
91 self.floats[index] = word;
92 },
93 .stack => |offset| {
94 std.debug.assert(offset >= 0);
95 const index: usize = @intCast(@divExact(offset, abi.stack_slot_size));
96 self.stack[index] = word;
97 },
98 }
99 }
100
101 fn arguments(self: *const Frame) EntryArguments {
102 var tuple: EntryArguments = undefined;
103 inline for (0..int_words) |index| tuple[index] = self.ints[index];
104 inline for (0..float_words) |index| tuple[int_words + index] = @bitCast(self.floats[index]);
105 inline for (0..stack_words) |index| {
106 tuple[int_words + float_words + index] = self.stack[index];
107 }
108 return tuple;
109 }
110 };
111
112 const ResultWords = struct {
113 ints: [abi.max_register_results]u64 = @splat(0),
114 floats: [abi.max_register_results]u64 = @splat(0),
115 };
116
117 const Shape = enum { none, int, float, int_int, int_float, float_int, float_float };
118
119 const IntInt = extern struct { first: u64, second: u64 };
120 const IntFloat = extern struct { first: u64, second: f64 };
121 const FloatInt = extern struct { first: f64, second: u64 };
122 const FloatFloat = extern struct { first: f64, second: f64 };
123
124 /// Calls the function at `address`, whose parameter and result types are `signature`, and stores
125 /// its results in `results`. Fails with `error.SignatureMismatch` before calling when `args`
126 /// differ from the parameters in count or type, or when `results` has a different length than the
127 /// signature's result list.
128 pub fn call(
129 address: usize,
130 signature: *const Signature,
131 args: []const Value,
132 results: []Value,
133 ) Error!void {
134 std.debug.assert(address != 0);
135 std.debug.assert(abi.admitsSignature(signature));
136 if (args.len != signature.parameter_count) return error.SignatureMismatch;
137 if (results.len != signature.result_count) return error.SignatureMismatch;
138 for (args, signature.parameters()) |arg, parameter| {
139 if (!arg.valueType().eql(parameter)) return error.SignatureMismatch;
140 }
141 const result_types = signature.results();
142 if (result_types.len == 1 and result_types[0] == .vector) {
143 results[0] = .{ .vector = callVector(address, signature, args) };
144 } else if (abi.returnsThroughRecord(result_types.len)) {
145 var record: [Signature.max_results]u64 = @splat(0);
146 const frame = pack(signature, @intFromPtr(&record), args);
147 _ = enter(address, .none, &frame);
148 for (result_types, results, 0..) |result_type, *result, index| {
149 std.debug.assert(abi.recordFieldOffset(index) == index * @sizeOf(u64));
150 result.* = valueOf(result_type, record[index]);
151 }
152 } else {
153 const frame = pack(signature, null, args);
154 const words = enter(address, shapeOf(result_types), &frame);
155 readRegisterResults(result_types, words, results);
156 }
157 }
158
159 fn pack(signature: *const Signature, hidden: ?usize, args: []const Value) Frame {
160 std.debug.assert(args.len == signature.parameter_count);
161 const first: usize = @intFromBool(hidden != null);
162 const count = first + args.len;
163 var names: [Signature.max_parameters + 1][]const u8 = undefined;
164 if (hidden != null) names[0] = "arith.index";
165 for (signature.parameters(), names[first..count]) |parameter, *name| {
166 name.* = typeName(parameter);
167 }
168 var locations: [Signature.max_parameters + 1]abi.ValueLocation = undefined;
169 abi.computeArgLocations(names[0..count], locations[0..count]);
170 var frame = Frame{};
171 if (hidden) |result_address| frame.place(locations[0], result_address);
172 for (args, locations[first..count]) |arg, location| frame.place(location, wordOf(arg));
173 return frame;
174 }
175
176 fn enter(address: usize, shape: Shape, frame: *const Frame) ResultWords {
177 const arguments = frame.arguments();
178 var words = ResultWords{};
179 switch (shape) {
180 .none => @call(.auto, entryAt(void, address), arguments),
181 .int => words.ints[0] = @call(.auto, entryAt(u64, address), arguments),
182 .float => words.floats[0] = @bitCast(@call(.auto, entryAt(f64, address), arguments)),
183 .int_int => {
184 const pair = @call(.auto, entryAt(IntInt, address), arguments);
185 words.ints = .{ pair.first, pair.second };
186 },
187 .int_float => {
188 const pair = @call(.auto, entryAt(IntFloat, address), arguments);
189 words.ints[0] = pair.first;
190 words.floats[0] = @bitCast(pair.second);
191 },
192 .float_int => {
193 const pair = @call(.auto, entryAt(FloatInt, address), arguments);
194 words.floats[0] = @bitCast(pair.first);
195 words.ints[0] = pair.second;
196 },
197 .float_float => {
198 const pair = @call(.auto, entryAt(FloatFloat, address), arguments);
199 words.floats = .{ @bitCast(pair.first), @bitCast(pair.second) };
200 },
201 }
202 return words;
203 }
204
205 fn entryAt(comptime Return: type, address: usize) *const EntryFn(Return) {
206 return @ptrFromInt(address);
207 }
208
209 fn shapeOf(result_types: []const ValueType) Shape {
210 std.debug.assert(result_types.len <= abi.max_register_results);
211 if (result_types.len == 0) return .none;
212 const first = abi.classifyType(typeName(result_types[0]));
213 if (result_types.len == 1) return switch (first) {
214 .integer => .int,
215 .floating_point => .float,
216 };
217 return switch (first) {
218 .integer => switch (abi.classifyType(typeName(result_types[1]))) {
219 .integer => .int_int,
220 .floating_point => .int_float,
221 },
222 .floating_point => switch (abi.classifyType(typeName(result_types[1]))) {
223 .integer => .float_int,
224 .floating_point => .float_float,
225 },
226 };
227 }
228
229 fn readRegisterResults(result_types: []const ValueType, words: ResultWords, results: []Value) void {
230 std.debug.assert(result_types.len == results.len);
231 std.debug.assert(result_types.len <= abi.max_register_results);
232 var names: [abi.max_register_results][]const u8 = undefined;
233 for (result_types, names[0..result_types.len]) |result_type, *name| {
234 name.* = typeName(result_type);
235 }
236 var locations: [abi.max_register_results]abi.ValueLocation = undefined;
237 abi.computeReturnLocations(names[0..result_types.len], &locations);
238 for (result_types, locations[0..result_types.len], results) |result_type, location, *result| {
239 const word = switch (location) {
240 .int_reg => |register| words.ints[
241 std.mem.indexOfScalar(registers.GPR, &abi.int_return_regs, register).?
242 ],
243 .fp_reg => |register| words.floats[
244 std.mem.indexOfScalar(registers.XMM, &abi.fp_return_regs, register).?
245 ],
246 .stack => unreachable,
247 };
248 result.* = valueOf(result_type, word);
249 }
250 }
251
252 fn callVector(address: usize, signature: *const Signature, args: []const Value) Vector {
253 const vector = signature.results()[0].vector;
254 std.debug.assert(vector.lanes != 0);
255 std.debug.assert(vector.lanes <= VectorType.max_lanes);
256 const lane = slot_layout.Slot{ .offset = 0, .width = scalarBits(vector.element), .ext = .unsigned };
257 const stride = slot_layout.laneStride(lane);
258 var bytes: [VectorType.max_lanes * @sizeOf(u64)]u8 = @splat(0);
259 const top = (@as(usize, vector.lanes) - 1) * stride;
260 const frame = pack(signature, @intFromPtr(&bytes[top]), args);
261 _ = enter(address, .none, &frame);
262 var result = Vector{ .element = vector.element, .lanes = vector.lanes, .bits = @splat(0) };
263 for (result.bits[0..vector.lanes], 0..) |*bits, index| {
264 const below: usize = @intCast(-slot_layout.laneDisplacement(lane, index));
265 bits.* = std.mem.readVarInt(u64, bytes[top - below ..][0..stride], .little);
266 }
267 return result;
268 }
269
270 fn scalarBits(scalar: ScalarType) u8 {
271 return switch (scalar) {
272 .i8, .u8, .bool => 8,
273 .i16, .u16 => 16,
274 .i32, .u32, .f32 => 32,
275 .i64, .u64, .index, .f64 => 64,
276 };
277 }
278
279 fn typeName(value_type: ValueType) []const u8 {
280 return switch (value_type) {
281 .scalar => |scalar| switch (scalar) {
282 inline else => |tag| "arith." ++ @tagName(tag),
283 },
284 .memref => dialects.MemrefDialect.name,
285 .vector => unreachable,
286 };
287 }
288
289 fn wordOf(value: Value) u64 {
290 return switch (value) {
291 inline .i8, .i16, .i32, .i64 => |signed| @bitCast(@as(i64, signed)),
292 inline .u8, .u16, .u32, .u64, .index, .memref => |unsigned| unsigned,
293 .bool => |flag| @intFromBool(flag),
294 .f32 => |float| @as(u32, @bitCast(float)),
295 .f64 => |float| @bitCast(float),
296 .vector => unreachable,
297 };
298 }
299
300 fn valueOf(value_type: ValueType, word: u64) Value {
301 switch (value_type) {
302 .memref => return .{ .memref = word },
303 .vector => unreachable,
304 .scalar => |scalar| switch (scalar) {
305 .index => return .{ .index = word },
306 .f64 => return .{ .f64 = @bitCast(word) },
307 .f32 => return .{ .f32 = @bitCast(@as(u32, @truncate(word))) },
308 .bool => {
309 const byte: u8 = @truncate(word);
310 std.debug.assert(byte <= 1);
311 return .{ .bool = byte == 1 };
312 },
313 inline .i8, .i16, .i32, .i64, .u8, .u16, .u32, .u64 => |tag| {
314 const Integer = @FieldType(Value, @tagName(tag));
315 const Bits = @Int(.unsigned, @bitSizeOf(Integer));
316 return @unionInit(Value, @tagName(tag), @bitCast(@as(Bits, @truncate(word))));
317 },
318 },
319 }
320 }
321
322 test "checked calls place arguments where the emitter reads them" {
323 var parameters: [18]ValueType = undefined;
324 @memset(parameters[0..7], .{ .scalar = .i8 });
325 @memset(parameters[7..16], .{ .scalar = .f32 });
326 parameters[16] = .memref;
327 parameters[17] = .{ .scalar = .f64 };
328 const record = [_]ValueType{ .{ .scalar = .u16 }, .{ .scalar = .bool }, .memref };
329 const signature = try Signature.init(¶meters, &record);
330 var args: [parameters.len]Value = undefined;
331 for (args[0..7], 0..) |*arg, index| arg.* = .{ .i8 = -@as(i8, @intCast(index)) - 1 };
332 for (args[7..16], 0..) |*arg, index| arg.* = .{ .f32 = @floatFromInt(index) };
333 args[16] = .{ .memref = 0x1000 };
334 args[17] = .{ .f64 = -0.5 };
335
336 const frame = pack(&signature, 0xabc0, &args);
337 try std.testing.expectEqual(@as(u64, 0xabc0), frame.ints[0]);
338 try std.testing.expectEqual(@as(u64, @bitCast(@as(i64, -5))), frame.ints[5]);
339 const f32_bits: u64 = @as(u32, @bitCast(@as(f32, 7.0)));
340 try std.testing.expectEqual(f32_bits, frame.floats[7]);
341 try std.testing.expectEqual(@as(u64, @bitCast(@as(i64, -6))), frame.stack[0]);
342 try std.testing.expectEqual(@as(u64, @bitCast(@as(i64, -7))), frame.stack[1]);
343 try std.testing.expectEqual(@as(u64, @as(u32, @bitCast(@as(f32, 8.0)))), frame.stack[2]);
344 try std.testing.expectEqual(@as(u64, 0x1000), frame.stack[3]);
345 try std.testing.expectEqual(@as(u64, @bitCast(@as(f64, -0.5))), frame.stack[4]);
346 try std.testing.expectEqual(@as(u64, 0), frame.stack[5]);
347 }
348
349 test "checked call results keep their natural width and signedness" {
350 const all_ones = std.math.maxInt(u64);
351 try std.testing.expectEqual(Value{ .i8 = -1 }, valueOf(.{ .scalar = .i8 }, all_ones));
352 try std.testing.expectEqual(Value{ .u8 = 0xff }, valueOf(.{ .scalar = .u8 }, all_ones));
353 try std.testing.expectEqual(Value{ .i32 = -1 }, valueOf(.{ .scalar = .i32 }, all_ones));
354 try std.testing.expectEqual(Value{ .u16 = 0x8001 }, valueOf(.{ .scalar = .u16 }, 0xff_8001));
355 try std.testing.expectEqual(Value{ .bool = true }, valueOf(.{ .scalar = .bool }, 0xff01));
356 const nan_bits: u32 = 0x7fc0_0001;
357 const nan = valueOf(.{ .scalar = .f32 }, 0xdead_0000_0000_0000 | @as(u64, nan_bits));
358 try std.testing.expectEqual(nan_bits, @as(u32, @bitCast(nan.f32)));
359 try std.testing.expectEqual(ValueType{ .scalar = .index }, (Value{ .index = 3 }).valueType());
360 const lanes = Value{ .vector = .{ .element = .u8, .lanes = 2, .bits = @splat(0) } };
361 const lanes_type = ValueType{ .vector = .{ .element = .u8, .lanes = 2 } };
362 try std.testing.expect(lanes.valueType().eql(lanes_type));
363 }