lib/simd/src/thread/spin.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const builtin = @import("builtin");
3 const sys = @import("sys");
4
5 pub const SpinType = enum(u8) {
6 monitor_x = 1,
7 u_monitor = 2,
8 pause = 3,
9 };
10
11 pub const SpinResult = struct {
12 value: u32,
13 repetitions: u32,
14 };
15
16 pub const SpinPause = struct {
17 pub fn spinType(_: SpinPause) SpinType {
18 return .pause;
19 }
20
21 pub fn untilDifferent(
22 _: SpinPause,
23 previous: u32,
24 watched: *const std.atomic.Value(u32),
25 ) SpinResult {
26 var repetitions: u32 = 0;
27 while (true) : (repetitions +%= 1) {
28 const current = watched.load(.acquire);
29 if (current != previous) {
30 return .{
31 .value = current,
32 .repetitions = repetitions,
33 };
34 }
35 std.atomic.spinLoopHint();
36 }
37 }
38
39 pub fn untilEqual(
40 _: SpinPause,
41 expected: u32,
42 watched: *const std.atomic.Value(u32),
43 ) usize {
44 var repetitions: usize = 0;
45 while (watched.load(.acquire) != expected) : (repetitions +%= 1) {
46 std.atomic.spinLoopHint();
47 }
48 return repetitions;
49 }
50 };
51
52 pub const SpinMonitorX = struct {
53 pub fn spinType(_: SpinMonitorX) SpinType {
54 return .monitor_x;
55 }
56
57 pub fn untilDifferent(
58 _: SpinMonitorX,
59 previous: u32,
60 watched: *const std.atomic.Value(u32),
61 ) SpinResult {
62 if (comptime !haveMonitorX()) {
63 return (SpinPause{}).untilDifferent(previous, watched);
64 }
65 var repetitions: u32 = 0;
66 while (true) : (repetitions +%= 1) {
67 var current = watched.load(.acquire);
68 if (current != previous) {
69 return .{
70 .value = current,
71 .repetitions = repetitions,
72 };
73 }
74 monitorX(&watched.raw);
75 current = watched.load(.acquire);
76 if (current != previous) {
77 return .{
78 .value = current,
79 .repetitions = repetitions,
80 };
81 }
82 waitX();
83 }
84 }
85
86 pub fn untilEqual(
87 _: SpinMonitorX,
88 expected: u32,
89 watched: *const std.atomic.Value(u32),
90 ) usize {
91 if (comptime !haveMonitorX()) {
92 return (SpinPause{}).untilEqual(expected, watched);
93 }
94 var repetitions: usize = 0;
95 while (true) : (repetitions +%= 1) {
96 var current = watched.load(.acquire);
97 if (current == expected) return repetitions;
98 monitorX(&watched.raw);
99 current = watched.load(.acquire);
100 if (current == expected) return repetitions;
101 waitX();
102 }
103 }
104 };
105
106 pub const SpinUMonitor = struct {
107 pub fn spinType(_: SpinUMonitor) SpinType {
108 return .u_monitor;
109 }
110
111 pub fn untilDifferent(
112 _: SpinUMonitor,
113 previous: u32,
114 watched: *const std.atomic.Value(u32),
115 ) SpinResult {
116 if (comptime !haveUMonitor()) {
117 return (SpinPause{}).untilDifferent(previous, watched);
118 }
119 var repetitions: u32 = 0;
120 while (true) : (repetitions +%= 1) {
121 var current = watched.load(.acquire);
122 if (current != previous) {
123 return .{
124 .value = current,
125 .repetitions = repetitions,
126 };
127 }
128 uMonitor(&watched.raw);
129 current = watched.load(.acquire);
130 if (current != previous) {
131 return .{
132 .value = current,
133 .repetitions = repetitions,
134 };
135 }
136 uWait();
137 }
138 }
139
140 pub fn untilEqual(
141 _: SpinUMonitor,
142 expected: u32,
143 watched: *const std.atomic.Value(u32),
144 ) usize {
145 if (comptime !haveUMonitor()) {
146 return (SpinPause{}).untilEqual(expected, watched);
147 }
148 var repetitions: usize = 0;
149 while (true) : (repetitions +%= 1) {
150 var current = watched.load(.acquire);
151 if (current == expected) return repetitions;
152 uMonitor(&watched.raw);
153 current = watched.load(.acquire);
154 if (current == expected) return repetitions;
155 uWait();
156 }
157 }
158 };
159
160 pub fn name(spin_type: SpinType) []const u8 {
161 return switch (spin_type) {
162 .monitor_x => "MonitorX_C1",
163 .u_monitor => "UMonitor_C0.2",
164 .pause => "Pause",
165 };
166 }
167
168 pub fn detectSpin(disabled: u8) SpinType {
169 if (comptime isX86()) {
170 const leaf0 = cpuid(0, 0);
171 if (comptime haveMonitorX()) {
172 if (enabled(disabled, .monitor_x) and isAmd(leaf0)) {
173 const extended = cpuid(0x8000_0000, 0);
174 if (extended.eax >= 0x8000_0001) {
175 const features = cpuid(0x8000_0001, 0);
176 if (features.ecx & (@as(u32, 1) << 29) != 0) {
177 return .monitor_x;
178 }
179 }
180 }
181 }
182 if (comptime haveUMonitor()) {
183 if (enabled(disabled, .u_monitor) and leaf0.eax >= 7) {
184 const features = cpuid(7, 0);
185 if (features.ecx & (@as(u32, 1) << 5) != 0) {
186 return .u_monitor;
187 }
188 }
189 }
190 }
191 return .pause;
192 }
193
194 pub fn callWithSpin(
195 spin_type: SpinType,
196 context: anytype,
197 comptime call: anytype,
198 ) void {
199 switch (spin_type) {
200 .monitor_x => if (comptime haveMonitorX())
201 call(context, SpinMonitorX{})
202 else
203 call(context, SpinPause{}),
204 .u_monitor => if (comptime haveUMonitor())
205 call(context, SpinUMonitor{})
206 else
207 call(context, SpinPause{}),
208 .pause => call(context, SpinPause{}),
209 }
210 }
211
212 const X86Leaf = struct {
213 eax: u32,
214 ebx: u32,
215 ecx: u32,
216 edx: u32,
217 };
218
219 fn isX86() bool {
220 return builtin.cpu.arch == .x86 or builtin.cpu.arch == .x86_64;
221 }
222
223 fn haveMonitorX() bool {
224 return builtin.cpu.arch == .x86_64;
225 }
226
227 fn haveUMonitor() bool {
228 return builtin.cpu.arch == .x86_64;
229 }
230
231 fn enabled(disabled: u8, spin_type: SpinType) bool {
232 const shift: u3 = @intCast(@backingInt(spin_type));
233 return disabled & (@as(u8, 1) << shift) == 0;
234 }
235
236 fn cpuid(leaf: u32, subleaf: u32) X86Leaf {
237 var eax: u32 = undefined;
238 var ebx: u32 = undefined;
239 var ecx: u32 = undefined;
240 var edx: u32 = undefined;
241 asm volatile ("cpuid"
242 : [_] "={eax}" (eax),
243 [_] "={ebx}" (ebx),
244 [_] "={ecx}" (ecx),
245 [_] "={edx}" (edx),
246 : [_] "{eax}" (leaf),
247 [_] "{ecx}" (subleaf),
248 );
249 return .{ .eax = eax, .ebx = ebx, .ecx = ecx, .edx = edx };
250 }
251
252 fn isAmd(leaf: X86Leaf) bool {
253 return leaf.eax >= 1 and
254 leaf.ebx == 0x6874_7541 and
255 leaf.ecx == 0x444d_4163 and
256 leaf.edx == 0x6974_6e65;
257 }
258
259 fn monitorX(address: *const u32) void {
260 if (comptime builtin.cpu.arch == .x86_64) {
261 asm volatile (
262 \\jmp 2f
263 \\1:
264 \\mov $0xc3fa010f, %%r11d
265 \\2:
266 \\lea 1b(%%rip), %%r11
267 \\add $2, %%r11
268 \\call *%%r11
269 :
270 : [address] "{rax}" (@intFromPtr(address)),
271 [extensions] "{rcx}" (@as(usize, 0)),
272 [hints] "{rdx}" (@as(usize, 0)),
273 : .{ .cc = true, .r11 = true, .memory = true });
274 } else unreachable;
275 }
276
277 fn waitX() void {
278 if (comptime builtin.cpu.arch == .x86_64) {
279 asm volatile (
280 \\jmp 2f
281 \\1:
282 \\mov $0xc3fb010f, %%r11d
283 \\2:
284 \\lea 1b(%%rip), %%r11
285 \\add $2, %%r11
286 \\call *%%r11
287 :
288 : [hints] "{eax}" (@as(u32, 0)),
289 [cycles] "{ebx}" (@as(u32, 0)),
290 [extensions] "{ecx}" (@as(u32, 0)),
291 : .{ .cc = true, .r11 = true, .memory = true });
292 } else unreachable;
293 }
294
295 fn uMonitor(address: *const u32) void {
296 if (comptime builtin.cpu.arch == .x86_64) {
297 asm volatile (
298 \\jmp 2f
299 \\1:
300 \\mov $0x000000c3f0ae0ff3, %%r11
301 \\2:
302 \\lea 1b(%%rip), %%r11
303 \\add $2, %%r11
304 \\call *%%r11
305 :
306 : [address] "{rax}" (@intFromPtr(address)),
307 : .{ .cc = true, .r11 = true, .memory = true });
308 } else unreachable;
309 }
310
311 fn uWait() void {
312 if (comptime builtin.cpu.arch == .x86_64) {
313 asm volatile (
314 \\jmp 2f
315 \\1:
316 \\mov $0x000000c3f1ae0ff2, %%r11
317 \\2:
318 \\lea 1b(%%rip), %%r11
319 \\add $2, %%r11
320 \\call *%%r11
321 :
322 : [control] "{ecx}" (@as(u32, 0)),
323 [deadline_low] "{eax}" (std.math.maxInt(u32)),
324 [deadline_high] "{edx}" (std.math.maxInt(u32)),
325 : .{ .cc = true, .r11 = true, .memory = true });
326 } else unreachable;
327 }
328
329 test "Highway spin names and disabled detection preserve pause fallback" {
330 try std.testing.expectEqualStrings("MonitorX_C1", name(.monitor_x));
331 try std.testing.expectEqualStrings("UMonitor_C0.2", name(.u_monitor));
332 try std.testing.expectEqualStrings("Pause", name(.pause));
333 try std.testing.expectEqual(
334 SpinType.pause,
335 detectSpin(
336 (@as(u8, 1) << @as(u3, @intCast(@backingInt(SpinType.monitor_x)))) |
337 (@as(u8, 1) << @as(u3, @intCast(@backingInt(SpinType.u_monitor)))),
338 ),
339 );
340 }
341
342 test "Highway spin pause observes changes and equality" {
343 var watched = std.atomic.Value(u32).init(3);
344 watched.store(5, .release);
345 const result = (SpinPause{}).untilDifferent(3, &watched);
346 try std.testing.expectEqual(@as(u32, 5), result.value);
347 try std.testing.expectEqual(@as(u32, 0), result.repetitions);
348 try std.testing.expectEqual(
349 @as(usize, 0),
350 (SpinPause{}).untilEqual(5, &watched),
351 );
352 }
353
354 const PingPongState = struct {
355 ready: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),
356 active: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
357 done: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
358 observed: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
359 first_repetitions: std.atomic.Value(u32) =
360 std.atomic.Value(u32).init(0),
361 second_repetitions: std.atomic.Value(usize) =
362 std.atomic.Value(usize).init(0),
363 };
364
365 fn PingPongWorker(comptime Policy: type) type {
366 return struct {
367 const Context = struct {
368 state: *PingPongState,
369 policy: Policy,
370 };
371
372 fn run(context: Context) void {
373 context.state.ready.store(true, .release);
374 const result = context.policy.untilDifferent(
375 0,
376 &context.state.active,
377 );
378 context.state.observed.store(result.value, .release);
379 context.state.first_repetitions.store(
380 result.repetitions,
381 .release,
382 );
383 sys.time.sleepNanoseconds(20 * std.time.ns_per_ms);
384 context.state.done.store(1, .release);
385 }
386 };
387 }
388
389 const PingPongInvocation = struct {
390 success: *bool,
391 elapsed_ns: *u64,
392 };
393
394 fn runPingPong(context: PingPongInvocation, policy: anytype) void {
395 var state = PingPongState{};
396 const Worker = PingPongWorker(@TypeOf(policy));
397 const handle = sys.thread.spawn(Worker.run, .{
398 Worker.Context{ .state = &state, .policy = policy },
399 }) catch return;
400 const started = sys.time.nanoTimestamp();
401 while (!state.ready.load(.acquire)) std.atomic.spinLoopHint();
402 sys.time.sleepNanoseconds(30 * std.time.ns_per_ms);
403 state.active.store(1, .release);
404 const repetitions = policy.untilEqual(1, &state.done);
405 state.second_repetitions.store(repetitions, .release);
406 handle.join();
407 const elapsed = sys.time.nanoTimestamp() - started;
408 if (elapsed < 0) return;
409 context.elapsed_ns.* = @intCast(elapsed);
410 context.success.* =
411 state.observed.load(.acquire) == 1 and
412 state.done.load(.acquire) == 1 and
413 context.elapsed_ns.* > 25 * std.time.ns_per_ms;
414 }
415
416 test "Highway detected spin policy completes delayed ping pong" {
417 if (!sys.thread.threadsSupported() or
418 !sys.time.supportsAwakeClock())
419 {
420 return error.SkipZigTest;
421 }
422 var success = false;
423 var elapsed_ns: u64 = 0;
424 callWithSpin(
425 detectSpin(0),
426 PingPongInvocation{
427 .success = &success,
428 .elapsed_ns = &elapsed_ns,
429 },
430 runPingPong,
431 );
432 try std.testing.expect(success);
433 try std.testing.expect(elapsed_ns > 25 * std.time.ns_per_ms);
434 }