lib/simd/src/target.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const builtin = @import("builtin");
3 const sys = @import("sys");
4
5 pub const Mask = u64;
6 pub const target_count: usize = 28;
7
8 pub const Target = enum(u6) {
9 avx10_2 = 3,
10 avx3_spr = 4,
11 avx3_zen4 = 6,
12 avx3_dl = 7,
13 avx3 = 8,
14 avx2 = 9,
15 sse4 = 11,
16 ssse3 = 12,
17 sse2 = 14,
18 sve2_128 = 18,
19 sve_256 = 19,
20 sve2 = 23,
21 sve = 24,
22 neon_bf16 = 26,
23 neon = 28,
24 neon_without_aes = 29,
25 rvv = 37,
26 lasx = 40,
27 lsx = 41,
28 ppc10 = 47,
29 ppc9 = 48,
30 ppc8 = 49,
31 z15 = 50,
32 z14 = 51,
33 wasm_emu256 = 58,
34 wasm = 59,
35 emu128 = 61,
36 scalar = 62,
37
38 pub fn bit(self: Target) Mask {
39 return @as(Mask, 1) << @backingInt(self);
40 }
41
42 pub fn name(self: Target) []const u8 {
43 return switch (self) {
44 .avx10_2 => "AVX10_2",
45 .avx3_spr => "AVX3_SPR",
46 .avx3_zen4 => "AVX3_ZEN4",
47 .avx3_dl => "AVX3_DL",
48 .avx3 => "AVX3",
49 .avx2 => "AVX2",
50 .sse4 => "SSE4",
51 .ssse3 => "SSSE3",
52 .sse2 => "SSE2",
53 .sve2_128 => "SVE2_128",
54 .sve_256 => "SVE_256",
55 .sve2 => "SVE2",
56 .sve => "SVE",
57 .neon_bf16 => "NEON_BF16",
58 .neon => "NEON",
59 .neon_without_aes => "NEON_WITHOUT_AES",
60 .rvv => "RVV",
61 .lasx => "LASX",
62 .lsx => "LSX",
63 .ppc10 => "PPC10",
64 .ppc9 => "PPC9",
65 .ppc8 => "PPC8",
66 .z15 => "Z15",
67 .z14 => "Z14",
68 .wasm_emu256 => "WASM_EMU256",
69 .wasm => "WASM",
70 .emu128 => "EMU128",
71 .scalar => "SCALAR",
72 };
73 }
74 };
75
76 pub const catalog = [_]Target{
77 .avx10_2,
78 .avx3_spr,
79 .avx3_zen4,
80 .avx3_dl,
81 .avx3,
82 .avx2,
83 .sse4,
84 .ssse3,
85 .sse2,
86 .sve2_128,
87 .sve_256,
88 .sve2,
89 .sve,
90 .neon_bf16,
91 .neon,
92 .neon_without_aes,
93 .rvv,
94 .lasx,
95 .lsx,
96 .ppc10,
97 .ppc9,
98 .ppc8,
99 .z15,
100 .z14,
101 .wasm_emu256,
102 .wasm,
103 .emu128,
104 .scalar,
105 };
106
107 pub const x86_targets = maskOf(&.{
108 Target.avx10_2,
109 .avx3_spr,
110 .avx3_zen4,
111 .avx3_dl,
112 .avx3,
113 .avx2,
114 .sse4,
115 .ssse3,
116 .sse2,
117 });
118 pub const arm_targets = maskOf(&.{
119 Target.sve2_128,
120 .sve_256,
121 .sve2,
122 .sve,
123 .neon_bf16,
124 .neon,
125 .neon_without_aes,
126 });
127 pub const riscv_targets = Target.rvv.bit();
128 pub const loongarch_targets = Target.lasx.bit() | Target.lsx.bit();
129 pub const power_targets = maskOf(&.{ Target.ppc10, .ppc9, .ppc8 });
130 pub const s390_targets = maskOf(&.{ Target.z15, .z14 });
131 pub const wasm_targets = Target.wasm_emu256.bit() | Target.wasm.bit();
132 pub const fallback_targets = Target.emu128.bit() | Target.scalar.bit();
133 pub const all_targets = maskOf(&catalog);
134
135 pub const Capabilities = struct {
136 integer64: bool,
137 float16: bool,
138 float64: bool,
139 };
140
141 pub const X86Leaf = packed struct {
142 eax: u32,
143 ebx: u32,
144 ecx: u32,
145 edx: u32,
146 };
147
148 pub const X86Snapshot = struct {
149 leaf0: X86Leaf,
150 leaf1: X86Leaf,
151 leaf7_0: X86Leaf,
152 leaf7_1: X86Leaf,
153 leaf24: X86Leaf,
154 extended0: X86Leaf,
155 extended1: X86Leaf,
156 xcr0: u32,
157 avx3_os_support: ?bool = null,
158 };
159
160 const AtomicMask64 = struct {
161 value: std.atomic.Value(Mask),
162
163 fn init(value: Mask) @This() {
164 return .{ .value = std.atomic.Value(Mask).init(value) };
165 }
166
167 fn load(self: *@This()) Mask {
168 return self.value.load(.acquire);
169 }
170
171 fn store(self: *@This(), value: Mask) void {
172 self.value.store(value, .release);
173 }
174 };
175
176 const AtomicMask32 = struct {
177 guard: std.atomic.Value(u32),
178 low: u32,
179 high: u32,
180
181 fn init(value: Mask) @This() {
182 return .{
183 .guard = std.atomic.Value(u32).init(0),
184 .low = @truncate(value),
185 .high = @truncate(value >> 32),
186 };
187 }
188
189 fn load(self: *@This()) Mask {
190 self.acquire();
191 defer self.release();
192 return @as(Mask, self.high) << 32 | self.low;
193 }
194
195 fn store(self: *@This(), value: Mask) void {
196 self.acquire();
197 defer self.release();
198 self.low = @truncate(value);
199 self.high = @truncate(value >> 32);
200 }
201
202 fn acquire(self: *@This()) void {
203 while (self.guard.cmpxchgWeak(0, 1, .acquire, .monotonic) != null) {
204 std.atomic.spinLoopHint();
205 }
206 }
207
208 fn release(self: *@This()) void {
209 self.guard.store(0, .release);
210 }
211 };
212
213 const AtomicMask = if (@bitSizeOf(usize) >= 64) AtomicMask64 else AtomicMask32;
214
215 pub const ChosenTarget = struct {
216 supported: AtomicMask = AtomicMask.init(0),
217
218 pub fn update(self: *ChosenTarget, targets: Mask) void {
219 std.debug.assert(targets != 0);
220 self.supported.store(targets);
221 }
222
223 pub fn deinit(self: *ChosenTarget) void {
224 self.supported.store(0);
225 }
226
227 pub fn isInitialized(self: *ChosenTarget) bool {
228 return self.supported.load() != 0;
229 }
230
231 pub fn load(self: *ChosenTarget) Mask {
232 return self.supported.load();
233 }
234 };
235
236 pub const Runtime = struct {
237 static: Target,
238 dispatch_fallback: Target,
239 disabled: AtomicMask = AtomicMask.init(0),
240 mocked: AtomicMask = AtomicMask.init(0),
241 chosen: ChosenTarget = .{},
242
243 pub fn init(static: Target) Runtime {
244 return .{ .static = static, .dispatch_fallback = .emu128 };
245 }
246
247 pub fn disableTargets(self: *Runtime, disabled: Mask) void {
248 self.disabled.store(disabled);
249 self.chosen.deinit();
250 }
251
252 pub fn setSupportedTargetsForTest(self: *Runtime, targets: Mask) void {
253 self.mocked.store(targets);
254 self.chosen.deinit();
255 }
256
257 pub fn supportedTargets(self: *Runtime, detected: Mask) Mask {
258 var targets = self.mocked.load();
259 if (targets == 0) targets = detected;
260 targets &= ~self.disabled.load();
261 return if (targets == 0) self.static.bit() else targets;
262 }
263
264 pub fn dispatchedTarget(
265 self: *Runtime,
266 detected: Mask,
267 generated: Mask,
268 ) Target {
269 if (self.cachedDispatchedTarget(generated)) |target| return target;
270 const supported = self.supportedTargets(detected);
271 self.chosen.update(supported);
272 return self.selectSupported(supported, generated);
273 }
274
275 fn cachedDispatchedTarget(self: *Runtime, generated: Mask) ?Target {
276 const supported = self.chosen.load();
277 return if (supported == 0) null else self.selectSupported(supported, generated);
278 }
279
280 fn selectSupported(self: *Runtime, supported: Mask, generated: Mask) Target {
281 return bestTarget(supported & generated) orelse self.dispatch_fallback;
282 }
283 };
284
285 pub fn Function(comptime Fn: type) type {
286 switch (@typeInfo(Fn)) {
287 .pointer => |pointer| if (@typeInfo(pointer.child) != .@"fn") {
288 @compileError("target functions require a function pointer type");
289 },
290 else => @compileError("target functions require a function pointer type"),
291 }
292 return struct {
293 target: Target,
294 implementation: Fn,
295 };
296 }
297
298 pub fn selectFunction(
299 runtime: *Runtime,
300 detected: Mask,
301 entries: anytype,
302 fallback: anytype,
303 ) @TypeOf(fallback) {
304 const Fn = @TypeOf(fallback);
305 const Entry = Function(Fn);
306 const slice: []const Entry = entries;
307 std.debug.assert(slice.len <= target_count);
308 var generated: Mask = 0;
309 for (slice) |entry| generated |= entry.target.bit();
310 const selected = runtime.dispatchedTarget(detected, generated);
311 for (slice) |entry| if (entry.target == selected) return entry.implementation;
312 return fallback;
313 }
314
315 pub fn maskOf(targets: []const Target) Mask {
316 var mask: Mask = 0;
317 for (targets) |target| mask |= target.bit();
318 return mask;
319 }
320
321 pub fn targetFromBit(bit: Mask) ?Target {
322 if (@popCount(bit) != 1) return null;
323 for (catalog) |target| if (target.bit() == bit) return target;
324 return null;
325 }
326
327 pub fn bestTarget(mask: Mask) ?Target {
328 if (mask == 0) return null;
329 return targetFromBit(mask & (~mask +% 1));
330 }
331
332 pub fn targetName(bit: Mask) []const u8 {
333 return if (targetFromBit(bit)) |target| target.name() else "Unknown";
334 }
335
336 pub fn architectureTargets(arch: std.Target.Cpu.Arch) Mask {
337 return switch (arch) {
338 .x86, .x86_64 => x86_targets,
339 .arm, .armeb, .thumb, .thumbeb, .aarch64, .aarch64_be => arm_targets,
340 .riscv32, .riscv32be, .riscv64, .riscv64be => riscv_targets,
341 .loongarch32, .loongarch64 => loongarch_targets,
342 .powerpc, .powerpcle, .powerpc64, .powerpc64le => power_targets,
343 .s390x => s390_targets,
344 .wasm32, .wasm64 => wasm_targets,
345 else => 0,
346 };
347 }
348
349 pub fn attainableTargets(arch: std.Target.Cpu.Arch) Mask {
350 return architectureTargets(arch) | Target.emu128.bit();
351 }
352
353 pub const static_target: Target = compileStaticTarget();
354 pub const generated_targets: Mask = defaultGeneratedTargets(builtin.target.cpu.arch, static_target);
355 pub const per_target_targets: Mask = attainableTargets(builtin.target.cpu.arch);
356
357 pub fn defaultGeneratedTargets(arch: std.Target.Cpu.Arch, static: Target) Mask {
358 const limit = static.bit() | (static.bit() - 1);
359 return attainableTargets(arch) & limit;
360 }
361
362 pub fn capabilitiesFor(target: Target, arch: std.Target.Cpu.Arch) Capabilities {
363 const float16 = switch (target) {
364 .avx10_2, .avx3_spr, .sve2_128, .sve_256, .sve2, .sve, .neon_bf16 => true,
365 else => false,
366 };
367 const float64 = switch (target) {
368 .neon, .neon_bf16, .neon_without_aes => switch (arch) {
369 .aarch64, .aarch64_be => true,
370 else => false,
371 },
372 else => true,
373 };
374 return .{ .integer64 = true, .float16 = float16, .float64 = float64 };
375 }
376
377 pub fn vectorBytesFor(target: Target, scalable_bytes: usize) usize {
378 return switch (target) {
379 .avx10_2, .avx3_spr, .avx3_zen4, .avx3_dl, .avx3 => 64,
380 .avx2, .sve_256, .lasx, .wasm_emu256 => 32,
381 .sse4,
382 .ssse3,
383 .sse2,
384 .sve2_128,
385 .neon_bf16,
386 .neon,
387 .neon_without_aes,
388 .lsx,
389 .ppc10,
390 .ppc9,
391 .ppc8,
392 .z15,
393 .z14,
394 .wasm,
395 .emu128,
396 => 16,
397 .sve2, .sve, .rvv => blk: {
398 std.debug.assert(scalable_bytes >= 16);
399 std.debug.assert(scalable_bytes <= 65_536);
400 break :blk scalable_bytes;
401 },
402 .scalar => 1,
403 };
404 }
405
406 var global_runtime = Runtime.init(static_target);
407
408 pub fn disableTargets(disabled: Mask) void {
409 global_runtime.disableTargets(disabled);
410 }
411
412 pub fn setSupportedTargetsForTest(targets: Mask) void {
413 global_runtime.setSupportedTargetsForTest(targets);
414 }
415
416 pub fn supportedTargets() Mask {
417 return global_runtime.supportedTargets(detectTargets());
418 }
419
420 pub fn dispatchedTarget() Target {
421 return global_runtime.cachedDispatchedTarget(per_target_targets) orelse
422 global_runtime.dispatchedTarget(detectTargets(), per_target_targets);
423 }
424
425 pub fn vectorBytes() usize {
426 const target = dispatchedTarget();
427 return vectorBytesFor(target, scalableVectorBytes(target));
428 }
429
430 pub fn haveInteger64() bool {
431 return capabilitiesFor(dispatchedTarget(), builtin.target.cpu.arch).integer64;
432 }
433
434 pub fn haveFloat16() bool {
435 const selected = dispatchedTarget();
436 return capabilitiesFor(selected, builtin.target.cpu.arch).float16 or
437 compileOptionalFloat16(selected);
438 }
439
440 pub fn haveFloat64() bool {
441 return capabilitiesFor(dispatchedTarget(), builtin.target.cpu.arch).float64;
442 }
443
444 pub fn supportedAndGeneratedTargets(output: *[target_count]Target) []const Target {
445 var remaining = supportedTargets() & generated_targets;
446 var count: usize = 0;
447 while (remaining != 0) {
448 const target = bestTarget(remaining).?;
449 output[count] = target;
450 count += 1;
451 remaining &= remaining - 1;
452 }
453 return output[0..count];
454 }
455
456 pub fn detectTargets() Mask {
457 var targets = fallback_targets;
458 targets |= switch (builtin.target.cpu.arch) {
459 .x86, .x86_64 => detectX86(captureX86(), builtin.target.cpu.arch == .x86_64),
460 .arm, .armeb, .thumb, .thumbeb, .aarch64, .aarch64_be => detectArm(),
461 .riscv32, .riscv32be, .riscv64, .riscv64be => detectRiscV(),
462 .loongarch32, .loongarch64 => detectLoongArch(),
463 .powerpc, .powerpcle, .powerpc64, .powerpc64le => detectPower(),
464 .s390x => detectS390x(),
465 .wasm32, .wasm64 => compileArchitectureTargets(),
466 else => compileArchitectureTargets(),
467 };
468 return targets;
469 }
470
471 const X86Feature = enum(u5) {
472 sse,
473 sse2,
474 sse3,
475 ssse3,
476 sse41,
477 sse42,
478 clmul,
479 aes,
480 avx,
481 avx2,
482 f16c,
483 fma,
484 lzcnt,
485 bmi,
486 bmi2,
487 avx512f,
488 avx512vl,
489 avx512cd,
490 avx512dq,
491 avx512bw,
492 avx512fp16,
493 avx512bf16,
494 vnni,
495 vpclmulqdq,
496 vbmi,
497 vbmi2,
498 vaes,
499 popcntdq,
500 bitalg,
501 gfni,
502 avx10,
503 apx,
504 };
505
506 fn x86FeatureBit(feature: X86Feature) u64 {
507 return @as(u64, 1) << @backingInt(feature);
508 }
509
510 fn x86Has(flags: u64, group: u64) bool {
511 return flags & group == group;
512 }
513
514 const X86Groups = struct {
515 const sse2 = x86FeatureBit(.sse) | x86FeatureBit(.sse2);
516 const ssse3 = x86FeatureBit(.sse3) | x86FeatureBit(.ssse3) | sse2;
517 const sse4 = x86FeatureBit(.sse41) | x86FeatureBit(.sse42) |
518 x86FeatureBit(.clmul) | x86FeatureBit(.aes) | ssse3;
519 const avx2 = x86FeatureBit(.avx) | x86FeatureBit(.avx2) |
520 x86FeatureBit(.lzcnt) | x86FeatureBit(.bmi) | x86FeatureBit(.bmi2) |
521 x86FeatureBit(.fma) | x86FeatureBit(.f16c) | sse4;
522 const avx3 = x86FeatureBit(.avx512f) | x86FeatureBit(.avx512vl) |
523 x86FeatureBit(.avx512dq) | x86FeatureBit(.avx512bw) |
524 x86FeatureBit(.avx512cd) | avx2;
525 const avx3_dl = x86FeatureBit(.vnni) | x86FeatureBit(.vpclmulqdq) |
526 x86FeatureBit(.vbmi) | x86FeatureBit(.vbmi2) | x86FeatureBit(.vaes) |
527 x86FeatureBit(.popcntdq) | x86FeatureBit(.bitalg) |
528 x86FeatureBit(.gfni) | avx3;
529 const zen4 = x86FeatureBit(.avx512bf16) | avx3_dl;
530 const spr = x86FeatureBit(.avx512fp16) | zen4;
531 const avx10 = x86FeatureBit(.avx10) | x86FeatureBit(.apx) |
532 x86FeatureBit(.vpclmulqdq) | x86FeatureBit(.vaes) |
533 x86FeatureBit(.gfni) | avx2;
534 };
535
536 fn x86Flags(snapshot: X86Snapshot) u64 {
537 var flags: u64 = 0;
538 const one = snapshot.leaf1;
539 flags |= if (bitSet(one.edx, 25)) x86FeatureBit(.sse) else 0;
540 flags |= if (bitSet(one.edx, 26)) x86FeatureBit(.sse2) else 0;
541 flags |= if (bitSet(one.ecx, 0)) x86FeatureBit(.sse3) else 0;
542 flags |= if (bitSet(one.ecx, 1)) x86FeatureBit(.clmul) else 0;
543 flags |= if (bitSet(one.ecx, 9)) x86FeatureBit(.ssse3) else 0;
544 flags |= if (bitSet(one.ecx, 12)) x86FeatureBit(.fma) else 0;
545 flags |= if (bitSet(one.ecx, 19)) x86FeatureBit(.sse41) else 0;
546 flags |= if (bitSet(one.ecx, 20)) x86FeatureBit(.sse42) else 0;
547 flags |= if (bitSet(one.ecx, 25)) x86FeatureBit(.aes) else 0;
548 flags |= if (bitSet(one.ecx, 28)) x86FeatureBit(.avx) else 0;
549 flags |= if (bitSet(one.ecx, 29)) x86FeatureBit(.f16c) else 0;
550 flags |= if (bitSet(snapshot.extended1.ecx, 5)) x86FeatureBit(.lzcnt) else 0;
551
552 if (snapshot.leaf0.eax >= 7) {
553 const seven = snapshot.leaf7_0;
554 flags |= if (bitSet(seven.ebx, 3)) x86FeatureBit(.bmi) else 0;
555 flags |= if (bitSet(seven.ebx, 5)) x86FeatureBit(.avx2) else 0;
556 flags |= if (bitSet(seven.ebx, 8)) x86FeatureBit(.bmi2) else 0;
557 flags |= if (bitSet(seven.ebx, 16)) x86FeatureBit(.avx512f) else 0;
558 flags |= if (bitSet(seven.ebx, 17)) x86FeatureBit(.avx512dq) else 0;
559 flags |= if (bitSet(seven.ebx, 28)) x86FeatureBit(.avx512cd) else 0;
560 flags |= if (bitSet(seven.ebx, 30)) x86FeatureBit(.avx512bw) else 0;
561 flags |= if (bitSet(seven.ebx, 31)) x86FeatureBit(.avx512vl) else 0;
562 flags |= if (bitSet(seven.ecx, 1)) x86FeatureBit(.vbmi) else 0;
563 flags |= if (bitSet(seven.ecx, 6)) x86FeatureBit(.vbmi2) else 0;
564 flags |= if (bitSet(seven.ecx, 8)) x86FeatureBit(.gfni) else 0;
565 flags |= if (bitSet(seven.ecx, 9)) x86FeatureBit(.vaes) else 0;
566 flags |= if (bitSet(seven.ecx, 10)) x86FeatureBit(.vpclmulqdq) else 0;
567 flags |= if (bitSet(seven.ecx, 11)) x86FeatureBit(.vnni) else 0;
568 flags |= if (bitSet(seven.ecx, 12)) x86FeatureBit(.bitalg) else 0;
569 flags |= if (bitSet(seven.ecx, 14)) x86FeatureBit(.popcntdq) else 0;
570 flags |= if (bitSet(seven.edx, 23)) x86FeatureBit(.avx512fp16) else 0;
571 flags |= if (bitSet(snapshot.leaf7_1.eax, 5)) x86FeatureBit(.avx512bf16) else 0;
572 flags |= if (bitSet(snapshot.leaf7_1.edx, 19)) x86FeatureBit(.avx10) else 0;
573 flags |= if (bitSet(snapshot.leaf7_1.edx, 21)) x86FeatureBit(.apx) else 0;
574 }
575 return flags;
576 }
577
578 pub fn detectX86(snapshot: X86Snapshot, is_64bit: bool) Mask {
579 const flags = x86Flags(snapshot);
580 const one = snapshot.leaf1;
581 var targets: Mask = if (is_64bit) Target.sse2.bit() else 0;
582 if (x86Has(flags, X86Groups.spr)) targets |= Target.avx3_spr.bit();
583 if (x86Has(flags, X86Groups.avx3_dl)) targets |= Target.avx3_dl.bit();
584 if (x86Has(flags, X86Groups.avx3)) targets |= Target.avx3.bit();
585 if (x86Has(flags, X86Groups.avx2)) targets |= Target.avx2.bit();
586 if (x86Has(flags, X86Groups.sse4)) targets |= Target.sse4.bit();
587 if (x86Has(flags, X86Groups.ssse3)) targets |= Target.ssse3.bit();
588 if (!is_64bit and x86Has(flags, X86Groups.sse2)) targets |= Target.sse2.bit();
589
590 if (x86Has(flags, X86Groups.avx10)) {
591 const version = snapshot.leaf24.ebx & 0xff;
592 if (version >= 1 and bitSet(snapshot.leaf24.ebx, 18)) {
593 targets |= Target.avx3_spr.bit() | Target.avx3_dl.bit() | Target.avx3.bit();
594 if (version >= 2) targets |= Target.avx10_2.bit();
595 }
596 }
597
598 const min_avx2 = Target.avx2.bit() | (Target.avx2.bit() - 1);
599 const min_avx3 = Target.avx3.bit() | (Target.avx3.bit() - 1);
600 if (!bitSet(one.ecx, 26) or !bitSet(one.ecx, 27)) {
601 targets &= ~min_avx2;
602 } else {
603 if (!bitSet(snapshot.xcr0, 1) or !bitSet(snapshot.xcr0, 2)) {
604 targets &= ~min_avx2;
605 }
606 const avx3_os_support = snapshot.avx3_os_support orelse
607 (bitSet(snapshot.xcr0, 5) and bitSet(snapshot.xcr0, 6) and
608 bitSet(snapshot.xcr0, 7));
609 if (!avx3_os_support) {
610 targets &= ~min_avx3;
611 }
612 }
613
614 if (targets & Target.avx3_dl.bit() != 0 and x86Has(flags, X86Groups.zen4) and
615 isAmd(snapshot.leaf0))
616 {
617 targets |= Target.avx3_zen4.bit();
618 }
619 return targets;
620 }
621
622 fn captureX86() X86Snapshot {
623 const leaf0 = cpuid(0, 0);
624 const leaf1 = if (leaf0.eax >= 1) cpuid(1, 0) else zeroX86Leaf();
625 const leaf7_0 = if (leaf0.eax >= 7) cpuid(7, 0) else zeroX86Leaf();
626 const leaf7_1 = if (leaf0.eax >= 7) cpuid(7, 1) else zeroX86Leaf();
627 const leaf24 = if (leaf0.eax >= 0x24) cpuid(0x24, 0) else zeroX86Leaf();
628 const extended0 = cpuid(0x8000_0000, 0);
629 const extended1 = if (extended0.eax >= 0x8000_0001)
630 cpuid(0x8000_0001, 0)
631 else
632 zeroX86Leaf();
633 const have_xcr0 = bitSet(leaf1.ecx, 26) and bitSet(leaf1.ecx, 27);
634 return .{
635 .leaf0 = leaf0,
636 .leaf1 = leaf1,
637 .leaf7_0 = leaf7_0,
638 .leaf7_1 = leaf7_1,
639 .leaf24 = leaf24,
640 .extended0 = extended0,
641 .extended1 = extended1,
642 .xcr0 = if (have_xcr0) readXcr0() else 0,
643 .avx3_os_support = if (builtin.target.os.tag.isDarwin())
644 appleAvx3OsSupport()
645 else
646 null,
647 };
648 }
649
650 fn cpuid(leaf: u32, subleaf: u32) X86Leaf {
651 var eax: u32 = undefined;
652 var ebx: u32 = undefined;
653 var ecx: u32 = undefined;
654 var edx: u32 = undefined;
655 asm volatile ("cpuid"
656 : [_] "={eax}" (eax),
657 [_] "={ebx}" (ebx),
658 [_] "={ecx}" (ecx),
659 [_] "={edx}" (edx),
660 : [_] "{eax}" (leaf),
661 [_] "{ecx}" (subleaf),
662 );
663 return .{ .eax = eax, .ebx = ebx, .ecx = ecx, .edx = edx };
664 }
665
666 fn readXcr0() u32 {
667 return asm volatile (
668 \\xor %%ecx, %%ecx
669 \\xgetbv
670 : [_] "={eax}" (-> u32),
671 :
672 : .{ .edx = true, .ecx = true });
673 }
674
675 fn zeroX86Leaf() X86Leaf {
676 return .{ .eax = 0, .ebx = 0, .ecx = 0, .edx = 0 };
677 }
678
679 fn isAmd(leaf0: X86Leaf) bool {
680 return leaf0.eax >= 1 and leaf0.ebx == 0x6874_7541 and
681 leaf0.ecx == 0x444d_4163 and leaf0.edx == 0x6974_6e65;
682 }
683
684 fn bitSet(value: u32, bit: u5) bool {
685 return value & (@as(u32, 1) << bit) != 0;
686 }
687
688 fn detectArm() Mask {
689 const arch = builtin.target.cpu.arch;
690 const is_64bit = arch == .aarch64 or arch == .aarch64_be;
691 if (builtin.target.os.tag.isDarwin() and is_64bit) return detectAppleArm();
692 const caps = readAuxCaps() orelse return compileArchitectureTargets();
693 if (is_64bit) return detectAarch64Caps(caps.hw, caps.hw2, detectedSveBytes(caps.hw));
694 return detectArm32Caps(caps.hw);
695 }
696
697 fn detectArm32Caps(hw: usize) Mask {
698 const required = (@as(usize, 1) << 12) | (@as(usize, 1) << 16);
699 return if (hw & required == required) Target.neon_without_aes.bit() else 0;
700 }
701
702 pub fn detectAarch64Caps(hw: usize, hw2: usize, sve_bytes: usize) Mask {
703 const aes = @as(usize, 1) << 3;
704 const asimdhp = @as(usize, 1) << 10;
705 const asimddp = @as(usize, 1) << 20;
706 const sve_cap = @as(usize, 1) << 22;
707 const sve2_cap = @as(usize, 1) << 1;
708 const sve_aes = @as(usize, 1) << 2;
709 const sve_i8mm = @as(usize, 1) << 9;
710 const sve_bf16 = @as(usize, 1) << 12;
711 const bf16 = @as(usize, 1) << 14;
712 var targets = Target.neon_without_aes.bit();
713 if (hw & aes != 0) {
714 targets |= Target.neon.bit();
715 if (hw & (asimdhp | asimddp) == asimdhp | asimddp and hw2 & bf16 != 0) {
716 targets |= Target.neon_bf16.bit();
717 }
718 }
719 if (hw & sve_cap != 0) targets |= Target.sve.bit();
720 if (hw2 & (sve2_cap | sve_aes) == sve2_cap | sve_aes) targets |= Target.sve2.bit();
721 if (targets & (Target.sve.bit() | Target.sve2.bit()) != 0) {
722 if (sve_bytes == 32) targets |= Target.sve_256.bit();
723 if (sve_bytes == 16 and targets & Target.sve2.bit() != 0 and
724 hw2 & (sve_i8mm | sve_bf16) == sve_i8mm | sve_bf16)
725 {
726 targets |= Target.sve2_128.bit();
727 }
728 }
729 return targets;
730 }
731
732 fn detectAppleArm() Mask {
733 var targets = Target.neon_without_aes.bit();
734 if (!appleFeature("hw.optional.arm.FEAT_AES")) return targets;
735 targets |= Target.neon.bit();
736 if ((appleFeature("hw.optional.AdvSIMD_HPFPCvt") or
737 appleFeature("hw.optional.arm.AdvSIMD_HPFPCvt")) and
738 appleFeature("hw.optional.arm.FEAT_DotProd") and
739 appleFeature("hw.optional.arm.FEAT_BF16") and
740 appleFeature("hw.optional.arm.FEAT_I8MM"))
741 {
742 targets |= Target.neon_bf16.bit();
743 }
744 return targets;
745 }
746
747 fn appleFeature(comptime name: [:0]const u8) bool {
748 var result: c_int = 0;
749 const byte_len = sys.process.systemControl(name, &result) catch return false;
750 return byte_len == @sizeOf(c_int) and result != 0;
751 }
752
753 fn appleAvx3OsSupport() bool {
754 var release_storage: [128]u8 = undefined;
755 const release = sys.process.kernelRelease(&release_storage) catch return false;
756 if (!darwinSupportsAvx3(release, true)) return false;
757 return appleFeature("hw.optional.avx512f");
758 }
759
760 fn darwinSupportsAvx3(release: []const u8, has_avx512: bool) bool {
761 var index: usize = 0;
762 const major = parseDecimal(release, &index) orelse return false;
763 if (index >= release.len or release[index] != '.') return false;
764 index += 1;
765 const minor = parseDecimal(release, &index) orelse return false;
766 return (major > 21 or (major == 21 and minor >= 3)) and has_avx512;
767 }
768
769 fn parseDecimal(text: []const u8, index: *usize) ?u32 {
770 const begin = index.*;
771 var value: u64 = 0;
772 while (index.* < text.len and text[index.*] >= '0' and text[index.*] <= '9') {
773 const digit: u64 = text[index.*] - '0';
774 if (value > (std.math.maxInt(u32) - digit) / 10) return null;
775 value = value * 10 + digit;
776 index.* += 1;
777 }
778 return if (index.* == begin) null else @intCast(value);
779 }
780
781 fn detectedSveBytes(hw: usize) usize {
782 const sve_cap = @as(usize, 1) << 22;
783 if (hw & sve_cap == 0) return 0;
784 return sveVectorBytes();
785 }
786
787 fn sveVectorBytes() usize {
788 return asm volatile (
789 \\.arch_extension sve
790 \\cntb %[bytes]
791 : [bytes] "=r" (-> usize),
792 );
793 }
794
795 fn detectRiscV() Mask {
796 if (builtin.target.os.tag != .linux) return compileArchitectureTargets();
797 const hw = sys.process.auxiliaryValue(.hardware_capabilities) catch 0;
798 const vector = @as(usize, 1) << ('V' - 'A');
799 if (hw & vector == 0) return 0;
800 var vtype: isize = undefined;
801 const bytes = asm volatile (
802 \\.option push
803 \\.option arch, +v
804 \\vsetvli %[bytes], zero, e8, m1, ta, ma
805 \\csrr %[vtype], vtype
806 \\.option pop
807 : [bytes] "=r" (-> usize),
808 [vtype] "=r" (vtype),
809 );
810 return detectRiscVCaps(hw, vtype, bytes);
811 }
812
813 fn detectRiscVCaps(hw: usize, vtype: isize, bytes: usize) Mask {
814 const vector = @as(usize, 1) << ('V' - 'A');
815 return if (hw & vector != 0 and vtype >= 0 and bytes >= 16)
816 Target.rvv.bit()
817 else
818 0;
819 }
820
821 fn detectLoongArch() Mask {
822 if (builtin.target.os.tag != .linux) return compileArchitectureTargets();
823 const hw = sys.process.auxiliaryValue(.hardware_capabilities) catch 0;
824 return detectLoongArchCaps(hw);
825 }
826
827 fn detectLoongArchCaps(hw: usize) Mask {
828 var targets: Mask = 0;
829 if (hw & (@as(usize, 1) << 4) != 0) targets |= Target.lsx.bit();
830 if (hw & (@as(usize, 1) << 5) != 0) targets |= Target.lasx.bit();
831 return targets;
832 }
833
834 fn detectPower() Mask {
835 const caps = readAuxCaps() orelse return compileArchitectureTargets();
836 return detectPowerCaps(caps.hw, caps.hw2);
837 }
838
839 fn detectPowerCaps(hw: usize, hw2: usize) Mask {
840 const vsx = @as(usize, 0x1000_0000) | 0x80;
841 if (hw & vsx != vsx) return 0;
842 const ppc8 = @as(usize, 0x8000_0000) | 0x0200_0000;
843 const ppc9 = ppc8 | 0x0080_0000;
844 const ppc10 = ppc9 | 0x0004_0000;
845 var targets: Mask = 0;
846 if (hw2 & ppc8 == ppc8) targets |= Target.ppc8.bit();
847 if (hw2 & ppc9 == ppc9) targets |= Target.ppc9.bit();
848 if (hw2 & ppc10 == ppc10) targets |= Target.ppc10.bit();
849 return targets;
850 }
851
852 const AuxCaps = struct {
853 hw: usize,
854 hw2: usize,
855 };
856
857 fn readAuxCaps() ?AuxCaps {
858 return switch (builtin.target.os.tag) {
859 .linux => .{
860 .hw = sys.process.auxiliaryValue(.hardware_capabilities) catch 0,
861 .hw2 = sys.process.auxiliaryValue(.hardware_capabilities_2) catch 0,
862 },
863 .freebsd, .openbsd => .{
864 .hw = bsdAuxValue(25),
865 .hw2 = bsdAuxValue(26),
866 },
867 else => null,
868 };
869 }
870
871 fn bsdAuxValue(identifier: c_int) usize {
872 var value: usize = 0;
873 const rc = elf_aux_info(identifier, &value, @sizeOf(usize));
874 return if (rc == 0) value else 0;
875 }
876
877 extern "c" fn elf_aux_info(aux: c_int, buffer: ?*anyopaque, length: c_int) c_int;
878
879 fn detectS390x() Mask {
880 if (builtin.target.os.tag != .linux) return compileArchitectureTargets();
881 const hw = sys.process.auxiliaryValue(.hardware_capabilities) catch 0;
882 return detectS390Caps(hw);
883 }
884
885 fn detectS390Caps(hw: usize) Mask {
886 const z14 = @as(usize, 2048 | 8192);
887 const z15 = z14 | 32768;
888 var targets: Mask = 0;
889 if (hw & z14 == z14) targets |= Target.z14.bit();
890 if (hw & z15 == z15) targets |= Target.z15.bit();
891 return targets;
892 }
893
894 fn scalableVectorBytes(target: Target) usize {
895 return switch (target) {
896 .sve, .sve2 => switch (builtin.target.cpu.arch) {
897 .aarch64, .aarch64_be => sveVectorBytes(),
898 else => 16,
899 },
900 .rvv => switch (builtin.target.cpu.arch) {
901 .riscv32, .riscv32be, .riscv64, .riscv64be => rvvVectorBytes(),
902 else => 16,
903 },
904 else => 16,
905 };
906 }
907
908 fn rvvVectorBytes() usize {
909 var vtype: isize = undefined;
910 return asm volatile (
911 \\.option push
912 \\.option arch, +v
913 \\vsetvli %[bytes], zero, e8, m1, ta, ma
914 \\csrr %[vtype], vtype
915 \\.option pop
916 : [bytes] "=r" (-> usize),
917 [vtype] "=r" (vtype),
918 );
919 }
920
921 fn compileArchitectureTargets() Mask {
922 return switch (compileStaticTarget()) {
923 .emu128, .scalar => 0,
924 else => |target| target.bit(),
925 };
926 }
927
928 fn compileOptionalFloat16(target: Target) bool {
929 return switch (builtin.target.cpu.arch) {
930 .aarch64, .aarch64_be => switch (target) {
931 .neon, .neon_without_aes => std.Target.aarch64.featureSetHas(
932 builtin.target.cpu.features,
933 .fullfp16,
934 ),
935 else => false,
936 },
937 .arm, .armeb, .thumb, .thumbeb => switch (target) {
938 .neon, .neon_without_aes => std.Target.arm.featureSetHas(
939 builtin.target.cpu.features,
940 .fullfp16,
941 ),
942 else => false,
943 },
944 .riscv32, .riscv32be, .riscv64, .riscv64be => target == .rvv and
945 std.Target.riscv.featureSetHas(builtin.target.cpu.features, .zvfh),
946 else => false,
947 };
948 }
949
950 fn compileStaticTarget() Target {
951 return switch (builtin.target.cpu.arch) {
952 .x86, .x86_64 => compileX86Target(),
953 .aarch64, .aarch64_be => compileAarch64Target(),
954 .arm, .armeb, .thumb, .thumbeb => compileArmTarget(),
955 .riscv32, .riscv32be, .riscv64, .riscv64be => if (std.Target.riscv.featureSetHas(
956 builtin.target.cpu.features,
957 .v,
958 )) .rvv else .emu128,
959 .loongarch32, .loongarch64 => if (std.Target.loongarch.featureSetHas(
960 builtin.target.cpu.features,
961 .lasx,
962 )) .lasx else if (std.Target.loongarch.featureSetHas(
963 builtin.target.cpu.features,
964 .lsx,
965 )) .lsx else .emu128,
966 .powerpc, .powerpcle, .powerpc64, .powerpc64le => compilePowerTarget(),
967 .s390x => if (std.Target.s390x.featureSetHas(
968 builtin.target.cpu.features,
969 .vector_enhancements_2,
970 )) .z15 else if (std.Target.s390x.featureSetHas(
971 builtin.target.cpu.features,
972 .vector_enhancements_1,
973 )) .z14 else .emu128,
974 .wasm32, .wasm64 => if (std.Target.wasm.featureSetHas(
975 builtin.target.cpu.features,
976 .simd128,
977 )) .wasm else .emu128,
978 else => .emu128,
979 };
980 }
981
982 fn compileX86Target() Target {
983 const features = builtin.target.cpu.features;
984 const has = std.Target.x86.featureSetHas;
985 const sse4 = has(features, .sse4_1) and has(features, .sse4_2) and
986 has(features, .pclmul) and has(features, .aes);
987 const avx2 = sse4 and has(features, .avx2) and has(features, .bmi2) and
988 has(features, .fma) and has(features, .f16c);
989 const avx3 = avx2 and has(features, .avx512f) and has(features, .avx512vl) and
990 has(features, .avx512dq) and has(features, .avx512bw);
991 const avx3_dl = avx3 and has(features, .avx512vnni) and
992 has(features, .vpclmulqdq) and has(features, .avx512vbmi) and
993 has(features, .avx512vbmi2) and has(features, .vaes) and
994 has(features, .avx512vpopcntdq) and has(features, .avx512bitalg);
995 const avx3_spr = avx3_dl and has(features, .avx512fp16) and
996 has(features, .avx512bf16);
997 if (avx3_spr and has(features, .avx10_2)) return .avx10_2;
998 if (avx3_spr) return .avx3_spr;
999 if (avx3_dl) return .avx3_dl;
1000 if (avx3) return .avx3;
1001 if (avx2) return .avx2;
1002 if (sse4) return .sse4;
1003 if (has(features, .ssse3)) return .ssse3;
1004 if (has(features, .sse2)) return .sse2;
1005 return .emu128;
1006 }
1007
1008 fn compileAarch64Target() Target {
1009 const features = builtin.target.cpu.features;
1010 const has = std.Target.aarch64.featureSetHas;
1011 if (has(features, .sve2)) return .sve2;
1012 if (has(features, .sve)) return .sve;
1013 if (has(features, .neon)) {
1014 if (has(features, .aes) and has(features, .fullfp16) and
1015 has(features, .dotprod) and has(features, .bf16) and
1016 has(features, .i8mm)) return .neon_bf16;
1017 if (has(features, .aes)) return .neon;
1018 return .neon_without_aes;
1019 }
1020 return .emu128;
1021 }
1022
1023 fn compileArmTarget() Target {
1024 return if (std.Target.arm.featureSetHas(
1025 builtin.target.cpu.features,
1026 .neon,
1027 )) .neon_without_aes else .emu128;
1028 }
1029
1030 fn compilePowerTarget() Target {
1031 const features = builtin.target.cpu.features;
1032 const has = std.Target.powerpc.featureSetHas;
1033 if (has(features, .power10_vector)) return .ppc10;
1034 if (has(features, .power9_vector)) return .ppc9;
1035 if (has(features, .power8_vector) and has(features, .altivec) and
1036 has(features, .vsx) and has(features, .crypto)) return .ppc8;
1037 return .emu128;
1038 }
1039
1040 test "pinned Highway target bits names and ordering remain exact" {
1041 try std.testing.expectEqual(target_count, catalog.len);
1042 var prior: u6 = 0;
1043 var combined: Mask = 0;
1044 for (catalog, 0..) |target, index| {
1045 const bit_index: u6 = @backingInt(target);
1046 if (index != 0) try std.testing.expect(bit_index > prior);
1047 prior = bit_index;
1048 try std.testing.expectEqual(target, targetFromBit(target.bit()).?);
1049 try std.testing.expectEqualStrings(target.name(), targetName(target.bit()));
1050 combined |= target.bit();
1051 }
1052 try std.testing.expectEqual(all_targets, combined);
1053 try std.testing.expectEqualStrings("Unknown", targetName(0));
1054 const multiple = Target.avx2.bit() | Target.sse4.bit();
1055 try std.testing.expectEqualStrings("Unknown", targetName(multiple));
1056 }
1057
1058 test "architecture target masks remain isolated and retain EMU128" {
1059 try std.testing.expectEqual(x86_targets, architectureTargets(.x86_64));
1060 try std.testing.expectEqual(arm_targets, architectureTargets(.aarch64));
1061 try std.testing.expectEqual(riscv_targets, architectureTargets(.riscv64));
1062 try std.testing.expectEqual(loongarch_targets, architectureTargets(.loongarch64));
1063 try std.testing.expectEqual(power_targets, architectureTargets(.powerpc64le));
1064 try std.testing.expectEqual(s390_targets, architectureTargets(.s390x));
1065 try std.testing.expectEqual(wasm_targets, architectureTargets(.wasm32));
1066 try std.testing.expectEqual(@as(Mask, 0), power_targets & s390_targets);
1067 try std.testing.expectEqual(
1068 s390_targets | Target.emu128.bit(),
1069 attainableTargets(.s390x),
1070 );
1071 }
1072
1073 test "pinned native x86 snapshot matches Highway target detection" {
1074 const snapshot = X86Snapshot{
1075 .leaf0 = .{
1076 .eax = 0x10,
1077 .ebx = 0x6874_7541,
1078 .ecx = 0x444d_4163,
1079 .edx = 0x6974_6e65,
1080 },
1081 .leaf1 = .{
1082 .eax = 0x00a6_0f12,
1083 .ebx = 0x0c10_0800,
1084 .ecx = 0x7ef8_320b,
1085 .edx = 0x178b_fbff,
1086 },
1087 .leaf7_0 = .{
1088 .eax = 1,
1089 .ebx = 0xf1bf_97a9,
1090 .ecx = 0x0040_5fde,
1091 .edx = 0x1000_0010,
1092 },
1093 .leaf7_1 = .{ .eax = 0x20, .ebx = 0, .ecx = 0, .edx = 0 },
1094 .leaf24 = zeroX86Leaf(),
1095 .extended0 = .{
1096 .eax = 0x8000_0028,
1097 .ebx = 0x6874_7541,
1098 .ecx = 0x444d_4163,
1099 .edx = 0x6974_6e65,
1100 },
1101 .extended1 = .{ .eax = 0x00a6_0f12, .ebx = 0, .ecx = 0x75c2_37ff, .edx = 0x2fd3_fbff },
1102 .xcr0 = 0x2e7,
1103 };
1104 try std.testing.expectEqual(@as(Mask, 0x5bc0), detectX86(snapshot, true));
1105 var without_ymm = snapshot;
1106 without_ymm.xcr0 &= ~@as(u32, 1 << 2);
1107 try std.testing.expectEqual(
1108 maskOf(&.{ Target.sse4, .ssse3, .sse2 }),
1109 detectX86(without_ymm, true),
1110 );
1111 var without_zmm = snapshot;
1112 without_zmm.xcr0 &= ~@as(u32, 1 << 7);
1113 try std.testing.expectEqual(
1114 maskOf(&.{ Target.avx2, .sse4, .ssse3, .sse2 }),
1115 detectX86(without_zmm, true),
1116 );
1117 without_zmm.avx3_os_support = true;
1118 try std.testing.expectEqual(@as(Mask, 0x5bc0), detectX86(without_zmm, true));
1119 var without_amd = snapshot;
1120 without_amd.leaf0.ebx = 0;
1121 try std.testing.expectEqual(@as(Mask, 0x5b80), detectX86(without_amd, true));
1122 var spr = snapshot;
1123 spr.leaf7_0.edx |= @as(u32, 1) << 23;
1124 try std.testing.expect(detectX86(spr, true) & Target.avx3_spr.bit() != 0);
1125 var avx10 = spr;
1126 avx10.leaf0.eax = 0x24;
1127 avx10.leaf7_1.edx = (@as(u32, 1) << 19) | (@as(u32, 1) << 21);
1128 avx10.leaf24.ebx = 2 | (@as(u32, 1) << 18);
1129 try std.testing.expect(detectX86(avx10, true) & Target.avx10_2.bit() != 0);
1130 }
1131
1132 test "Darwin AVX3 support begins at kernel 21.3 and requires AVX512" {
1133 try std.testing.expect(!darwinSupportsAvx3("21.2.0", true));
1134 try std.testing.expect(darwinSupportsAvx3("21.3.0", true));
1135 try std.testing.expect(darwinSupportsAvx3("22.0.0", true));
1136 try std.testing.expect(!darwinSupportsAvx3("22.0.0", false));
1137 try std.testing.expect(!darwinSupportsAvx3("4294967296.0", true));
1138 try std.testing.expect(!darwinSupportsAvx3("unknown", true));
1139 }
1140
1141 test "runtime masks disable mock invalidate and retain the static fallback" {
1142 var runtime = Runtime.init(.avx2);
1143 const detected = maskOf(&.{ Target.avx3_dl, .avx3, .avx2, .sse4 });
1144 const generated = maskOf(&.{ Target.avx3, .avx2, .sse4 });
1145 try std.testing.expectEqual(detected, runtime.supportedTargets(detected));
1146 try std.testing.expectEqual(Target.avx3, runtime.dispatchedTarget(detected, generated));
1147 try std.testing.expect(runtime.chosen.isInitialized());
1148 runtime.disableTargets(Target.avx3_dl.bit() | Target.avx3.bit());
1149 try std.testing.expect(!runtime.chosen.isInitialized());
1150 try std.testing.expectEqual(Target.avx2, runtime.dispatchedTarget(detected, generated));
1151 runtime.setSupportedTargetsForTest(Target.sse4.bit());
1152 try std.testing.expectEqual(Target.sse4, runtime.dispatchedTarget(detected, generated));
1153 runtime.disableTargets(std.math.maxInt(Mask));
1154 try std.testing.expectEqual(Target.avx2.bit(), runtime.supportedTargets(detected));
1155 try std.testing.expectEqual(Target.avx2, runtime.dispatchedTarget(detected, generated));
1156 runtime.disableTargets(0);
1157 runtime.setSupportedTargetsForTest(Target.scalar.bit());
1158 try std.testing.expectEqual(Target.emu128, runtime.dispatchedTarget(detected, generated));
1159 }
1160
1161 test "runtime target table selects every pinned target bit" {
1162 for (catalog) |target| {
1163 var runtime = Runtime.init(.emu128);
1164 runtime.setSupportedTargetsForTest(target.bit());
1165 try std.testing.expectEqual(target, runtime.dispatchedTarget(0, target.bit()));
1166 }
1167 }
1168
1169 test "global Highway target controls expose generated lists and reset cleanly" {
1170 defer {
1171 disableTargets(0);
1172 setSupportedTargetsForTest(0);
1173 }
1174 disableTargets(0);
1175 setSupportedTargetsForTest(static_target.bit());
1176 try std.testing.expectEqual(static_target.bit(), supportedTargets());
1177 try std.testing.expectEqual(static_target, dispatchedTarget());
1178 setSupportedTargetsForTest(Target.scalar.bit());
1179 try std.testing.expectEqual(Target.scalar.bit(), supportedTargets());
1180 try std.testing.expectEqual(Target.emu128, dispatchedTarget());
1181 setSupportedTargetsForTest(generated_targets);
1182 var storage: [target_count]Target = undefined;
1183 const generated = supportedAndGeneratedTargets(&storage);
1184 var remaining = generated_targets;
1185 try std.testing.expectEqual(@as(usize, @popCount(remaining)), generated.len);
1186 for (generated) |actual| {
1187 const expected = bestTarget(remaining).?;
1188 try std.testing.expectEqual(expected, actual);
1189 remaining &= remaining - 1;
1190 }
1191 setSupportedTargetsForTest(0);
1192 disableTargets(std.math.maxInt(Mask));
1193 try std.testing.expectEqual(static_target.bit(), supportedTargets());
1194 try std.testing.expectEqual(static_target, dispatchedTarget());
1195 }
1196
1197 fn dispatchSse(_: u32) u32 {
1198 return 4;
1199 }
1200
1201 fn dispatchAvx2(_: u32) u32 {
1202 return 2;
1203 }
1204
1205 fn dispatchFallback(_: u32) u32 {
1206 return 0;
1207 }
1208
1209 test "caller supplied target functions select the best shared implementation" {
1210 const Fn = *const fn (u32) u32;
1211 const Entry = Function(Fn);
1212 const entries = [_]Entry{
1213 .{ .target = .sse4, .implementation = dispatchSse },
1214 .{ .target = .avx2, .implementation = dispatchAvx2 },
1215 };
1216 var runtime = Runtime.init(.emu128);
1217 const detected = maskOf(&.{ Target.avx3, .avx2, .sse4 });
1218 const selected = selectFunction(&runtime, detected, &entries, @as(Fn, dispatchFallback));
1219 try std.testing.expectEqual(@as(u32, 2), selected(9));
1220 runtime.disableTargets(Target.avx2.bit());
1221 const next = selectFunction(&runtime, detected, &entries, @as(Fn, dispatchFallback));
1222 try std.testing.expectEqual(@as(u32, 4), next(9));
1223 runtime.disableTargets(Target.avx2.bit() | Target.sse4.bit());
1224 const fallback = selectFunction(&runtime, detected, &entries, @as(Fn, dispatchFallback));
1225 try std.testing.expectEqual(@as(u32, 0), fallback(9));
1226 }
1227
1228 test "selected target geometry and capabilities match pinned per-target values" {
1229 try std.testing.expectEqual(@as(usize, 64), vectorBytesFor(.avx3_zen4, 16));
1230 try std.testing.expectEqual(@as(usize, 32), vectorBytesFor(.avx2, 16));
1231 try std.testing.expectEqual(@as(usize, 16), vectorBytesFor(.emu128, 16));
1232 try std.testing.expectEqual(@as(usize, 1), vectorBytesFor(.scalar, 16));
1233 try std.testing.expectEqual(@as(usize, 48), vectorBytesFor(.sve2, 48));
1234 try std.testing.expect(!capabilitiesFor(.avx3_zen4, .x86_64).float16);
1235 try std.testing.expect(capabilitiesFor(.avx3_spr, .x86_64).float16);
1236 try std.testing.expect(capabilitiesFor(.neon, .aarch64).float64);
1237 try std.testing.expect(!capabilitiesFor(.neon, .arm).float64);
1238 }
1239
1240 test "AArch64 capability groups retain specialized SVE and NEON targets" {
1241 const hw = (@as(usize, 1) << 3) | (@as(usize, 1) << 10) |
1242 (@as(usize, 1) << 20) | (@as(usize, 1) << 22);
1243 const hw2 = (@as(usize, 1) << 1) | (@as(usize, 1) << 2) |
1244 (@as(usize, 1) << 9) | (@as(usize, 1) << 12) | (@as(usize, 1) << 14);
1245 try std.testing.expectEqual(
1246 maskOf(&.{
1247 Target.sve2_128,
1248 .sve2,
1249 .sve,
1250 .neon_bf16,
1251 .neon,
1252 .neon_without_aes,
1253 }),
1254 detectAarch64Caps(hw, hw2, 16),
1255 );
1256 try std.testing.expectEqual(
1257 maskOf(&.{
1258 Target.sve_256,
1259 .sve2,
1260 .sve,
1261 .neon_bf16,
1262 .neon,
1263 .neon_without_aes,
1264 }),
1265 detectAarch64Caps(hw, hw2, 32),
1266 );
1267 }
1268
1269 test "auxiliary vector capability groups retain every non-x86 runtime target" {
1270 const arm_required = (@as(usize, 1) << 12) | (@as(usize, 1) << 16);
1271 try std.testing.expectEqual(Target.neon_without_aes.bit(), detectArm32Caps(arm_required));
1272 try std.testing.expectEqual(
1273 @as(Mask, 0),
1274 detectArm32Caps(arm_required & ~(@as(usize, 1) << 12)),
1275 );
1276
1277 const riscv_vector = @as(usize, 1) << ('V' - 'A');
1278 try std.testing.expectEqual(Target.rvv.bit(), detectRiscVCaps(riscv_vector, 0, 16));
1279 try std.testing.expectEqual(@as(Mask, 0), detectRiscVCaps(riscv_vector, -1, 16));
1280 try std.testing.expectEqual(@as(Mask, 0), detectRiscVCaps(riscv_vector, 0, 15));
1281
1282 const loongarch = (@as(usize, 1) << 4) | (@as(usize, 1) << 5);
1283 try std.testing.expectEqual(loongarch_targets, detectLoongArchCaps(loongarch));
1284
1285 const power_hw = @as(usize, 0x1000_0000) | 0x80;
1286 const power10_hw2 = @as(usize, 0x8000_0000) | 0x0200_0000 |
1287 0x0080_0000 | 0x0004_0000;
1288 try std.testing.expectEqual(power_targets, detectPowerCaps(power_hw, power10_hw2));
1289 try std.testing.expectEqual(@as(Mask, 0), detectPowerCaps(power_hw, 0x8000_0000));
1290
1291 try std.testing.expectEqual(s390_targets, detectS390Caps(2048 | 8192 | 32768));
1292 try std.testing.expectEqual(Target.z14.bit(), detectS390Caps(2048 | 8192));
1293 }