lib/simd/src/cache.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

 1 const std = @import("std");
 2 const builtin = @import("builtin");
 3 const aligned = @import("aligned.zig");
 4 
 5 pub const stream_multiple: usize = 16;
 6 pub const words_per_line: usize = aligned.alignment / @sizeOf(u64);
 7 
 8 pub inline fn loadFence() void {
 9     switch (comptime builtin.target.cpu.arch) {
10         .x86, .x86_64 => asm volatile ("lfence" ::: .{ .memory = true }),
11         else => {},
12     }
13 }
14 
15 pub fn streamCacheLine(
16     source: *const [words_per_line]u64,
17     destination: *[words_per_line]u64,
18 ) void {
19     std.debug.assert(aligned.isAligned(source));
20     std.debug.assert(aligned.isAligned(destination));
21     std.debug.assert(@intFromPtr(source) != @intFromPtr(destination));
22     destination.* = source.*;
23 }
24 
25 pub inline fn flushStream() void {
26     switch (comptime builtin.target.cpu.arch) {
27         .x86, .x86_64 => asm volatile ("sfence" ::: .{ .memory = true }),
28         else => {},
29     }
30 }
31 
32 pub inline fn prefetch(pointer: anytype) void {
33     @prefetch(pointer, .{});
34 }
35 
36 pub inline fn flushCacheline(pointer: *const anyopaque) void {
37     switch (comptime builtin.target.cpu.arch) {
38         .x86_64 => asm volatile ("clflush (%%rax)"
39             :
40             : [address] "{rax}" (pointer),
41             : .{ .memory = true }),
42         .x86 => asm volatile ("clflush (%%eax)"
43             :
44             : [address] "{eax}" (pointer),
45             : .{ .memory = true }),
46         else => {},
47     }
48 }
49 
50 pub inline fn pause() void {
51     switch (comptime builtin.target.cpu.arch) {
52         .x86, .x86_64 => asm volatile ("pause"),
53         .aarch64, .aarch64_be, .arm, .armeb, .thumb, .thumbeb => asm volatile ("yield"),
54         .powerpc, .powerpcle, .powerpc64, .powerpc64le => asm volatile ("or 27, 27, 27"),
55         else => {},
56     }
57 }
58 
59 test "Highway streamed cache line copies exactly one aligned line" {
60     try std.testing.expectEqual(@as(usize, 128), aligned.alignment);
61     try std.testing.expectEqual(@as(usize, 16), stream_multiple);
62     var source: [words_per_line]u64 align(aligned.alignment) = undefined;
63     for (&source, 0..) |*value, index| {
64         value.* = 0x0102_0304_0506_0708 + @as(u64, @intCast(index));
65     }
66     var blocks: [3][words_per_line]u64 align(aligned.alignment) = @splat(@splat(0xa5a5_a5a5_a5a5_a5a5));
67     streamCacheLine(&source, &blocks[1]);
68     flushStream();
69     try std.testing.expectEqualSlices(u64, &source, &blocks[1]);
70     var digest: u64 = 0;
71     for (blocks[1]) |value| digest +%= value;
72     try std.testing.expectEqual(@as(u64, 0x1020_3040_5060_70f8), digest);
73     try std.testing.expectEqualSlices(
74         u64,
75         &@as([words_per_line]u64, @splat(0xa5a5_a5a5_a5a5_a5a5)),
76         &blocks[0],
77     );
78     try std.testing.expectEqualSlices(
79         u64,
80         &@as([words_per_line]u64, @splat(0xa5a5_a5a5_a5a5_a5a5)),
81         &blocks[2],
82     );
83 }
84 
85 test "Highway cache controls preserve accessible values" {
86     var value: u64 align(aligned.alignment) = 42;
87     loadFence();
88     prefetch(&value);
89     flushCacheline(&value);
90     pause();
91     try std.testing.expectEqual(@as(u64, 42), value);
92 }