lib/simd/src/lanes.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2
3 pub fn lowerHalf(comptime D: type, value: anytype) D.Vector {
4 if (D.lane_count == 1) {
5 if (@TypeOf(value) != D.Vector) @compileError("scalar lowerHalf requires the same descriptor");
6 return value;
7 }
8 if (@TypeOf(value) != D.twice().Vector) {
9 @compileError("lowerHalf source must have twice the destination lanes");
10 }
11 var result: D.Vector = undefined;
12 inline for (0..D.lane_count) |index| result[index] = value[index];
13 return result;
14 }
15
16 pub fn upperHalf(comptime D: type, value: D.twice().Vector) D.Vector {
17 var result: D.Vector = undefined;
18 inline for (0..D.lane_count) |index| result[index] = value[D.lane_count + index];
19 return result;
20 }
21
22 pub fn zeroExtendVector(comptime D: type, value: D.half().Vector) D.Vector {
23 validate(D);
24 var result: D.Vector = @splat(0);
25 inline for (0..D.lane_count / 2) |index| result[index] = value[index];
26 return result;
27 }
28
29 pub fn combine(
30 comptime D: type,
31 high: D.half().Vector,
32 low: D.half().Vector,
33 ) D.Vector {
34 validate(D);
35 var result: D.Vector = undefined;
36 inline for (0..D.lane_count / 2) |index| {
37 result[index] = low[index];
38 result[D.lane_count / 2 + index] = high[index];
39 }
40 return result;
41 }
42
43 pub fn concatLowerLower(comptime D: type, high: D.Vector, low: D.Vector) D.Vector {
44 const H = D.half();
45 return combine(D, lowerHalf(H, high), lowerHalf(H, low));
46 }
47
48 pub fn concatUpperUpper(comptime D: type, high: D.Vector, low: D.Vector) D.Vector {
49 const H = D.half();
50 return combine(D, upperHalf(H, high), upperHalf(H, low));
51 }
52
53 pub fn concatLowerUpper(comptime D: type, high: D.Vector, low: D.Vector) D.Vector {
54 const H = D.half();
55 return combine(D, lowerHalf(H, high), upperHalf(H, low));
56 }
57
58 pub fn concatUpperLower(comptime D: type, high: D.Vector, low: D.Vector) D.Vector {
59 const H = D.half();
60 return combine(D, upperHalf(H, high), lowerHalf(H, low));
61 }
62
63 pub fn concatOdd(comptime D: type, high: D.Vector, low: D.Vector) D.Vector {
64 validate(D);
65 var result: D.Vector = undefined;
66 inline for (0..D.lane_count / 2) |index| {
67 result[index] = low[index * 2 + 1];
68 result[D.lane_count / 2 + index] = high[index * 2 + 1];
69 }
70 return result;
71 }
72
73 pub fn concatEven(comptime D: type, high: D.Vector, low: D.Vector) D.Vector {
74 validate(D);
75 var result: D.Vector = undefined;
76 inline for (0..D.lane_count / 2) |index| {
77 result[index] = low[index * 2];
78 result[D.lane_count / 2 + index] = high[index * 2];
79 }
80 return result;
81 }
82
83 pub fn interleaveWholeLower(comptime D: type, a: D.Vector, b: D.Vector) D.Vector {
84 validate(D);
85 var result: D.Vector = undefined;
86 inline for (0..D.lane_count / 2) |index| {
87 result[index * 2] = a[index];
88 result[index * 2 + 1] = b[index];
89 }
90 return result;
91 }
92
93 pub fn interleaveWholeUpper(comptime D: type, a: D.Vector, b: D.Vector) D.Vector {
94 validate(D);
95 var result: D.Vector = undefined;
96 inline for (0..D.lane_count / 2) |index| {
97 result[index * 2] = a[D.lane_count / 2 + index];
98 result[index * 2 + 1] = b[D.lane_count / 2 + index];
99 }
100 return result;
101 }
102
103 pub fn lowerHalfOfMask(comptime D: type, mask: D.twice().Mask) D.Mask {
104 var result: D.Mask = undefined;
105 inline for (0..D.lane_count) |index| result[index] = mask[index];
106 return result;
107 }
108
109 pub fn upperHalfOfMask(comptime D: type, mask: D.twice().Mask) D.Mask {
110 var result: D.Mask = undefined;
111 inline for (0..D.lane_count) |index| result[index] = mask[D.lane_count + index];
112 return result;
113 }
114
115 pub fn combineMasks(
116 comptime D: type,
117 high: D.half().Mask,
118 low: D.half().Mask,
119 ) D.Mask {
120 validate(D);
121 var result: D.Mask = undefined;
122 inline for (0..D.lane_count / 2) |index| {
123 result[index] = low[index];
124 result[D.lane_count / 2 + index] = high[index];
125 }
126 return result;
127 }
128
129 fn validate(comptime D: type) void {
130 if (D.lane_count < 2) @compileError("lane combination requires at least two lanes");
131 }
132
133 fn verifyLaneType(comptime T: type) !void {
134 const simd = @import("root.zig");
135 const D = simd.FixedTag(T, 8);
136 const H = D.half();
137 const value: D.Vector = @splat(0);
138 const low = lowerHalf(H, value);
139 const high = upperHalf(H, value);
140 try std.testing.expect(@reduce(.And, combine(D, high, low) == value));
141 try std.testing.expect(@reduce(.And, concatOdd(D, value, value) == value));
142 try std.testing.expect(@reduce(.And, concatEven(D, value, value) == value));
143 try std.testing.expect(@reduce(.And, interleaveWholeLower(D, value, value) == value));
144 try std.testing.expect(@reduce(.And, interleaveWholeUpper(D, value, value) == value));
145 }
146
147 test "Highway lane geometry instantiates every lane type" {
148 inline for (.{ u8, i8, u16, i16, u32, i32, u64, i64, f16, f32, f64 }) |T| {
149 try verifyLaneType(T);
150 }
151 const D = @import("root.zig").FixedTag(u32, 1);
152 try std.testing.expect(@reduce(.And, lowerHalf(D, @as(D.Vector, .{9})) ==
153 @as(D.Vector, .{9})));
154 }
155
156 test "Highway vector halves combine and zero extend in lane order" {
157 const simd = @import("root.zig");
158 const D = simd.FixedTag(u32, 8);
159 const H = D.half();
160 const value: D.Vector = .{ 0, 1, 2, 3, 4, 5, 6, 7 };
161 const low = lowerHalf(H, value);
162 const high = upperHalf(H, value);
163 try std.testing.expect(@reduce(.And, low == @as(H.Vector, .{ 0, 1, 2, 3 })));
164 try std.testing.expect(@reduce(.And, high == @as(H.Vector, .{ 4, 5, 6, 7 })));
165 try std.testing.expect(@reduce(.And, combine(D, high, low) == value));
166 try std.testing.expect(@reduce(.And, zeroExtendVector(D, low) ==
167 @as(D.Vector, .{ 0, 1, 2, 3, 0, 0, 0, 0 })));
168 }
169
170 test "Highway whole-vector concatenations select exact quarters" {
171 const simd = @import("root.zig");
172 const D = simd.FixedTag(u16, 8);
173 const low: D.Vector = .{ 0, 1, 2, 3, 4, 5, 6, 7 };
174 const high: D.Vector = .{ 10, 11, 12, 13, 14, 15, 16, 17 };
175 try std.testing.expect(@reduce(.And, concatLowerLower(D, high, low) ==
176 @as(D.Vector, .{ 0, 1, 2, 3, 10, 11, 12, 13 })));
177 try std.testing.expect(@reduce(.And, concatUpperUpper(D, high, low) ==
178 @as(D.Vector, .{ 4, 5, 6, 7, 14, 15, 16, 17 })));
179 try std.testing.expect(@reduce(.And, concatLowerUpper(D, high, low) ==
180 @as(D.Vector, .{ 4, 5, 6, 7, 10, 11, 12, 13 })));
181 try std.testing.expect(@reduce(.And, concatUpperLower(D, high, low) ==
182 @as(D.Vector, .{ 0, 1, 2, 3, 14, 15, 16, 17 })));
183 try std.testing.expect(@reduce(.And, concatOdd(D, high, low) ==
184 @as(D.Vector, .{ 1, 3, 5, 7, 11, 13, 15, 17 })));
185 try std.testing.expect(@reduce(.And, concatEven(D, high, low) ==
186 @as(D.Vector, .{ 0, 2, 4, 6, 10, 12, 14, 16 })));
187 }
188
189 test "Highway whole interleave and mask halves retain ordering" {
190 const simd = @import("root.zig");
191 const D = simd.FixedTag(i32, 8);
192 const H = D.half();
193 const a: D.Vector = .{ 0, 1, 2, 3, 4, 5, 6, 7 };
194 const b: D.Vector = .{ 10, 11, 12, 13, 14, 15, 16, 17 };
195 try std.testing.expect(@reduce(.And, interleaveWholeLower(D, a, b) ==
196 @as(D.Vector, .{ 0, 10, 1, 11, 2, 12, 3, 13 })));
197 try std.testing.expect(@reduce(.And, interleaveWholeUpper(D, a, b) ==
198 @as(D.Vector, .{ 4, 14, 5, 15, 6, 16, 7, 17 })));
199 const mask: D.Mask = .{ true, false, true, false, false, true, false, true };
200 const low = lowerHalfOfMask(H, mask);
201 const high = upperHalfOfMask(H, mask);
202 try std.testing.expect(@reduce(.And, combineMasks(D, high, low) == mask));
203 }