lib/bench/src/stats/test.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const stats = @import("root.zig");
3
4 const Storage = stats.Storage;
5 const BootstrapConfidenceIntervals = stats.BootstrapConfidenceIntervals;
6 const SampleSetCursor = stats.SampleSetCursor;
7 const IndexedDeltaTotals = stats.IndexedDeltaTotals;
8 const IndexedMemoryTotals = stats.IndexedMemoryTotals;
9 const percentile = stats.percentile;
10 const computeSampleStats = stats.computeSampleStats;
11 const computeSampleStatsWithBootstrap = stats.computeSampleStatsWithBootstrap;
12 const sampleSetValueCapacity = stats.sampleSetValueCapacity;
13 const sampleSets = stats.sampleSets;
14 const nsPerUnit = stats.nsPerUnit;
15 const nsPerFloatUnit = stats.nsPerFloatUnit;
16
17 test "percentiles preserve the benchmark runner policy" {
18 try std.testing.expectEqual(@as(u64, 0), percentile(&.{}, 50));
19 try std.testing.expectEqual(@as(u64, 42), percentile(&.{42}, 99));
20 const data = [_]u64{ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 };
21 try std.testing.expectEqual(@as(u64, 60), percentile(&data, 50));
22 try std.testing.expectEqual(@as(u64, 80), percentile(&data, 75));
23 try std.testing.expectEqual(@as(u64, 100), percentile(&data, 99));
24 }
25
26 test "sample stats use benchmark-runner percentile policy" {
27 const samples = [_]u64{ 100, 10, 40, 30, 20, 90, 80, 70, 60, 50 };
28 var storage = try Storage.init(std.testing.allocator, .{ .samples = samples.len });
29 defer storage.deinit(std.testing.allocator);
30 storage.activate();
31 const result = try computeSampleStats(&storage, &samples);
32
33 try std.testing.expectEqual(@as(u64, 10), result.min_ns);
34 try std.testing.expectEqual(@as(u64, 100), result.max_ns);
35 try std.testing.expectEqual(@as(u64, 550), result.total_ns);
36 try std.testing.expectEqual(@as(u64, 60), result.median_ns);
37 try std.testing.expectEqual(@as(u64, 80), result.p75_ns);
38 try std.testing.expectEqual(@as(u64, 100), result.p95_ns);
39 try std.testing.expectEqual(@as(u64, 100), result.p99_ns);
40 try std.testing.expect(result.confidence_intervals != null);
41 }
42
43 test "statistics storage rejects every limit at max plus one" {
44 comptime {
45 @stardustClaim(
46 @import("alloc_phase").capacity.witness(@import("./root.zig").Storage, "bench_statistics_boundaries"),
47 null,
48 null,
49 null,
50 null,
51 null,
52 null,
53 );
54 }
55
56 var storage = try Storage.init(std.testing.allocator, .{
57 .samples = 3,
58 .bootstrap_iterations = 20,
59 });
60 defer storage.deinit(std.testing.allocator);
61 storage.activate();
62 try std.testing.expectError(error.EmptySampleSet, computeSampleStats(&storage, &.{}));
63 try std.testing.expectError(
64 error.SampleCapacityExceeded,
65 computeSampleStats(&storage, &.{ 1, 2, 3, 4 }),
66 );
67 try std.testing.expectError(
68 error.BootstrapIterationCapacityExceeded,
69 computeSampleStatsWithBootstrap(&storage, &.{ 1, 2, 3 }, .{ .iterations = 21 }),
70 );
71 try std.testing.expectError(
72 error.InvalidBootstrapConfidence,
73 computeSampleStatsWithBootstrap(
74 &storage,
75 &.{ 1, 2, 3 },
76 .{ .iterations = 1, .confidence_per_mille = 0 },
77 ),
78 );
79 try std.testing.expectError(
80 error.InvalidBootstrapConfidence,
81 computeSampleStatsWithBootstrap(
82 &storage,
83 &.{ 1, 2, 3 },
84 .{ .iterations = 1, .confidence_per_mille = 1001 },
85 ),
86 );
87 try std.testing.expect(!storage.status().in_use);
88 _ = try computeSampleStatsWithBootstrap(&storage, &.{ 1, 2, 3 }, .{ .iterations = 20 });
89 }
90
91 test "statistics storage preserves results and reusable state" {
92 comptime {
93 @stardustClaim(
94 @import("alloc_phase").capacity.witness(@import("./root.zig").Storage, "bench_statistics_reuse"),
95 null,
96 null,
97 null,
98 null,
99 null,
100 null,
101 );
102 }
103
104 var storage = try Storage.init(std.testing.allocator, .{
105 .samples = 7,
106 .bootstrap_iterations = 200,
107 });
108 defer storage.deinit(std.testing.allocator);
109 storage.activate();
110 const samples = [_]u64{ 10, 20, 30, 40, 50, 60, 70 };
111 const first = try computeSampleStatsWithBootstrap(
112 &storage,
113 &samples,
114 .{ .iterations = 200, .seed = 99 },
115 );
116 try std.testing.expectEqual(BootstrapConfidenceIntervals{
117 .confidence = 0.95,
118 .iterations = 200,
119 .seed = 99,
120 .mean_ns = .{ .low_ns = 25.714285714285715, .high_ns = 54.285714285714285 },
121 .median_ns = .{ .low_ns = 20, .high_ns = 60 },
122 .p75_ns = .{ .low_ns = 30, .high_ns = 70 },
123 .p95_ns = .{ .low_ns = 40, .high_ns = 70 },
124 .p99_ns = .{ .low_ns = 40, .high_ns = 70 },
125 }, first.confidence_intervals.?);
126 var reversed = samples;
127 std.mem.reverse(u64, &reversed);
128 const second = try computeSampleStatsWithBootstrap(
129 &storage,
130 &reversed,
131 .{ .iterations = 200, .seed = 99 },
132 );
133 try std.testing.expectEqual(first, second);
134 try std.testing.expect(!storage.status().in_use);
135
136 const regions = try storage.acquire(samples.len, .{ .iterations = 200 });
137 _ = regions;
138 try std.testing.expectError(
139 error.StatisticsStorageInUse,
140 storage.acquire(samples.len, .{ .iterations = 200 }),
141 );
142 storage.reset();
143 }
144
145 test "activated statistics storage performs no backing allocation" {
146 comptime {
147 @stardustClaim(
148 @import("alloc_phase").capacity.witness(@import("./root.zig").Storage, "bench_statistics_sealed_transitive_risk"),
149 null,
150 null,
151 null,
152 null,
153 null,
154 null,
155 );
156 }
157 comptime {
158 @stardustClaim(
159 @import("alloc_phase").capacity.witness(@import("./root.zig").Storage, "bench_statistics_sealed_foreign_risk"),
160 null,
161 null,
162 null,
163 null,
164 null,
165 null,
166 );
167 }
168
169 const samples = [_]u64{ 13, 2, 11, 5, 7, 3, 17, 19, 23, 29, 31, 37, 41 };
170 var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{});
171 var storage = try Storage.init(failing.allocator(), .{ .samples = samples.len });
172 defer storage.deinit(failing.allocator());
173 storage.activate();
174 const alloc_index = failing.alloc_index;
175 const result = try computeSampleStats(&storage, &samples);
176
177 try std.testing.expectEqual(@as(usize, 1), alloc_index);
178 try std.testing.expectEqual(alloc_index, failing.alloc_index);
179 try std.testing.expectEqual(@as(usize, 40_156), failing.allocated_bytes);
180 try std.testing.expectEqual(@as(u64, 2), result.min_ns);
181 try std.testing.expectEqual(@as(u64, 41), result.max_ns);
182 }
183
184 test "sample sets partition one exact caller region" {
185 comptime {
186 @stardustClaim(
187 @import("alloc_phase").capacity.witness(@import("./root.zig").Storage, "bench_statistics_sets"),
188 null,
189 null,
190 null,
191 null,
192 null,
193 null,
194 );
195 }
196
197 var values: [52]u64 = undefined;
198 var cursor = SampleSetCursor.init(&values);
199 const first = try cursor.take(1, 13);
200 const partitioned = try cursor.take(3, 13);
201 try std.testing.expectEqual(@as(usize, 52), try sampleSetValueCapacity(4, 13));
202 try std.testing.expectEqual(@intFromPtr(&values), @intFromPtr(first[0].ptr));
203 try std.testing.expectEqual(
204 @intFromPtr(&values[13]),
205 @intFromPtr(partitioned[0].ptr),
206 );
207 for (partitioned) |set| {
208 try std.testing.expectEqualSlices(u64, &(@as([13]u64, @splat(0))), set);
209 }
210 try std.testing.expect(cursor.complete());
211 try std.testing.expectError(error.SampleSetStorageTooSmall, cursor.take(1, 13));
212 try std.testing.expect(cursor.complete());
213 }
214
215 test "sample sets need one exact caller acquisition" {
216 var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{});
217 const value_count = try sampleSetValueCapacity(4, 13);
218 const values = try failing.allocator().alloc(u64, value_count);
219 defer failing.allocator().free(values);
220 var cursor = SampleSetCursor.init(values);
221 _ = try cursor.take(4, 13);
222
223 try std.testing.expect(cursor.complete());
224 try std.testing.expectEqual(@as(usize, 1), failing.alloc_index);
225 try std.testing.expectEqual(@as(usize, 416), failing.allocated_bytes);
226 }
227
228 test "indexed totals and unit rates retain zero denominator behavior" {
229 var deltas = IndexedDeltaTotals(2){};
230 deltas.record(0, 4, 16, 12);
231 deltas.record(0, 8, 32, 24);
232 try std.testing.expectEqual(@as(f64, 6), deltas.meanBefore(0, 2));
233 try std.testing.expectEqual(@as(f64, 18), deltas.meanDelta(0, 2));
234
235 var memory = IndexedMemoryTotals(1){};
236 memory.record(0, 2, 1, 40);
237 memory.record(0, 4, 3, 80);
238 try std.testing.expectEqual(@as(f64, 3), memory.meanAllocCount(0, 2));
239 try std.testing.expectEqual(@as(f64, 60), memory.meanAllocBytes(0, 2));
240 try std.testing.expectEqual(@as(f64, 0), nsPerUnit(10, 0));
241 try std.testing.expectEqual(@as(f64, 5), nsPerUnit(10, 2));
242 try std.testing.expectEqual(@as(f64, 0), nsPerFloatUnit(10, 0));
243 try std.testing.expectEqual(@as(f64, 2.5), nsPerFloatUnit(10, 4));
244 }
245
246 test "sample stats can disable bootstrap intervals" {
247 var storage = try Storage.init(std.testing.allocator, .{
248 .samples = 3,
249 .bootstrap_iterations = 0,
250 });
251 defer storage.deinit(std.testing.allocator);
252 storage.activate();
253 const result = try computeSampleStatsWithBootstrap(
254 &storage,
255 &.{ 10, 20, 30 },
256 .{ .iterations = 0 },
257 );
258 try std.testing.expect(result.confidence_intervals == null);
259 }