lib/sql/src/history/segment/test.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const sql = @import("../../root.zig");
3 const namespace = @import("root.zig");
4
5 const manifest = namespace.manifest;
6
7 fn makeSegment(
8 storage: []align(namespace.Sealer.storage_alignment) u8,
9 ordinal: u64,
10 previous: sql.Hash,
11 ) !namespace.Sealed {
12 var sealer = try namespace.Sealer.init(storage, .{
13 .blocks = 1,
14 .block_bytes = 5,
15 .closure_digests = 1,
16 });
17 sealer.activate();
18 try sealer.addClosure(@splat(0x44));
19 try sealer.addBlock(.{
20 .first_event = ordinal * 2,
21 .event_count = 2,
22 .raw_bytes = 5,
23 .bytes = "block",
24 });
25 const sealed = try sealer.finish(.{
26 .epoch = 7,
27 .ordinal = ordinal,
28 .first_event = ordinal * 2,
29 .event_count = 2,
30 .previous_segment = previous,
31 .event_chain = @splat(0x11),
32 .checkpoint_root = @splat(0x22),
33 .schema_identity = @splat(0x33),
34 .stream_identity = @splat(0x55),
35 });
36 _ = sealer.deinit();
37 return sealed;
38 }
39
40 test "segment capacity matches the independent record model" {
41 comptime {
42 @stardustClaim(
43 @import("alloc_phase").capacity.witness(
44 namespace.Sealer,
45 "sql_history_segment_capacity",
46 ),
47 null,
48 null,
49 null,
50 null,
51 null,
52 null,
53 );
54 }
55 const limits = namespace.Sealer.Limits{
56 .blocks = 3,
57 .block_bytes = 1025,
58 .closure_digests = 7,
59 };
60 const capacity = try namespace.Sealer.Capacity.derive(limits);
61 const expected = namespace.header_bytes + namespace.trailer_bytes +
62 limits.blocks * namespace.block_header_bytes +
63 limits.closure_digests * namespace.closure_record_bytes +
64 limits.block_bytes;
65 try std.testing.expectEqual(expected, capacity.storage_bytes);
66 try std.testing.expectError(
67 error.CapacityExceeded,
68 namespace.Sealer.Capacity.derive(.{
69 .blocks = namespace.blocks_max + 1,
70 .block_bytes = 1,
71 .closure_digests = 0,
72 }),
73 );
74 }
75
76 test "segment reservation rejects overload before output mutation" {
77 comptime {
78 @stardustClaim(
79 @import("alloc_phase").capacity.witness(
80 namespace.Sealer,
81 "sql_history_segment_overload",
82 ),
83 null,
84 null,
85 null,
86 null,
87 null,
88 null,
89 );
90 }
91 const capacity = try namespace.Sealer.Capacity.derive(.{
92 .blocks = 1,
93 .block_bytes = 5,
94 .closure_digests = 1,
95 });
96 const storage = try std.testing.allocator.alignedAlloc(
97 u8,
98 .fromByteUnits(namespace.Sealer.storage_alignment),
99 capacity.storage_bytes,
100 );
101 defer std.testing.allocator.free(storage);
102 @memset(storage, 0xa5);
103 var sealer = try namespace.Sealer.init(storage, .{
104 .blocks = 1,
105 .block_bytes = 5,
106 .closure_digests = 1,
107 });
108 sealer.activate();
109 try sealer.addClosure(@splat(1));
110 const before = storage[namespace.header_bytes..][0..namespace.closure_record_bytes].*;
111 try std.testing.expectError(
112 error.TooManyClosureDigests,
113 sealer.addClosure(@splat(2)),
114 );
115 try std.testing.expectEqualSlices(
116 u8,
117 &before,
118 storage[namespace.header_bytes..][0..namespace.closure_record_bytes],
119 );
120 _ = sealer.deinit();
121 }
122
123 test "segment codec accepts final counters and rejects overflow before encoding" {
124 var storage: [
125 namespace.header_bytes + namespace.block_header_bytes + 1 +
126 namespace.trailer_bytes
127 ]u8 align(namespace.Sealer.storage_alignment) = undefined;
128 @memset(&storage, 0xa5);
129 var sealer = try namespace.Sealer.init(&storage, .{
130 .blocks = 1,
131 .block_bytes = 1,
132 .closure_digests = 0,
133 });
134 sealer.activate();
135 try sealer.addBlock(.{
136 .first_event = namespace.EventRange.budget_events - 1,
137 .event_count = 1,
138 .raw_bytes = 1,
139 .bytes = "x",
140 });
141 const sealed = try sealer.finish(.{
142 .epoch = 0,
143 .ordinal = namespace.SegmentRange.budget_segments - 1,
144 .first_event = namespace.EventRange.budget_events - 1,
145 .event_count = 1,
146 .previous_segment = @splat(0),
147 .event_chain = @splat(1),
148 .checkpoint_root = @splat(2),
149 .schema_identity = @splat(3),
150 .stream_identity = @splat(4),
151 });
152 _ = sealer.deinit();
153 const view = try namespace.verify(sealed.bytes);
154 try std.testing.expectEqual(
155 namespace.SegmentRange.budget_segments - 1,
156 view.spec.ordinal,
157 );
158 try std.testing.expectEqual(
159 namespace.EventRange.budget_events,
160 view.spec.first_event + view.spec.event_count,
161 );
162
163 var rejected = try namespace.Sealer.init(&storage, .{
164 .blocks = 1,
165 .block_bytes = 1,
166 .closure_digests = 0,
167 });
168 rejected.activate();
169 const before = storage;
170 try std.testing.expectError(error.InvalidSegment, rejected.addBlock(.{
171 .first_event = namespace.EventRange.budget_events,
172 .event_count = 1,
173 .raw_bytes = 1,
174 .bytes = "x",
175 }));
176 try std.testing.expectEqualSlices(u8, &before, &storage);
177 try rejected.addBlock(.{
178 .first_event = namespace.EventRange.budget_events - 1,
179 .event_count = 1,
180 .raw_bytes = 1,
181 .bytes = "x",
182 });
183 _ = rejected.deinit();
184 }
185
186 test "manifest rejects over-budget entries before changing output buffers" {
187 var page: [manifest.page_bytes]u8 = @splat(0xa5);
188 var root: [manifest.root_bytes]u8 = @splat(0x5a);
189 const page_before = page;
190 const root_before = root;
191 var entry = manifest.Entry{
192 .segment = @splat(1),
193 .previous_segment = @splat(0),
194 .stream_identity = @splat(2),
195 .checkpoint_root = @splat(3),
196 .epoch = 0,
197 .ordinal = namespace.SegmentRange.budget_segments,
198 .first_event = 0,
199 .event_count = 1,
200 .encoded_bytes = 1,
201 };
202 try std.testing.expectError(
203 error.InvalidManifest,
204 manifest.append(null, null, entry, &page, &root),
205 );
206 try std.testing.expectEqualSlices(u8, &page_before, &page);
207 try std.testing.expectEqualSlices(u8, &root_before, &root);
208
209 entry.ordinal = 0;
210 entry.first_event = namespace.EventRange.budget_events;
211 try std.testing.expectError(
212 error.InvalidManifest,
213 manifest.append(null, null, entry, &page, &root),
214 );
215 try std.testing.expectEqualSlices(u8, &page_before, &page);
216 try std.testing.expectEqualSlices(u8, &root_before, &root);
217 }
218
219 test "segment decoder rejects checksummed over-budget counters" {
220 var storage: [
221 namespace.header_bytes + namespace.closure_record_bytes +
222 namespace.block_header_bytes + 5 + namespace.trailer_bytes
223 ]u8 align(namespace.Sealer.storage_alignment) = undefined;
224 const sealed = try makeSegment(&storage, 0, @splat(0));
225 std.mem.writeInt(
226 u64,
227 storage[24..32],
228 namespace.SegmentRange.budget_segments,
229 .little,
230 );
231 const trailer_start = sealed.bytes.len - namespace.trailer_bytes;
232 var trailer: sql.Hash = undefined;
233 std.crypto.hash.sha2.Sha256.hash(
234 storage[0..trailer_start],
235 &trailer,
236 .{},
237 );
238 @memcpy(storage[trailer_start..], &trailer);
239 try std.testing.expectError(error.InvalidSegment, namespace.verify(&storage));
240
241 std.mem.writeInt(u64, storage[24..32], 0, .little);
242 std.mem.writeInt(
243 u64,
244 storage[32..40],
245 namespace.EventRange.budget_events,
246 .little,
247 );
248 std.crypto.hash.sha2.Sha256.hash(
249 storage[0..trailer_start],
250 &trailer,
251 .{},
252 );
253 @memcpy(storage[trailer_start..], &trailer);
254 try std.testing.expectError(error.InvalidSegment, namespace.verify(&storage));
255 }
256
257 test "manifest decoders reject checksummed over-budget counters" {
258 var segment_storage: [
259 namespace.header_bytes + namespace.closure_record_bytes +
260 namespace.block_header_bytes + 5 + namespace.trailer_bytes
261 ]u8 align(namespace.Sealer.storage_alignment) = undefined;
262 const sealed = try makeSegment(&segment_storage, 0, @splat(0));
263 var root: [manifest.root_bytes]u8 = undefined;
264 var page: [manifest.page_bytes]u8 = undefined;
265 _ = try manifest.append(
266 null,
267 null,
268 manifest.Entry.fromVerified(try namespace.verify(sealed.bytes)),
269 &page,
270 &root,
271 );
272
273 const final_tail_entries: u32 = @intCast(
274 (namespace.SegmentRange.budget_segments - 1) %
275 @as(u64, manifest.entries_per_page) + 1,
276 );
277 std.mem.writeInt(u32, root[12..16], final_tail_entries, .little);
278 std.mem.writeInt(
279 u64,
280 root[16..24],
281 namespace.SegmentRange.budget_segments,
282 .little,
283 );
284 std.mem.writeInt(
285 u64,
286 root[24..32],
287 manifest.PageRange.budget_pages,
288 .little,
289 );
290 var checksum: sql.Hash = undefined;
291 std.crypto.hash.sha2.Sha256.hash(
292 root[0 .. root.len - checksum.len],
293 &checksum,
294 .{},
295 );
296 @memcpy(root[root.len - checksum.len ..], &checksum);
297 const final_root = try manifest.verifyRoot(&root);
298 try std.testing.expectEqual(
299 namespace.SegmentRange.budget_segments,
300 final_root.segment_count,
301 );
302 try std.testing.expectEqual(
303 manifest.PageRange.budget_pages,
304 final_root.page_count,
305 );
306
307 std.mem.writeInt(
308 u64,
309 root[16..24],
310 namespace.SegmentRange.budget_segments + 1,
311 .little,
312 );
313 std.crypto.hash.sha2.Sha256.hash(root[0 .. root.len - checksum.len], &checksum, .{});
314 @memcpy(root[root.len - checksum.len ..], &checksum);
315 try std.testing.expectError(error.InvalidManifest, manifest.verifyRoot(&root));
316
317 std.mem.writeInt(
318 u64,
319 root[16..24],
320 namespace.SegmentRange.budget_segments,
321 .little,
322 );
323 std.mem.writeInt(
324 u64,
325 root[24..32],
326 manifest.PageRange.budget_pages + 1,
327 .little,
328 );
329 std.crypto.hash.sha2.Sha256.hash(
330 root[0 .. root.len - checksum.len],
331 &checksum,
332 .{},
333 );
334 @memcpy(root[root.len - checksum.len ..], &checksum);
335 try std.testing.expectError(error.InvalidManifest, manifest.verifyRoot(&root));
336
337 const entry_start = manifest.page_header_bytes;
338 std.mem.writeInt(
339 u64,
340 page[16..24],
341 namespace.SegmentRange.budget_segments - 1,
342 .little,
343 );
344 std.mem.writeInt(
345 u64,
346 page[entry_start + 168 ..][0..8],
347 namespace.SegmentRange.budget_segments - 1,
348 .little,
349 );
350 std.mem.writeInt(
351 u64,
352 page[entry_start + 176 ..][0..8],
353 namespace.EventRange.budget_events - 1,
354 .little,
355 );
356 std.mem.writeInt(u64, page[entry_start + 184 ..][0..8], 1, .little);
357 const entries_end = entry_start + manifest.entry_bytes;
358 var entries_digest: sql.Hash = undefined;
359 std.crypto.hash.sha2.Sha256.hash(
360 page[entry_start..entries_end],
361 &entries_digest,
362 .{},
363 );
364 @memcpy(page[64..96], &entries_digest);
365 std.crypto.hash.sha2.Sha256.hash(
366 page[0 .. page.len - checksum.len],
367 &checksum,
368 .{},
369 );
370 @memcpy(page[page.len - checksum.len ..], &checksum);
371 const final_page = try manifest.verifyPage(&page);
372 const final_entry = try final_page.latest();
373 try std.testing.expectEqual(
374 namespace.SegmentRange.budget_segments - 1,
375 final_entry.ordinal,
376 );
377 try std.testing.expectEqual(
378 namespace.EventRange.budget_events - 1,
379 final_entry.first_event,
380 );
381
382 std.mem.writeInt(
383 u64,
384 page[entry_start + 176 ..][0..8],
385 namespace.EventRange.budget_events,
386 .little,
387 );
388 std.crypto.hash.sha2.Sha256.hash(
389 page[entry_start..entries_end],
390 &entries_digest,
391 .{},
392 );
393 @memcpy(page[64..96], &entries_digest);
394 std.crypto.hash.sha2.Sha256.hash(
395 page[0 .. page.len - checksum.len],
396 &checksum,
397 .{},
398 );
399 @memcpy(page[page.len - checksum.len ..], &checksum);
400 try std.testing.expectError(error.InvalidManifest, manifest.verifyPage(&page));
401 }
402
403 test "sealed segment verifies every header record and digest" {
404 comptime {
405 @stardustClaim(
406 @import("alloc_phase").capacity.witness(
407 namespace.Sealer,
408 "sql_history_segment_verification",
409 ),
410 null,
411 null,
412 null,
413 null,
414 null,
415 null,
416 );
417 }
418 comptime {
419 @stardustClaim(
420 @import("alloc_phase").capacity.witness(
421 namespace.Sealer,
422 "sql_history_segment_work_bound",
423 ),
424 null,
425 null,
426 null,
427 null,
428 null,
429 null,
430 );
431 }
432 const capacity = try namespace.Sealer.Capacity.derive(.{
433 .blocks = 1,
434 .block_bytes = 5,
435 .closure_digests = 1,
436 });
437 const storage = try std.testing.allocator.alignedAlloc(
438 u8,
439 .fromByteUnits(namespace.Sealer.storage_alignment),
440 capacity.storage_bytes,
441 );
442 defer std.testing.allocator.free(storage);
443 const sealed = try makeSegment(storage, 0, @splat(0));
444 const view = try namespace.verify(sealed.bytes);
445 try std.testing.expectEqual(@as(u64, 2), view.spec.event_count);
446 const demand = try view.demand();
447 try std.testing.expectEqual(@as(u64, 1), demand.blocks);
448 try std.testing.expectEqual(@as(u64, 2), demand.events);
449 try std.testing.expectEqual(@as(u64, 5), demand.raw_event_bytes);
450 try std.testing.expectEqual(@as(u64, 1), demand.closure_digests);
451 var iterator = view.iterator();
452 switch ((try iterator.next()).?) {
453 .closure => {},
454 .block => return error.TestUnexpectedResult,
455 }
456 switch ((try iterator.next()).?) {
457 .block => {},
458 .closure => return error.TestUnexpectedResult,
459 }
460 try std.testing.expect((try iterator.next()) == null);
461 storage[
462 namespace.header_bytes + namespace.closure_record_bytes +
463 namespace.block_header_bytes
464 ] ^= 1;
465 try std.testing.expectError(error.InvalidSegment, namespace.verify(storage));
466 }
467
468 test "paged manifest appends with one-page replay memory" {
469 var first_storage: [
470 namespace.header_bytes + namespace.closure_record_bytes +
471 namespace.block_header_bytes + 5 + namespace.trailer_bytes
472 ]u8 align(namespace.Sealer.storage_alignment) = undefined;
473 const first = try makeSegment(&first_storage, 0, @splat(0));
474 var root: [manifest.root_bytes]u8 = undefined;
475 var page: [manifest.page_bytes]u8 = undefined;
476 var publication = try manifest.append(
477 null,
478 null,
479 manifest.Entry.fromVerified(try namespace.verify(first.bytes)),
480 &page,
481 &root,
482 );
483 try std.testing.expectEqual(@as(u64, 1), publication.root.segment_count);
484 var previous = first.digest;
485 var segment_storage: [
486 namespace.header_bytes + namespace.closure_record_bytes +
487 namespace.block_header_bytes + 5 + namespace.trailer_bytes
488 ]u8 align(namespace.Sealer.storage_alignment) = undefined;
489 var first_page: [manifest.page_bytes]u8 = undefined;
490 var ordinal: u64 = 1;
491 while (ordinal <= manifest.entries_per_page) : (ordinal += 1) {
492 if (ordinal == manifest.entries_per_page) first_page = page;
493 const next = try makeSegment(&segment_storage, ordinal, previous);
494 var next_root: [manifest.root_bytes]u8 = undefined;
495 var next_page: [manifest.page_bytes]u8 = undefined;
496 publication = try manifest.append(
497 &root,
498 &page,
499 manifest.Entry.fromVerified(try namespace.verify(next.bytes)),
500 &next_page,
501 &next_root,
502 );
503 root = next_root;
504 page = next_page;
505 previous = next.digest;
506 }
507 try std.testing.expectEqual(@as(u64, 2), publication.root.page_count);
508 var reader = manifest.Reader.init(publication.root);
509 const accepted = try reader.accept(&page);
510 try std.testing.expectEqual(@as(u32, 1), accepted.count);
511 try std.testing.expect(!reader.done());
512 const older = try reader.accept(&first_page);
513 try std.testing.expectEqual(
514 @as(u32, manifest.entries_per_page),
515 older.count,
516 );
517 try std.testing.expect(reader.done());
518 const demand = try reader.demand();
519 try std.testing.expectEqual(@as(u64, 20), demand.segments);
520 try std.testing.expectEqual(@as(u64, 40), demand.event_count);
521 try std.testing.expectEqual(@as(u64, 20), demand.sealed_segments);
522 try std.testing.expectEqual(@as(u64, 0), demand.cold_fetch_count);
523 }
524
525 test "cold receipt is forward only and preserves sealed bytes" {
526 var segment_storage: [
527 namespace.header_bytes + namespace.closure_record_bytes +
528 namespace.block_header_bytes + 5 + namespace.trailer_bytes
529 ]u8 align(namespace.Sealer.storage_alignment) = undefined;
530 const sealed = try makeSegment(&segment_storage, 0, @splat(0));
531 var root: [manifest.root_bytes]u8 = undefined;
532 var page: [manifest.page_bytes]u8 = undefined;
533 _ = try manifest.append(
534 null,
535 null,
536 manifest.Entry.fromVerified(try namespace.verify(sealed.bytes)),
537 &page,
538 &root,
539 );
540 const before = segment_storage;
541 var corrupt = segment_storage;
542 corrupt[namespace.header_bytes] ^= 1;
543 var cold_root: [manifest.root_bytes]u8 = undefined;
544 var cold_page: [manifest.page_bytes]u8 = undefined;
545 try std.testing.expectError(
546 error.InvalidSegment,
547 manifest.promoteTailCold(
548 &root,
549 &page,
550 &corrupt,
551 &cold_page,
552 &cold_root,
553 ),
554 );
555 const promoted = try manifest.promoteTailCold(
556 &root,
557 &page,
558 &segment_storage,
559 &cold_page,
560 &cold_root,
561 );
562 try std.testing.expect((try promoted.page.latest()).localReleaseAllowed());
563 try std.testing.expectEqualSlices(u8, &before, &segment_storage);
564 var reader = manifest.Reader.init(promoted.root);
565 _ = try reader.accept(&cold_page);
566 const demand = try reader.demand();
567 try std.testing.expectEqual(@as(u64, 1), demand.cold_segments);
568 try std.testing.expectEqual(@as(u64, 1), demand.cold_fetch_count);
569 try std.testing.expectEqual(
570 @as(u64, segment_storage.len),
571 demand.cold_bytes,
572 );
573 try std.testing.expectError(
574 error.InvalidLifecycleTransition,
575 manifest.promoteTailCold(
576 &cold_root,
577 &cold_page,
578 &segment_storage,
579 &page,
580 &root,
581 ),
582 );
583 }
584
585 test "segment namespace" {
586 std.testing.refAllDecls(namespace);
587 }