tiny.sql.history.repair
Defined in history.
API (4)
Actions
Public operations.
fixturePack: Builds the pack that the repair tests and the command-line tests run against, so they both start from this pack and exercise the same topology, by writing two commits over a small database and history in a temporary directory, then exporting the whole history as a pack.repair: Removes a row an interrupted deletion left behind when an operator repairs a backup through thepack repaircommand, rewriting one pack in place to repair its row topology around a single candidate row, named by its relation and its rowid, and returning how many roots it repaired and how many rows it dropped.
Types and contracts
Public types and contracts.
Source
Source: lib/sql/src/history/repair.zig
zig
const std = @import("std");const sql = @import("../root.zig");const record = @import("record.zig");const materialize = @import("materialize.zig");const Allocator = std.mem.Allocator;const Hash = sql.version.Hash;const State = sql.lattice.State;const Frame = sql.sync.PackFrame;const Reader = record.PayloadReader;pub const Candidate = struct { relation: []const u8, rowid: i64 };pub const Summary = struct { roots_repaired: usize = 0, rows_dropped: usize = 0 };const Aggregate = struct { states: []State, dropped: ?[]State = null, count: u64 = 0, bytes: u64 = 0, drop_bytes: u64 = 0, first: ?i64 = null, last: ?i64 = null, fn init(allocator: Allocator, count: usize) !Aggregate { const states = try allocator.alloc(State, count); @memset(states, State.empty); return .{ .states = states }; } fn deinit(self: *Aggregate, allocator: Allocator) void { allocator.free(self.states); if (self.dropped) |states| allocator.free(states); } fn add(self: *Aggregate, allocator: Allocator, other: *const Aggregate) !void { if (self.last) |last| if (other.first) |first| { if (first <= last) return error.UnprovablePackRepair; }; if (self.first == null) self.first = other.first; if (other.last != null) self.last = other.last; self.count = try std.math.add(u64, self.count, other.count); self.bytes = try std.math.add(u64, self.bytes, other.bytes); for (self.states, other.states) |*state, *part| state.add(part); if (other.dropped) |states| { if (self.dropped != null) return error.UnprovablePackRepair; self.dropped = try allocator.dupe(State, states); self.drop_bytes = other.drop_bytes; } } fn removeCandidate(self: *Aggregate) !void { const dropped = self.dropped orelse return error.UnprovablePackRepair; self.count = try std.math.sub(u64, self.count, 1); self.bytes = try std.math.sub(u64, self.bytes, self.drop_bytes); for (self.states, dropped) |*state, *part| state.subtract(part); }};const Engine = struct { allocator: Allocator, pack: *sql.sync.Pack, candidate: Candidate, objects: [6]std.AutoHashMapUnmanaged(Hash, usize) = @splat(.empty), chunks: std.AutoHashMapUnmanaged([64]u8, Aggregate) = .empty, pages: std.AutoHashMapUnmanaged([64]u8, Aggregate) = .empty, replacements: std.AutoHashMapUnmanaged(Hash, Hash) = .empty, additions: std.ArrayList(Frame) = .empty, added: std.AutoHashMapUnmanaged(Hash, void) = .empty, summary: Summary = .{}, fn deinit(self: *Engine) void { for (&self.objects) |*map| map.deinit(self.allocator); for ([_]*std.AutoHashMapUnmanaged([64]u8, Aggregate){ &self.chunks, &self.pages }) |cache| { var values = cache.valueIterator(); while (values.next()) |value| value.deinit(self.allocator); cache.deinit(self.allocator); } self.replacements.deinit(self.allocator); self.added.deinit(self.allocator); for (self.additions.items) |frame| self.allocator.free(frame.payload); self.additions.deinit(self.allocator); } fn index(self: *Engine) !void { for (self.pack.frames, 0..) |frame, i| { const kind = @backingInt(frame.kind); if (kind > 5) continue; var reader = Reader.init(frame.payload); const hash = if (kind == 5) blk: { if (frame.payload.len < 32) return error.InvalidPack; break :blk frame.payload[frame.payload.len - 32 ..][0..32].*; } else try reader.hash(); const slot = try self.objects[kind].getOrPut(self.allocator, hash); if (slot.found_existing) return error.InvalidPack; slot.value_ptr.* = i; } } fn payload(self: *const Engine, kind: usize, hash: Hash) ![]const u8 { const i = self.objects[kind].get(hash) orelse return error.InvalidPack; return self.pack.frames[i].payload; } fn validateLinks(self: *Engine) !void { var roots: std.AutoHashMapUnmanaged(Hash, void) = .empty; defer roots.deinit(self.allocator); var commits: std.AutoHashMapUnmanaged(Hash, void) = .empty; defer commits.deinit(self.allocator); for (self.pack.frames) |frame| { if (frame.kind != .database_root) continue; var reader = Reader.init(frame.payload); const conflicts = try reader.hash(); const count = try reader.readU32(); if (count > reader.remaining() / 36) return error.InvalidPack; const entries = try self.allocator.alloc(sql.version.RelationEntry, count); defer self.allocator.free(entries); for (entries) |*entry| { entry.* = .{ .name = try reader.readBytes(), .hash = try reader.hash() }; if (!self.objects[5].contains(entry.hash) or !self.objects[4].contains(entry.hash)) return error.InvalidPack; } try reader.finish(); var root = try sql.version.DatabaseRoot.initSorted( self.allocator, entries, .{ .hash = conflicts }, ); defer root.deinit(); try roots.put(self.allocator, root.hash, {}); } for (self.pack.commits) |commit| { try commits.put(self.allocator, commit.hash, {}); } for (self.pack.commits) |commit| { if (!roots.contains(commit.root)) return error.InvalidPack; for (commit.parents) |parent| { if (!commits.contains(parent)) return error.InvalidPack; } } for (self.pack.refs) |ref| { if (!commits.contains(ref.target)) return error.InvalidPack; } var row_roots = self.objects[4].keyIterator(); while (row_roots.next()) |hash| { if (!self.objects[5].contains(hash.*)) return error.InvalidPack; } } fn inspect(self: *Engine) !void { for (self.pack.frames) |frame| { if (frame.kind != .relation_root) continue; var reader = Reader.init(frame.payload); var decoded = try materialize.readRelationRootShallow(self.allocator, &reader); defer decoded.root.deinit(); defer self.allocator.free(decoded.index_keys); try reader.finish(); try self.inspectRoot(&decoded.root); } } fn inspectRoot(self: *Engine, root: *const sql.version.RelationRoot) !void { var aggregate = try Aggregate.init(self.allocator, root.indexes.len + 1); defer aggregate.deinit(self.allocator); var reader = Reader.init(try self.payload(4, root.hash)); _ = try reader.hash(); const count = try reader.readU32(); for (0..count) |_| { const page = try self.aggregatePage(root, try reader.hash()); try aggregate.add(self.allocator, page); } try reader.finish(); if (try matches(root, &aggregate)) return; if (!std.mem.eql(u8, root.name, self.candidate.relation)) { return error.UnprovablePackRepair; } try aggregate.removeCandidate(); if (!try matches(root, &aggregate)) return error.UnprovablePackRepair; try self.rewriteRows(root.hash); self.summary.roots_repaired += 1; self.summary.rows_dropped += 1; } fn aggregatePage( self: *Engine, root: *const sql.version.RelationRoot, hash: Hash, ) anyerror!*const Aggregate { const key = cacheKey(root, hash); if (self.pages.getPtr(key)) |found| return found; var aggregate = try Aggregate.init(self.allocator, root.indexes.len + 1); errdefer aggregate.deinit(self.allocator); const payload_bytes = try self.payload(2, hash); const hashes = try hashList(self.allocator, payload_bytes, true); defer self.allocator.free(hashes); for (hashes) |chunk| { const part = try self.aggregateChunk(root, chunk); try aggregate.add(self.allocator, part); } try self.pages.put(self.allocator, key, aggregate); return self.pages.getPtr(key).?; } fn aggregateChunk( self: *Engine, root: *const sql.version.RelationRoot, hash: Hash, ) !*const Aggregate { const key = cacheKey(root, hash); if (self.chunks.getPtr(key)) |found| return found; var aggregate = try Aggregate.init(self.allocator, root.indexes.len + 1); errdefer aggregate.deinit(self.allocator); const rows = try self.chunkRows(hash); defer sql.version.freeRelationRows(self.allocator, rows); for (rows) |row| { if (aggregate.last) |last| { if (row.rowid <= last) return error.UnprovablePackRepair; } if (aggregate.first == null) aggregate.first = row.rowid; aggregate.last = row.rowid; aggregate.count += 1; aggregate.bytes = try std.math.add(u64, aggregate.bytes, row.bytes.len); const states = try rowStates(self.allocator, root, row); defer self.allocator.free(states); for (aggregate.states, states) |*state, *part| state.add(part); if (row.rowid == self.candidate.rowid and std.mem.eql(u8, root.name, self.candidate.relation)) { aggregate.dropped = try self.allocator.dupe(State, states); aggregate.drop_bytes = row.bytes.len; } } try self.chunks.put(self.allocator, key, aggregate); return self.chunks.getPtr(key).?; } fn chunkRows(self: *Engine, hash: Hash) ![]sql.version.RelationRow { var reader = Reader.init(try self.payload(1, hash)); _ = try reader.hash(); const rows = try record.readRelationRows(self.allocator, &reader); errdefer sql.version.freeRelationRows(self.allocator, rows); try reader.finish(); if (!sql.version.same(hash, sql.chunk.digest(rows))) return error.InvalidPack; return rows; } fn rewriteRows(self: *Engine, hash: Hash) !void { const i = self.objects[4].get(hash) orelse return error.InvalidPack; const pages = try hashList(self.allocator, self.pack.frames[i].payload, false); defer self.allocator.free(pages); for (pages) |*page| page.* = try self.rewritePage(page.*); const payload_bytes = try encodeHashes(self.allocator, hash, pages); self.allocator.free(self.pack.frames[i].payload); self.pack.frames[i].payload = payload_bytes; } fn rewritePage(self: *Engine, hash: Hash) !Hash { if (self.replacements.get(hash)) |found| return found; const chunks = try hashList(self.allocator, try self.payload(2, hash), true); defer self.allocator.free(chunks); var changed = false; for (chunks) |*chunk| { const replacement = try self.rewriteChunk(chunk.*); changed = changed or !sql.version.same(chunk.*, replacement); chunk.* = replacement; } const result = if (changed) sql.chunk.pageDigest(chunks) else hash; if (changed) try self.addFrame(.{ .kind = .chunk_index_page, .payload = try encodeHashes(self.allocator, result, chunks), }); try self.replacements.put(self.allocator, hash, result); return result; } fn rewriteChunk(self: *Engine, hash: Hash) !Hash { if (self.replacements.get(hash)) |found| return found; const rows = try self.chunkRows(hash); defer sql.version.freeRelationRows(self.allocator, rows); var retained: std.ArrayList(sql.version.RelationRow) = .empty; defer retained.deinit(self.allocator); for (rows) |row| { if (row.rowid != self.candidate.rowid) try retained.append(self.allocator, row); } const changed = retained.items.len != rows.len; const result = if (changed) sql.chunk.digest(retained.items) else hash; if (changed) { var payload_bytes: std.ArrayList(u8) = .empty; errdefer payload_bytes.deinit(self.allocator); try record.appendHash(self.allocator, &payload_bytes, result); try record.appendRelationRows(self.allocator, &payload_bytes, retained.items); try self.addFrame(.{ .kind = .row_chunk, .payload = try payload_bytes.toOwnedSlice(self.allocator), }); } try self.replacements.put(self.allocator, hash, result); return result; } fn addFrame(self: *Engine, frame: Frame) !void { errdefer self.allocator.free(frame.payload); const hash = frame.payload[0..32].*; if (self.objects[@backingInt(frame.kind)].contains(hash) or self.added.contains(hash)) { self.allocator.free(frame.payload); return; } try self.added.put(self.allocator, hash, {}); try self.additions.append(self.allocator, frame); } fn finish(self: *Engine) !void { if (self.additions.items.len == 0) return; const length = try std.math.add(usize, self.pack.frames.len, self.additions.items.len); const frames = try self.allocator.alloc(Frame, length); var cursor: usize = 0; for ([_]record.PackRecordKind{ .row_chunk, .chunk_index_page }) |kind| { for (self.additions.items) |frame| { if (frame.kind != kind) continue; frames[cursor] = frame; cursor += 1; } for (self.pack.frames) |frame| { if (frame.kind != kind) continue; frames[cursor] = frame; cursor += 1; } } for (self.pack.frames) |frame| { if (frame.kind == .row_chunk or frame.kind == .chunk_index_page) continue; frames[cursor] = frame; cursor += 1; } std.debug.assert(cursor == length); self.allocator.free(self.pack.frames); self.pack.frames = frames; self.additions.clearRetainingCapacity(); }};/// Removes a row an interrupted deletion left behind when an operator repairs a/// backup through the `pack repair` command, rewriting one pack in place to/// repair its row topology around a single candidate row, named by its relation/// and its rowid, and returning how many roots it repaired and how many rows it/// dropped. The function drops the candidate only where every committed/// relation of that history rebuilds to the hash the commit recorded, and/// reports an error when any of them does not, while a relation whose roots/// already agree keeps the row, so the candidate names a row that may be/// dropped. The repair leaves the commits, the database roots, the relation/// roots, and the refs exactly as they were, and replaces only the chunk and/// page objects that disagreed, so a pack that already agrees comes back byte/// for byte and repeating the same repair on the same candidate changes nothing/// further. The caller supplies the allocator, and a failed repair frees what/// it took.pub fn repair(allocator: Allocator, pack: *sql.sync.Pack, candidate: Candidate) !Summary { var engine = Engine{ .allocator = allocator, .pack = pack, .candidate = candidate }; defer engine.deinit(); try engine.index(); try engine.validateLinks(); try engine.inspect(); try engine.finish(); return engine.summary;}fn cacheKey(root: *const sql.version.RelationRoot, hash: Hash) [64]u8 { var context = std.crypto.hash.sha2.Sha256.init(.{}); context.update(root.name); context.update(&root.schema); var result: [64]u8 = undefined; result[0..32].* = hash; context.final(result[32..64]); return result;}fn hashList(allocator: Allocator, payload_bytes: []const u8, page: bool) ![]Hash { var reader = Reader.init(payload_bytes); const expected = try reader.hash(); const count = try reader.readU32(); if (count > reader.remaining() / 32) return error.InvalidPack; const hashes = try allocator.alloc(Hash, count); errdefer allocator.free(hashes); for (hashes) |*hash| hash.* = try reader.hash(); try reader.finish(); if (page and !sql.version.same(expected, sql.chunk.pageDigest(hashes))) { return error.InvalidPack; } return hashes;}fn encodeHashes(allocator: Allocator, hash: Hash, hashes: []const Hash) ![]u8 { var payload: std.ArrayList(u8) = .empty; errdefer payload.deinit(allocator); try record.appendHash(allocator, &payload, hash); try record.appendU32(allocator, &payload, std.math.cast(u32, hashes.len) orelse return error.InvalidPack); for (hashes) |item| try record.appendHash(allocator, &payload, item); return try payload.toOwnedSlice(allocator);}fn rowStates( allocator: Allocator, root: *const sql.version.RelationRoot, row: sql.version.RelationRow,) ![]State { const result = try allocator.alloc(State, root.indexes.len + 1); errdefer allocator.free(result); var key: [sql.page.size]u8 = undefined; const table_key = try sql.key.encodeRowId(&key, row.rowid); result[0] = sql.lattice.entryState(table_key, row.bytes); const view = try sql.row.View.init(row.bytes); if (root.indexes.len != root.schema_descriptor.indexes.len) return error.InvalidPack; for (root.schema_descriptor.indexes, result[1..]) |index, *state| { if (index.fields.len > sql.relation.max_index_fields) return error.InvalidPack; var values: [sql.relation.max_index_fields]sql.row.Value = undefined; const projected = try view.project(index.fields, &values); const index_key = try sql.key.encodeIndex(&key, projected, index.columns, row.rowid); state.* = sql.lattice.entryState(index_key, ""); } return result;}const Hasher = struct { value: std.crypto.hash.sha2.Sha256, fn init(tag: []const u8) Hasher { var self = Hasher{ .value = std.crypto.hash.sha2.Sha256.init(.{}) }; self.bytes(tag); return self; } fn integer(self: *Hasher, comptime T: type, value: T) void { var buffer: [@sizeOf(T)]u8 = undefined; std.mem.writeInt(T, &buffer, value, .big); self.value.update(&buffer); } fn bytes(self: *Hasher, value: []const u8) void { self.integer(u64, value.len); self.value.update(value); } fn finish(self: *Hasher) Hash { var hash: Hash = undefined; self.value.final(&hash); return hash; }};fn matches(root: *const sql.version.RelationRoot, aggregate: *const Aggregate) !bool { if (root.format != sql.version.format_version) return error.InvalidPack; const schema = sql.version.schemaHash( root.schema_descriptor.columns, root.schema_descriptor.indexes, ); if (!sql.version.same(schema, root.schema)) return error.InvalidPack; if (aggregate.count != root.table.summary.entries or aggregate.bytes != root.table.summary.value_bytes) return false; const table_hash = aggregate.states[0].digest(aggregate.count); if (!sql.version.same(table_hash, root.table.hash)) return false; var hash = Hasher.init("sql.relation"); hash.integer(u32, root.format); hash.bytes(root.name); hash.value.update(&schema); hash.value.update(&root.table.hash); hash.value.update(&root.stats.hash); hash.integer(u64, root.indexes.len); for ( root.indexes, root.schema_descriptor.indexes, aggregate.states[1..], ) |index, definition, *state| { if (!sql.version.same(state.digest(aggregate.count), index.map.hash)) return false; var fields = Hasher.init("sql.index.fields"); fields.integer(u64, definition.fields.len); for (definition.fields) |field| fields.integer(u64, field); fields.integer(u64, definition.columns.len); for (definition.columns) |column| fields.integer(u8, @backingInt(column.collation)); const fields_hash = fields.finish(); if (!sql.version.same(fields_hash, index.fields)) return error.InvalidPack; var item = Hasher.init("sql.index"); item.value.update(&fields_hash); item.value.update(&index.map.hash); item.value.update(&index.stats); const index_hash = item.finish(); if (!sql.version.same(index_hash, index.hash)) return error.InvalidPack; hash.value.update(&index_hash); } return sql.version.same(hash.finish(), root.hash);}fn execute(connection: *sql.Connection, statement: []const u8) !void { var result = try connection.execute( std.testing.allocator, statement, .{ .durability = .buffered }, ); defer result.deinit(std.testing.allocator);}fn fixture(database: *sql.FileDatabase, history: *sql.History, corrupt: bool) !void { const allocator = std.testing.allocator; var connection = try sql.Connection.create(allocator, database, history, .{}); defer connection.deinit(); try execute(&connection, "CREATE TABLE items (value, INDEX by_value (value))"); for (1..514) |i| { var buffer: [128]u8 = undefined; const statement = try std.fmt.bufPrint( &buffer, "INSERT INTO items (rowid, value) VALUES ({d}, 'value-{d}')", .{ i, i }, ); try execute(&connection, statement); } try connection.stage(); _ = try connection.commit(history); if (!corrupt) return; var base = try connection.materializedWorkingValue(allocator); defer base.deinit(); try execute(&connection, "DELETE FROM items WHERE rowid = 1"); try execute(&connection, "INSERT INTO items (rowid, value) VALUES (514, 'new')"); try publishCorruptFixture(&connection, history, &base);}fn publishCorruptFixture( connection: *sql.Connection, history: *sql.History, base: *const sql.version.DatabaseValue,) !void { const allocator = std.testing.allocator; var value = try connection.materializedWorkingValue(allocator); defer value.deinit(); const before = base.findRelation("items").?; const after = value.findRelation("items").?; const rows = try allocator.alloc(sql.version.RelationRow, after.rows.len + 1); defer allocator.free(rows); rows[0] = before.rows[0]; @memcpy(rows[1..], after.rows); try history.putRelationRoot(after.root); try history.putRelationRows(after.root.hash, rows); try history.putDatabaseRoot(value.root); _ = try history.commitBranch("main", value.root.hash);}/// Builds the pack that the repair tests and the command-line tests run/// against, so they both start from this pack and exercise the same topology,/// by writing two commits over a small database and history in a temporary/// directory, then exporting the whole history as a pack. With `corrupt` true,/// the pack carries the stale row topology an interrupted deletion leaves,/// while with it false, the pack already agrees. The function compiles only in/// test builds, and the caller frees the returned pack.pub fn fixturePack(corrupt: bool) !sql.sync.Pack { if (!@import("builtin").is_test) @compileError("fixturePack is available only in tests"); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var database = try sql.FileDatabase.openForTesting(std.testing.allocator, tmp.dir, .{ .paths = .{ .database = "source.db", .wal = "source.wal" }, .header = .{ .sequence = 1, .salt = .{ .first = 17, .second = 19 } }, }); defer database.deinit(); try database.reserve(.{ .wal_frames = 8192 }); var history = try sql.History.open(std.testing.allocator, tmp.dir, .{ .path = "source.history", .recovery = .reject, }); defer history.deinit(); try fixture(&database, &history, corrupt); return try sql.exportHistoryPack(std.testing.allocator, &history);}test "pack repair proves interrupted deletion and imports unchanged committed identities" { const allocator = std.testing.allocator; var pack = try fixturePack(true); defer pack.deinit(); const head = pack.refs[0].target; const commits = pack.commits.len; const result = try repair(allocator, &pack, .{ .relation = "items", .rowid = 1 }); try std.testing.expectEqual(@as(usize, 1), result.roots_repaired); try std.testing.expectEqual(@as(usize, 1), result.rows_dropped); try std.testing.expectEqualSlices(u8, &head, &pack.refs[0].target); try std.testing.expectEqual(commits, pack.commits.len); try verifyImport(&pack); const repaired_bytes = try sql.encodeHistoryPack(allocator, &pack); defer allocator.free(repaired_bytes); const repeated = try repair(allocator, &pack, .{ .relation = "items", .rowid = 1 }); try std.testing.expectEqual(@as(usize, 0), repeated.roots_repaired); const repeated_bytes = try sql.encodeHistoryPack(allocator, &pack); defer allocator.free(repeated_bytes); try std.testing.expectEqualSlices(u8, repaired_bytes, repeated_bytes);}fn verifyImport(pack: *const sql.sync.Pack) !void { const allocator = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var database = try sql.FileDatabase.openForTesting(allocator, tmp.dir, .{ .paths = .{ .database = "target.db", .wal = "target.wal" }, .header = .{ .sequence = 2, .salt = .{ .first = 23, .second = 29 } }, }); defer database.deinit(); try database.reserve(.{ .wal_frames = 8192 }); var history = try sql.History.open(allocator, tmp.dir, .{ .path = "target.history", .recovery = .reject, }); defer history.deinit(); _ = try sql.importHistoryPack(&history, pack); var connection = try sql.Connection.open(allocator, &database, &history, .{}); defer connection.deinit(); try connection.checkoutBranch(allocator, &history, "main"); for (pack.commits) |commit| { var value = try history.databaseValue(allocator, commit.root); defer value.deinit(); for (value.relations) |relation| { var rebuilt = try sql.version.relationRootFromRows( allocator, &relation.root, relation.rows, ); defer rebuilt.deinit(); try std.testing.expect(sql.version.same(rebuilt.hash, relation.root.hash)); } }}test "pack repair preserves consistent bytes and refuses an unproved deletion" { const allocator = std.testing.allocator; var good = try fixturePack(false); defer good.deinit(); const before = try sql.encodeHistoryPack(allocator, &good); defer allocator.free(before); const summary = try repair(allocator, &good, .{ .relation = "items", .rowid = 1 }); try std.testing.expectEqual(@as(usize, 0), summary.roots_repaired); const after = try sql.encodeHistoryPack(allocator, &good); defer allocator.free(after); try std.testing.expectEqualSlices(u8, before, after); var bad = try fixturePack(true); defer bad.deinit(); try std.testing.expectError( error.UnprovablePackRepair, repair(allocator, &bad, .{ .relation = "items", .rowid = 2 }), );}Source: lib/sql/src/history/root.zig:33
zig
pub const repair = @import("repair.zig");Audit
| Definitions | 5 |
|---|---|
| Public names | 5 |
| Members | 4 |
| Version | 26.7.0 |
| Revision | daab053ee433 |