tiny.glom.Database

1pub const Database = struct {2    allocator: std.mem.Allocator,3    path: []u8,4    dir_path: []u8,5    db_name: []u8,6    wal_name: []u8,7    temporary_dir: ?[]u8,8    lock_file: std.Io.File,9    workspace: *sql.FileDatabase.Workspace,10    file: *sql.FileDatabase,11    file_live: bool,12    space: sql.Space,13    documents: sql.RowIdTable,14    search_refs: sql.RowIdTable,15    documents_search: sql.Search,16    initialized: bool,17    next_document_rowid: ?i64,18    max_wal_bytes: usize,19    checkpoint_wal_bytes: usize,20    normal_capacity: sql.PagerCapacity,21    bulk_capacity: sql.PagerCapacity,22    write_mode: WriteMode,2324    pub fn init(allocator: std.mem.Allocator, maybe_path: ?[]const u8) !Database {25        return try initWithPolicy(allocator, maybe_path, .{});26    }2728    fn initWithPolicy(29        allocator: std.mem.Allocator,30        maybe_path: ?[]const u8,31        policy: InitPolicy,32    ) !Database {33        try validateInitPolicy(policy);34        const path_info = if (maybe_path) |value|35            if (std.mem.eql(u8, value, ":memory:")) try temporaryPath(allocator) else PathInfo{ .path = try allocator.dupe(u8, value), .temporary_dir = null }36        else37            PathInfo{ .path = try defaultPath(allocator), .temporary_dir = null };38        errdefer freePathInfo(allocator, path_info);3940        const dir_part = std.fs.path.dirname(path_info.path) orelse ".";41        try std.Io.Dir.cwd().createDirPath(fs_io, dir_part);42        const dir_path = try allocator.dupe(u8, dir_part);43        errdefer allocator.free(dir_path);44        const db_name = try allocator.dupe(u8, std.fs.path.basename(path_info.path));45        errdefer allocator.free(db_name);46        const wal_name = try std.fmt.allocPrint(allocator, "{s}.wal", .{db_name});47        errdefer allocator.free(wal_name);48        const wal_absolute = try std.fmt.allocPrint(allocator, "{s}.wal", .{path_info.path});49        defer allocator.free(wal_absolute);50        const initialized = fileSize(path_info.path) != 0 or fileSize(wal_absolute) != 0;5152        var dir = try openDir(dir_path);53        defer dir.close(fs_io);54        const lock_name = try std.fmt.allocPrint(allocator, "{s}.lock", .{db_name});55        defer allocator.free(lock_name);56        const lock_file = try dir.createFile(fs_io, lock_name, .{57            .truncate = false,58            .read = true,59            .permissions = @fromBackingInt(@intCast(0o600)),60        });61        errdefer lock_file.close(fs_io);62        if (!try lock_file.tryLock(fs_io, .exclusive)) return error.DatabaseBusy;63        errdefer {64            lock_file.unlock(fs_io);65            lock_file.close(fs_io);66        }6768        const file_ptr = try allocator.create(sql.FileDatabase);69        errdefer allocator.destroy(file_ptr);70        const workspace = try allocator.create(sql.FileDatabase.Workspace);71        errdefer allocator.destroy(workspace);72        workspace.* = try sql.FileDatabase.Workspace.allocate(allocator, .{73            .header = databaseHeader(),74            .max_wal_bytes = policy.max_wal_bytes,75            .path_storage = sql.file.PathStorage.Limits.forDirect(.{76                .database = db_name,77                .wal = wal_name,78            }),79        });80        errdefer workspace.deallocate(allocator);81        file_ptr.* = try openFileWithPolicy(82            allocator,83            workspace,84            dir,85            db_name,86            wal_name,87            policy,88        );89        errdefer file_ptr.deinit();9091        const space = try sql.Space.open(file_ptr, .{ .meta_page = meta_root, .roots = &roots, .reserved_page_max = root_max });92        return .{93            .allocator = allocator,94            .path = path_info.path,95            .dir_path = dir_path,96            .db_name = db_name,97            .wal_name = wal_name,98            .temporary_dir = path_info.temporary_dir,99            .lock_file = lock_file,100            .workspace = workspace,101            .file = file_ptr,102            .file_live = true,103            .space = space,104            .documents = try space.rowidTable(documents_root),105            .search_refs = try space.rowidTable(search_refs_root),106            .documents_search = try openDocumentSearch(file_ptr),107            .initialized = initialized,108            .next_document_rowid = null,109            .max_wal_bytes = policy.max_wal_bytes,110            .checkpoint_wal_bytes = policy.checkpoint_wal_bytes,111            .normal_capacity = policy.normal_capacity,112            .bulk_capacity = policy.bulk_capacity,113            .write_mode = .normal,114        };115    }116117    pub fn deinit(self: *Database) void {118        if (self.file_live) self.file.deinit();119        self.allocator.destroy(self.file);120        self.workspace.deallocate(self.allocator);121        self.allocator.destroy(self.workspace);122        self.lock_file.unlock(fs_io);123        self.lock_file.close(fs_io);124        if (self.temporary_dir) |dir| {125            std.Io.Dir.cwd().deleteTree(fs_io, dir) catch {};126            self.allocator.free(dir);127        }128        self.allocator.free(self.wal_name);129        self.allocator.free(self.db_name);130        self.allocator.free(self.dir_path);131        self.allocator.free(self.path);132    }133134    pub fn commit(self: *Database) !void {135        try self.file.syncWal();136    }137138    pub fn checkpoint(self: *Database) !void {139        _ = try nativeCheckpoint(self);140    }141142    pub fn begin(self: *Database) !void {143        _ = self;144    }145146    pub fn getAllDocumentStates(self: *Database, allocator: std.mem.Allocator) !std.StringHashMap(DocumentState) {147        var map = std.StringHashMap(DocumentState).init(allocator);148        if (!self.initialized or !try self.rootExists(documents_root)) return map;149        errdefer {150            var iter = map.keyIterator();151            while (iter.next()) |key| allocator.free(key.*);152            map.deinit();153        }154        var scan: sql.TableScan = undefined;155        try self.documents.scan(&scan, allocator, null, null);156        defer scan.deinit();157        while (try scan.next()) |entry| {158            const doc = try documentViewFromBytes(entry.bytes);159            const transcript = isTranscriptKind(doc.kind);160            const records_current = transcript and161                transcriptRecordsCurrent(doc.transcript_records, doc.content.len);162            try map.put(try allocator.dupe(u8, doc.path), .{163                .rowid = entry.rowid,164                .mtime = if (transcript and !records_current) -std.math.inf(f64) else doc.mtime,165                .size = doc.size,166                .transcript_saturated = records_current and transcriptRecordsSaturate(167                    doc.transcript_records,168                    doc.content.len,169                ),170            });171        }172        return map;173    }174175    pub fn getAllPaths(self: *Database, allocator: std.mem.Allocator) !std.StringHashMap(void) {176        var map = std.StringHashMap(void).init(allocator);177        if (!self.initialized or !try self.rootExists(documents_root)) return map;178        errdefer {179            var iter = map.keyIterator();180            while (iter.next()) |key| allocator.free(key.*);181            map.deinit();182        }183        var scan: sql.TableScan = undefined;184        try self.documents.scan(&scan, allocator, null, null);185        defer scan.deinit();186        while (try scan.next()) |entry| {187            const doc = try documentFromBytes(allocator, entry.bytes);188            defer freeDocumentRow(allocator, doc);189            try map.put(try allocator.dupe(u8, doc.path), {});190        }191        return map;192    }193194    pub fn nativeSearchAvailable(self: *Database) bool {195        _ = self;196        return true;197    }198199    pub fn upsert(self: *Database, doc: data.Document) !void {200        const rowid = try self.rowidForDocumentPath(doc.path);201        try self.writeDocument(rowid, doc, true);202    }203204    pub fn update(self: *Database, rowid: i64, doc: data.Document) !void {205        std.debug.assert(rowid > 0);206        try self.writeDocument(rowid, doc, true);207    }208209    pub fn insertNew(self: *Database, doc: data.Document) !void {210        const rowid = try self.nextCachedRowid(&self.documents, documents_root, &self.next_document_rowid);211        try self.writeDocument(rowid, doc, true);212    }213214    pub fn updateSaturatedTranscript(215        self: *Database,216        rowid: i64,217        mtime_value: f64,218        size: i64,219    ) !void {220        std.debug.assert(rowid > 0);221        std.debug.assert(size >= 0);222        const bytes = (try self.documents.get(self.allocator, rowid)) orelse223            return error.KeyNotFound;224        defer self.allocator.free(bytes);225        const doc = try documentViewFromBytes(bytes);226        if (!isTranscriptKind(doc.kind) or227            !transcriptRecordsSaturate(doc.transcript_records, doc.content.len))228        {229            return error.InvalidRow;230        }231        const mtime = f64Bytes(mtime_value);232        const indexed_at = f64Bytes(nowSeconds());233        const values = [_]sql.RowValue{234            .{ .text = doc.source },235            .{ .text = doc.path },236            .{ .text = doc.kind },237            optionalText(doc.project),238            optionalText(doc.title),239            .{ .text = doc.content },240            optionalText(doc.metadata),241            .{ .blob = &mtime },242            .{ .integer = size },243            .{ .blob = &indexed_at },244            optionalBlob(doc.transcript_records),245        };246        try self.putValuesWithWalRetry(&self.documents, rowid, &values);247        self.initialized = true;248        try self.checkpointIfNeeded();249    }250251    pub fn upsertDocumentRow(self: *Database, doc: data.Document) !void {252        const rowid = try self.rowidForDocumentPath(doc.path);253        try self.writeDocument(rowid, doc, false);254    }255256    pub fn updateDocumentRow(self: *Database, rowid: i64, doc: data.Document) !void {257        std.debug.assert(rowid > 0);258        try self.writeDocument(rowid, doc, false);259    }260261    pub fn insertDocumentRowNew(self: *Database, doc: data.Document) !void {262        const rowid = try self.nextCachedRowid(&self.documents, documents_root, &self.next_document_rowid);263        try self.writeDocument(rowid, doc, false);264    }265266    pub fn insertDocumentRowNewSearch(self: *Database, doc: data.Document, batch: *SearchBatch) !void {267        const rowid = try self.nextCachedRowid(&self.documents, documents_root, &self.next_document_rowid);268        try self.putDocument(rowid, doc);269        try self.appendDocumentSearch(rowid, doc, batch);270    }271272    pub fn upsertDocumentRowSearch(self: *Database, doc: data.Document, batch: *SearchBatch) !void {273        const rowid = try self.rowidForDocumentPath(doc.path);274        try self.putDocument(rowid, doc);275        try self.appendDocumentSearch(rowid, doc, batch);276    }277278    pub fn updateDocumentRowSearch(self: *Database, rowid: i64, doc: data.Document, batch: *SearchBatch) !void {279        std.debug.assert(rowid > 0);280        try self.putDocument(rowid, doc);281        try self.appendDocumentSearch(rowid, doc, batch);282    }283284    fn writeDocument(self: *Database, rowid: i64, doc: data.Document, index_search: bool) !void {285        try self.putDocument(rowid, doc);286        if (!index_search) return;287        const text = try documentSearchText(self.allocator, doc.source, doc.path, doc.kind, doc.project, doc.title, doc.content);288        defer self.allocator.free(text);289        try self.putDocumentSearch(rowid, text);290    }291292    pub fn deletePath(self: *Database, path: []const u8) !void {293        const docs = try self.documentRowidsByPath(self.allocator, path);294        defer self.allocator.free(docs);295        for (docs) |rowid| {296            try self.deleteRowWithWalRetry(&self.documents, rowid);297            try self.checkpointIfNeeded();298            try self.deleteDocumentSearch(rowid);299        }300    }301302    pub fn beginBulk(self: *Database) !void {303        std.debug.assert(self.write_mode == .normal);304        try self.file.reserve(self.bulk_capacity);305        self.write_mode = .bulk;306    }307308    pub fn endBulk(self: *Database) !void {309        std.debug.assert(self.write_mode == .bulk);310        _ = try self.checkpointAndReopen(self.normal_capacity);311        self.write_mode = .normal;312    }313314    pub fn bulkCheckpoint(self: *Database) !void {315        std.debug.assert(self.write_mode == .bulk);316        _ = try self.checkpointAndReopen(self.bulk_capacity);317    }318319    fn rebuildCheckpointIfNeeded(self: *Database) !void {320        try self.rebuildCheckpointIfNeededAt(rebuild_checkpoint_wal_bytes);321    }322323    fn rebuildCheckpointIfNeededAt(self: *Database, wal_bytes: usize) !void {324        if (self.file.pager.walBytes().len >= wal_bytes) {325            _ = try self.checkpointAndReopen(self.bulk_capacity);326        }327    }328329    pub fn rebuildSearch(self: *Database) !void {330        try self.rebuildDocumentSearch();331    }332333    pub fn clearSearch(self: *Database) !void {334        try self.clearSearchWithWalRetry(&self.documents_search);335        try self.rebuildCheckpointIfNeeded();336    }337338    pub fn flushDocumentSearchBatch(self: *Database, batch: *SearchBatch) !void {339        if (try self.rebuildDocumentSearchIfCapabilitiesMissing()) {340            clearSearchDocumentBatch(self.allocator, batch);341            return;342        }343        flushSearchDocumentBatch(self, &self.documents_search, batch, rebuild_checkpoint_wal_bytes) catch |err| switch (err) {344            error.SearchIndexRebuildRequired => {345                clearSearchDocumentBatch(self.allocator, batch);346                try self.rebuildDocumentSearch();347            },348            else => return err,349        };350    }351352    pub fn deinitSearchBatch(self: *Database, batch: *SearchBatch) void {353        freeSearchDocumentBatch(self.allocator, batch);354    }355356    fn rebuildDocumentSearch(self: *Database) !void {357        try self.rebuildDocumentSearchWithPolicy(search_rebuild_document_batch_limit, rebuild_checkpoint_wal_bytes);358    }359360    fn rebuildDocumentSearchWithPolicy(self: *Database, batch_limit: usize, checkpoint_wal_bytes: usize) !void {361        std.debug.assert(batch_limit > 0);362        std.debug.assert(checkpoint_wal_bytes > 0);363        try self.clearSearchWithWalRetry(&self.documents_search);364        try self.rebuildCheckpointIfNeededAt(checkpoint_wal_bytes);365        var batch: std.ArrayList(sql.SearchDocument) = .empty;366        defer freeSearchDocumentBatch(self.allocator, &batch);367        if (!try self.rootExists(documents_root)) return;368369        var next_rowid: ?i64 = null;370        var complete = false;371        while (!complete) {372            var batch_bytes: usize = 0;373            complete = true;374            {375                var scan: sql.TableScan = undefined;376                try self.documents.scan(&scan, self.allocator, next_rowid, null);377                defer scan.deinit();378                while (try scan.next()) |entry| {379                    next_rowid = std.math.add(i64, entry.rowid, 1) catch return error.InvalidRow;380                    const doc = try documentFromBytes(self.allocator, entry.bytes);381                    defer freeDocumentRow(self.allocator, doc);382                    {383                        const text = try documentSearchText(self.allocator, doc.source, doc.path, doc.kind, doc.project, doc.title, doc.content);384                        errdefer self.allocator.free(text);385                        batch_bytes = std.math.add(usize, batch_bytes, text.len) catch std.math.maxInt(usize);386                        try batch.append(self.allocator, .{ .rowid = entry.rowid, .text = text });387                    }388                    if (batch.items.len >= batch_limit or batch_bytes >= search_rebuild_document_batch_bytes) {389                        complete = false;390                        break;391                    }392                }393            }394            try flushSearchDocumentBatch(self, &self.documents_search, &batch, checkpoint_wal_bytes);395        }396    }397398    pub fn saveDocumentSearchRefs(self: *Database, rows: []const data.SearchResult) !void {399        const row_count = std.math.cast(i64, rows.len) orelse400            return error.SearchReferenceCapacityExceeded;401        _ = std.math.add(i64, 1_000_000, row_count) catch402            return error.SearchReferenceCapacityExceeded;403        var count_buffer: [32]u8 = undefined;404        const count_text = try std.fmt.bufPrint(&count_buffer, "{d}", .{rows.len});405        try preflightSearchRef("documents", 0, count_text);406        for (rows, 1..) |row, ordinal| {407            try preflightSearchRef("documents", @intCast(ordinal), row.path);408        }409        var write = try self.space.beginWrite();410        defer write.deinit();411        var row_buffer: [search_ref_row_bytes]u8 = undefined;412        try putSearchRefIn(413            &self.search_refs,414            &write,415            &row_buffer,416            searchRefRowid("documents", 0),417            "documents",418            0,419            count_text,420        );421        for (rows, 1..) |row, ordinal| {422            try putSearchRefIn(423                &self.search_refs,424                &write,425                &row_buffer,426                searchRefRowid("documents", @intCast(ordinal)),427                "documents",428                @intCast(ordinal),429                row.path,430            );431        }432        _ = try write.commit(.{ .durability = .buffered });433        self.initialized = true;434    }435436    pub fn resolveSearchRef(self: *Database, allocator: std.mem.Allocator, scope: []const u8, ref: []const u8) !?[]u8 {437        if (!std.mem.startsWith(u8, ref, "@")) return null;438        if (!try self.rootExists(search_refs_root)) return null;439        const ordinal = std.fmt.parseInt(i64, ref[1..], 10) catch return null;440        const count_bytes = (try self.search_refs.get(allocator, searchRefRowid(scope, 0))) orelse return null;441        defer allocator.free(count_bytes);442        const count_row = try searchRefFromBytes(count_bytes);443        const count = std.fmt.parseInt(i64, count_row.target, 10) catch return null;444        if (ordinal <= 0 or ordinal > count) return null;445        const bytes = (try self.search_refs.get(allocator, searchRefRowid(scope, ordinal))) orelse return null;446        defer allocator.free(bytes);447        const row = try searchRefFromBytes(bytes);448        if (!std.mem.eql(u8, row.scope, scope) or row.ordinal != ordinal) return null;449        return try allocator.dupe(u8, row.target);450    }451452    pub fn search(453        self: *Database,454        storage: *search_mod.Storage,455        query: []const u8,456        filters: data.SearchFilters,457    ) !data.SearchPage {458        return try storage.execute(self, query, filters, searchInto);459    }460461    fn searchInto(462        self: *Database,463        allocator: std.mem.Allocator,464        storage: *search_mod.Storage,465        query: []const u8,466        filters: data.SearchFilters,467    ) !data.SearchPage {468        if (!try self.rootExists(documents_search_rows_root)) return .{ .rows = &.{}, .total = 0 };469        const native_limit = nativeDocumentSearchLimit(filters);470        var prepared = try self.documents_search.prepare(allocator, query);471        defer prepared.deinit(allocator);472        var results = try prepared.executeRanked(473            &self.documents_search,474            allocator,475            native_limit,476            .{},477        );478        defer results.deinit();479        var rows: std.ArrayList(data.SearchResult) = .empty;480        errdefer {481            freeSearchResults(allocator, rows.items);482            rows.deinit(allocator);483        }484        var total: usize = if (native_limit == 0 or filters.role != null) 0 else results.total;485        for (results.hits) |hit| {486            const bytes = (try storage.get(&self.documents, hit.rowid)) orelse continue;487            const doc = try documentViewFromBytes(bytes);488            if (!documentMatches(doc, filters)) continue;489            if (filters.role) |role| {490                const record_bytes = doc.transcript_records orelse continue;491                const records = try TranscriptRecordsView.init(record_bytes);492                var record_index: usize = 0;493                while (record_index < records.count()) : (record_index += 1) {494                    const record = try records.record(record_index, doc.content.len);495                    if (record.role != role or !transcriptRecordMatchesBounds(record, filters)) continue;496                    const content = doc.content[record.start..record.end];497                    if (!prepared.matches(content)) continue;498                    total += 1;499                    if (filters.limit > 0 and rows.items.len >= filters.limit) continue;500                    try rows.append(allocator, try searchResultFromDocument(allocator, doc, .{ .prepared = &prepared }, hit.rank, record));501                }502                continue;503            }504            if (native_limit == 0) total += 1;505            if (filters.limit > 0 and rows.items.len >= filters.limit) continue;506            try rows.append(allocator, try searchResultFromDocument(allocator, doc, .{ .prepared = &prepared }, hit.rank, null));507        }508        return .{ .rows = try rows.toOwnedSlice(allocator), .total = total };509    }510511    pub fn getDocument(self: *Database, allocator: std.mem.Allocator, path: []const u8) !?data.DocumentRow {512        if (!try self.rootExists(documents_root)) return null;513        var scan: sql.TableScan = undefined;514        try self.documents.scan(&scan, allocator, null, null);515        defer scan.deinit();516        while (try scan.next()) |entry| {517            const doc = try documentFromBytes(allocator, entry.bytes);518            if (std.mem.eql(u8, doc.path, path)) return doc;519            freeDocumentRow(allocator, doc);520        }521        return null;522    }523524    pub fn findDocument(self: *Database, allocator: std.mem.Allocator, fragment: []const u8) !?data.DocumentRow {525        if (try self.getDocument(allocator, fragment)) |doc| return doc;526        if (!try self.rootExists(documents_root)) return null;527        var scan: sql.TableScan = undefined;528        try self.documents.scan(&scan, allocator, null, null);529        defer scan.deinit();530        while (try scan.next()) |entry| {531            const doc = try documentFromBytes(allocator, entry.bytes);532            if (std.mem.indexOf(u8, doc.path, fragment) != null) return doc;533            freeDocumentRow(allocator, doc);534        }535        return null;536    }537538    pub fn hasDocuments(self: *Database) bool {539        return self.rootExists(documents_root) catch false;540    }541542    pub fn indexedAgeNanoseconds(self: *Database) ?i128 {543        const mtime = self.indexedMtimeNanoseconds() orelse return null;544        const age = sys.time.realNanoTimestamp() - mtime;545        return if (age < 0) 0 else age;546    }547548    fn indexedMtimeNanoseconds(self: *Database) ?i128 {549        const wal_path = walPath(self.allocator, self.path) catch return newestIndexMtimeNanoseconds(self.path, null);550        defer self.allocator.free(wal_path);551        return newestIndexMtimeNanoseconds(self.path, wal_path);552    }553554    pub fn stats(self: *Database, allocator: std.mem.Allocator) !data.Stats {555        const wal_path = try walPath(allocator, self.path);556        defer allocator.free(wal_path);557        if (!try self.rootExists(documents_root)) return .{558            .total = 0,559            .by_kind = &.{},560            .by_source = &.{},561            .total_content_bytes = 0,562            .db_bytes = fileSize(self.path) + fileSize(wal_path),563            .last_indexed = null,564        };565        var by_kind_map = std.StringHashMap(i64).init(allocator);566        defer deinitCountMap(allocator, &by_kind_map);567        var by_source_map = std.StringHashMap(i64).init(allocator);568        defer deinitCountMap(allocator, &by_source_map);569        var total: i64 = 0;570        var total_content_bytes: i64 = 0;571        var last_indexed: ?f64 = null;572        var scan: sql.TableScan = undefined;573        try self.documents.scan(&scan, allocator, null, null);574        defer scan.deinit();575        while (try scan.next()) |entry| {576            const doc = try documentFromBytes(allocator, entry.bytes);577            defer freeDocumentRow(allocator, doc);578            total += 1;579            total_content_bytes += doc.size;580            if (last_indexed == null or doc.indexed_at > last_indexed.?) last_indexed = doc.indexed_at;581            try incrementCount(allocator, &by_kind_map, doc.kind);582            try incrementCount(allocator, &by_source_map, doc.source);583        }584        return .{585            .total = total,586            .by_kind = try countsFromMap(allocator, &by_kind_map),587            .by_source = try countsFromMap(allocator, &by_source_map),588            .total_content_bytes = total_content_bytes,589            .db_bytes = fileSize(self.path) + fileSize(wal_path),590            .last_indexed = last_indexed,591        };592    }593594    pub fn optimize(self: *Database, rebuild_search: bool, vacuum: bool) !OptimizeReport {595        const wal_path = try walPath(self.allocator, self.path);596        defer self.allocator.free(wal_path);597        const before = fileSize(self.path) + fileSize(wal_path);598        if (rebuild_search) try self.rebuildSearch();599        const checkpoint_result = try nativeCheckpoint(self);600        if (vacuum) try self.file.flush();601        const after = fileSize(self.path) + fileSize(wal_path);602        return .{ .db_path = self.path, .rebuild_search = rebuild_search, .vacuum = vacuum, .before = before, .after = after, .checkpoint = checkpoint_result };603    }604605    fn putDocument(self: *Database, rowid: i64, doc: data.Document) !void {606        const mtime = f64Bytes(doc.mtime);607        const indexed_at = f64Bytes(nowSeconds());608        const transcript_records = if (doc.transcript_records) |records| try encodeTranscriptRecords(self.allocator, records) else null;609        defer if (transcript_records) |bytes| self.allocator.free(bytes);610        const values = [_]sql.RowValue{611            .{ .text = doc.source },612            .{ .text = doc.path },613            .{ .text = doc.kind },614            optionalText(doc.project),615            optionalText(doc.title),616            .{ .text = documentContent(doc.content) },617            optionalText(doc.metadata),618            .{ .blob = &mtime },619            .{ .integer = doc.size },620            .{ .blob = &indexed_at },621            optionalBlob(transcript_records),622        };623        try self.putValuesWithWalRetry(&self.documents, rowid, &values);624        if (self.next_document_rowid) |next| {625            if (rowid >= next) self.next_document_rowid = rowid + 1;626        }627        self.initialized = true;628        try self.checkpointIfNeeded();629    }630631    fn appendDocumentSearch(self: *Database, rowid: i64, doc: data.Document, batch: *SearchBatch) !void {632        const text = try documentSearchText(self.allocator, doc.source, doc.path, doc.kind, doc.project, doc.title, doc.content);633        errdefer self.allocator.free(text);634        try batch.append(self.allocator, .{ .rowid = rowid, .text = text });635    }636637    fn checkpointAndReopen(self: *Database, capacity: sql.PagerCapacity) !Checkpoint {638        const checkpoint_result = try nativeCheckpoint(self);639        try self.reopenFile(capacity);640        return checkpoint_result;641    }642643    fn reopenFile(self: *Database, capacity: sql.PagerCapacity) !void {644        var dir = try openDir(self.dir_path);645        defer dir.close(fs_io);646        self.file.deinit();647        self.file_live = false;648        self.file.* = try sql.FileDatabase.open(649            self.allocator,650            self.workspace,651            dir,652            .{653                .paths = .{ .database = self.db_name, .wal = self.wal_name },654                .header = databaseHeader(),655                .max_wal_bytes = self.max_wal_bytes,656            },657        );658        self.file_live = true;659        try self.file.reserve(capacity);660        self.space = try sql.Space.open(self.file, .{ .meta_page = meta_root, .roots = &roots, .reserved_page_max = root_max });661        self.documents = try self.space.rowidTable(documents_root);662        self.search_refs = try self.space.rowidTable(search_refs_root);663        self.documents_search = try openDocumentSearch(self.file);664    }665666    fn writeCapacity(self: *const Database) sql.PagerCapacity {667        return switch (self.write_mode) {668            .normal => self.normal_capacity,669            .bulk => self.bulk_capacity,670        };671    }672673    fn checkpointIfNeeded(self: *Database) !void {674        if (self.file.pager.walBytes().len >= self.checkpoint_wal_bytes) {675            _ = try self.checkpointAndReopen(self.writeCapacity());676        }677    }678679    fn checkpointForWalRetry(self: *Database) !void {680        _ = try self.checkpointAndReopen(self.writeCapacity());681    }682683    fn retryWalMutation(self: *Database, comptime mutation: anytype, args: anytype) !void {684        for (0..2) |attempt| {685            @call(.auto, mutation, args) catch |err| switch (err) {686                error.WalLimitExceeded => {687                    if (attempt != 0) return err;688                    try self.checkpointForWalRetry();689                    continue;690                },691                else => return err,692            };693            return;694        }695        unreachable;696    }697698    fn putValuesWithWalRetry(699        self: *Database,700        table: *sql.RowIdTable,701        rowid: i64,702        values: []const sql.RowValue,703    ) !void {704        try self.retryWalMutation(putValuesMutation, .{ table, self.allocator, rowid, values });705    }706707    fn deleteRowWithWalRetry(self: *Database, table: *sql.RowIdTable, rowid: i64) !void {708        try self.retryWalMutation(deleteRowMutation, .{ table, rowid });709    }710711    fn putDocumentSearchWithWalRetry(self: *Database, rowid: i64, value: []const u8) !void {712        try self.retryWalMutation(713            putSearchMutation,714            .{ &self.documents_search, self.allocator, rowid, value },715        );716    }717718    fn deleteDocumentSearchWithWalRetry(self: *Database, rowid: i64) !void {719        try self.retryWalMutation(720            deleteSearchMutation,721            .{ &self.documents_search, self.allocator, rowid },722        );723    }724725    fn clearSearchWithWalRetry(self: *Database, search_index: *sql.Search) !void {726        try self.retryWalMutation(clearSearchMutation, .{ search_index, self.allocator });727    }728729    fn loadSearchBatchWithWalRetry(730        self: *Database,731        search_index: *sql.Search,732        documents: []const sql.SearchDocument,733    ) !void {734        try self.retryWalMutation(735            loadSearchBatchMutation,736            .{ search_index, self.allocator, documents },737        );738    }739740    fn putDocumentSearch(self: *Database, rowid: i64, text: []const u8) !void {741        if (try self.rebuildDocumentSearchIfCapabilitiesMissing()) return;742        self.putDocumentSearchWithWalRetry(rowid, text) catch |err| switch (err) {743            error.SearchIndexRebuildRequired => return try self.rebuildDocumentSearch(),744            else => return err,745        };746        try self.checkpointIfNeeded();747    }748749    fn deleteDocumentSearch(self: *Database, rowid: i64) !void {750        _ = try self.rebuildDocumentSearchIfCapabilitiesMissing();751        self.deleteDocumentSearchWithWalRetry(rowid) catch |err| switch (err) {752            error.KeyNotFound => return,753            error.SearchIndexRebuildRequired => return try self.rebuildDocumentSearch(),754            else => return err,755        };756        try self.checkpointIfNeeded();757    }758759    fn rebuildDocumentSearchIfCapabilitiesMissing(self: *Database) !bool {760        if (!documents_search_index_phrase_pairs) return false;761        if ((try self.documents_search.capabilities(self.allocator)).phrase_pair_postings) return false;762        if (!try self.tableHasRows(documents_root, &self.documents)) return false;763        try self.rebuildDocumentSearch();764        return true;765    }766767    fn rowidForDocumentPath(self: *Database, path: []const u8) !i64 {768        if (!self.initialized or !try self.rootExists(documents_root)) return 1;769        var scan: sql.TableScan = undefined;770        try self.documents.scan(&scan, self.allocator, null, null);771        defer scan.deinit();772        var max_rowid: i64 = 0;773        while (try scan.next()) |entry| {774            max_rowid = @max(max_rowid, entry.rowid);775            const view = try sql.RowView.init(entry.bytes);776            if (std.mem.eql(u8, (try view.column(1)).text, path)) return entry.rowid;777        }778        return max_rowid + 1;779    }780781    fn nextCachedRowid(self: *Database, table: *const sql.RowIdTable, root_page: u32, cache: *?i64) !i64 {782        if (!self.initialized or !try self.rootExists(root_page)) {783            cache.* = 2;784            return 1;785        }786        if (cache.*) |rowid| {787            cache.* = rowid + 1;788            return rowid;789        }790        var scan: sql.TableScan = undefined;791        try table.scan(&scan, self.allocator, null, null);792        defer scan.deinit();793        var max_rowid: i64 = 0;794        while (try scan.next()) |entry| max_rowid = @max(max_rowid, entry.rowid);795        const rowid = max_rowid + 1;796        cache.* = rowid + 1;797        return rowid;798    }799800    fn documentRowidsByPath(self: *Database, allocator: std.mem.Allocator, path: []const u8) ![]i64 {801        if (!try self.rootExists(documents_root)) return &.{};802        var rowids: std.ArrayList(i64) = .empty;803        var scan: sql.TableScan = undefined;804        try self.documents.scan(&scan, allocator, null, null);805        defer scan.deinit();806        while (try scan.next()) |entry| {807            const view = try sql.RowView.init(entry.bytes);808            if (std.mem.eql(u8, (try view.column(1)).text, path)) try rowids.append(allocator, entry.rowid);809        }810        return try rowids.toOwnedSlice(allocator);811    }812813    fn documentByRowid(self: *Database, allocator: std.mem.Allocator, rowid: i64) !?data.DocumentRow {814        const bytes = (try self.documents.get(allocator, rowid)) orelse return null;815        defer allocator.free(bytes);816        return try documentFromBytes(allocator, bytes);817    }818819    fn tableHasRows(self: *Database, root_page: u32, table: *const sql.RowIdTable) !bool {820        if (!try self.rootExists(root_page)) return false;821        var scan: sql.TableScan = undefined;822        try table.scan(&scan, self.allocator, null, null);823        defer scan.deinit();824        return (try scan.next()) != null;825    }826827    fn rootExists(self: *Database, root_page: u32) !bool {828        var read = try self.file.beginRead();829        defer read.deinit();830        const snapshot = read.snapshot();831        var image: [sql.page.size]u8 = undefined;832        if (!try snapshot.copyPage(root_page, &image)) return false;833        _ = sql.page.kind(&image) catch return false;834        return true;835    }836};

1pub const Database = database.Database;