lib/sql/src/search/engine.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const sql = @import("../root.zig");
3 const hit_mod = @import("hit.zig");
4 const posting_mod = @import("posting.zig");
5 const query_mod = @import("query.zig");
6 const rank_mod = @import("rank.zig");
7 const snippet_mod = @import("snippet.zig");
8 const text_mod = @import("text.zig");
9 const token_mod = @import("token.zig");
10 const file = sql.file;
11 const index_mod = sql.index;
12 const key_mod = sql.key;
13 const page = sql.page;
14 const row = sql.row;
15 const space_mod = sql.space;
16 const table_mod = sql.table;
17 const trace = sql.trace;
18 const tree = sql.tree;
19 const wal = sql.wal;
20
21 const Allocator = std.mem.Allocator;
22
23 const SearchError = error{
24 EmptyDocument,
25 InvalidQuery,
26 InvalidSearchIndex,
27 SearchIndexRebuildRequired,
28 };
29
30 pub const Error =
31 space_mod.Error ||
32 table_mod.Error ||
33 index_mod.Error ||
34 row.Error ||
35 std.mem.Allocator.Error ||
36 SearchError;
37
38 pub const Options = struct {
39 meta_page: u32 = 1,
40 documents_root: u32 = 2,
41 terms_root: u32 = 3,
42 reserved_page_max: u32 = 0,
43 index_prefixes: bool = true,
44 index_phrase_pairs: bool = false,
45 omit_phrase_pair_singletons: bool = false,
46 field_names: []const []const u8 = &.{},
47 };
48
49 pub const Document = struct {
50 rowid: i64,
51 text: []const u8,
52 };
53
54 pub const Source = posting_mod.Source;
55
56 pub const Results = struct {
57 allocator: Allocator,
58 hits: []hit_mod.Hit,
59 total: usize,
60
61 pub fn deinit(self: *Results) void {
62 if (self.hits.len != 0) self.allocator.free(self.hits);
63 self.* = undefined;
64 }
65 };
66
67 pub const RankedResults = struct {
68 allocator: Allocator,
69 hits: []hit_mod.RankedHit,
70 total: usize,
71
72 pub fn deinit(self: *RankedResults) void {
73 if (self.hits.len != 0) self.allocator.free(self.hits);
74 self.* = undefined;
75 }
76 };
77
78 pub const Prepared = struct {
79 parsed: query_mod.Query,
80
81 pub fn deinit(self: *Prepared, allocator: Allocator) void {
82 self.parsed.deinit(allocator);
83 self.* = undefined;
84 }
85
86 pub fn execute(self: *const Prepared, search: anytype, allocator: Allocator, limit: usize) Error!Results {
87 const phase = trace.scope("search.prepared.execute");
88 defer phase.end();
89
90 return try search.queryParsed(allocator, self.parsed, limit);
91 }
92
93 pub fn executeScoreBand(self: *const Prepared, search: anytype, allocator: Allocator, limit: usize) Error!Results {
94 const phase = trace.scope("search.prepared.execute_score_band");
95 defer phase.end();
96
97 return try search.queryParsedScoreBand(allocator, self.parsed, limit);
98 }
99
100 pub fn executeRanked(self: *const Prepared, search: anytype, allocator: Allocator, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
101 const phase = trace.scope("search.prepared.execute_ranked");
102 defer phase.end();
103
104 return try search.queryParsedRanked(allocator, self.parsed, limit, options);
105 }
106
107 pub fn snippet(self: *const Prepared, allocator: Allocator, text: []const u8, options: snippet_mod.SnippetOptions) Error![]u8 {
108 return try snippet_mod.snippetParsed(allocator, text, self.parsed, options);
109 }
110
111 pub fn matches(self: *const Prepared, text: []const u8) bool {
112 for (self.parsed.clauses) |clause| {
113 if (text_mod.clauseScore(text, clause) != 0) return true;
114 }
115 return false;
116 }
117 };
118
119 pub const Search = struct {
120 space: space_mod.Space,
121 documents: table_mod.Table,
122 terms: index_mod.Index,
123 index_prefixes: bool,
124 index_phrase_pairs: bool,
125 omit_phrase_pair_singletons: bool,
126 field_names: []const []const u8,
127
128 pub fn open(database: *file.Database, options: Options) Error!Search {
129 const roots = [_]space_mod.RootSpec{
130 .{ .root_page = options.documents_root },
131 .{ .root_page = options.terms_root },
132 };
133 const space = try space_mod.Space.open(database, .{
134 .meta_page = options.meta_page,
135 .roots = &roots,
136 .reserved_page_max = options.reserved_page_max,
137 });
138 return .{
139 .space = space,
140 .documents = try space.rowidTable(options.documents_root),
141 .terms = try space.index(options.terms_root, &.{.{ .collation = .binary }}),
142 .index_prefixes = options.index_prefixes,
143 .index_phrase_pairs = options.index_phrase_pairs,
144 .omit_phrase_pair_singletons = options.omit_phrase_pair_singletons,
145 .field_names = options.field_names,
146 };
147 }
148
149 pub fn put(self: *Search, allocator: Allocator, rowid: i64, text: []const u8, options: file.CommitOptions) Error!file.Commit {
150 const phase = trace.scope("search.put");
151 defer phase.end();
152
153 var write = try self.space.beginWrite();
154 defer write.deinit();
155 try self.putIn(&write, allocator, rowid, text);
156 return try write.commit(options);
157 }
158
159 pub fn putForSource(
160 self: *Search,
161 allocator: Allocator,
162 rowid: i64,
163 text: []const u8,
164 expected: Source,
165 next: Source,
166 options: file.CommitOptions,
167 ) Error!?file.Commit {
168 const phase = trace.scope("search.put_for_source");
169 defer phase.end();
170
171 if (!try self.sourceMatches(allocator, expected)) return null;
172 var write = try self.space.beginWrite();
173 defer write.deinit();
174 try self.putIn(&write, allocator, rowid, text);
175 try self.putSourceIn(&write, next);
176 return try write.commit(options);
177 }
178
179 pub fn advanceSource(self: *Search, allocator: Allocator, expected: Source, next: Source, options: file.CommitOptions) Error!?file.Commit {
180 const phase = trace.scope("search.advance_source");
181 defer phase.end();
182
183 if (!try self.sourceMatches(allocator, expected)) return null;
184 var write = try self.space.beginWrite();
185 defer write.deinit();
186 try self.putSourceIn(&write, next);
187 return try write.commit(options);
188 }
189
190 pub fn putAll(self: *Search, allocator: Allocator, documents: []const Document, options: file.CommitOptions) Error!file.Commit {
191 const phase = trace.scope("search.put_all");
192 defer phase.end();
193
194 var write = try self.space.beginWrite();
195 defer write.deinit();
196 var stats = try self.corpusStatsOrEmpty(allocator);
197 try self.ensureRequestedCapabilitiesIn(&write, allocator, stats);
198 for (documents) |document| {
199 const indexed_text = token_mod.documentText(document.text);
200 const old = try self.documents.get(allocator, document.rowid);
201 defer if (old) |bytes| allocator.free(bytes);
202 if (old) |bytes| {
203 try self.deletePostingsIn(&write, allocator, document.rowid, try token_mod.textFromRow(bytes));
204 const old_length = try self.documentLength(allocator, document.rowid);
205 try self.deleteDocumentLengthIn(&write, document.rowid);
206 stats = try posting_mod.subtractDocumentStats(stats, old_length);
207 }
208 try self.putTextIn(&write, allocator, document.rowid, indexed_text);
209 try self.putPostingsIn(&write, allocator, document.rowid, indexed_text);
210 const length = token_mod.documentTokenLength(indexed_text);
211 try self.putDocumentLengthIn(&write, document.rowid, length);
212 stats = try posting_mod.addDocumentStats(stats, length);
213 }
214 try self.putCorpusStatsIn(&write, stats);
215 return try write.commit(options);
216 }
217
218 pub fn loadAll(self: *Search, allocator: Allocator, documents: []const Document, options: file.CommitOptions) Error!file.Commit {
219 const phase = trace.scope("search.load_all");
220 defer phase.end();
221
222 return try self.loadAllWithSource(allocator, documents, null, options);
223 }
224
225 pub fn loadAllForSource(self: *Search, allocator: Allocator, documents: []const Document, source_value: Source, options: file.CommitOptions) Error!file.Commit {
226 const phase = trace.scope("search.load_all_for_source");
227 defer phase.end();
228
229 return try self.loadAllWithSource(allocator, documents, source_value, options);
230 }
231
232 fn loadAllWithSource(self: *Search, allocator: Allocator, documents: []const Document, source_value: ?Source, options: file.CommitOptions) Error!file.Commit {
233 var postings = try self.collectPostings(allocator, documents);
234 var postings_live = true;
235 defer if (postings_live) postings.deinit(allocator);
236 std.mem.sort(posting_mod.Posting, postings.items, postings.keys, posting_mod.postingLess);
237
238 var write = try self.space.beginWrite();
239 defer write.deinit();
240 try self.clearIn(&write, allocator);
241 try self.putRequestedCapabilitiesIn(&write);
242 try self.putSortedPostingsIn(&write, allocator, postings.items, postings.keys);
243 postings.deinit(allocator);
244 postings_live = false;
245 var stats: posting_mod.CorpusStats = .{ .documents = 0, .total_tokens = 0 };
246 {
247 var lengths: std.ArrayList(posting_mod.DocumentLength) = .empty;
248 defer lengths.deinit(allocator);
249 for (documents) |document| {
250 const indexed_text = token_mod.documentText(document.text);
251 try self.putTextIn(&write, allocator, document.rowid, indexed_text);
252 const length = token_mod.documentTokenLength(indexed_text);
253 try lengths.append(allocator, .{ .rowid = document.rowid, .length = length });
254 stats = try posting_mod.addDocumentStats(stats, length);
255 }
256 try self.putSortedDocumentLengthsIn(&write, allocator, lengths.items);
257 }
258 try self.putCorpusStatsIn(&write, stats);
259 if (source_value) |value| try self.putSourceIn(&write, value);
260 return try write.commit(options);
261 }
262
263 pub fn loadAllNew(self: *Search, allocator: Allocator, documents: []const Document, options: file.CommitOptions) Error!file.Commit {
264 const phase = trace.scope("search.load_all_new");
265 defer phase.end();
266
267 var postings = try self.collectPostings(allocator, documents);
268 var postings_live = true;
269 defer if (postings_live) postings.deinit(allocator);
270 std.mem.sort(posting_mod.Posting, postings.items, postings.keys, posting_mod.postingLess);
271
272 var write = try self.space.beginWrite();
273 defer write.deinit();
274 var stats = try self.corpusStatsOrEmpty(allocator);
275 try self.ensureRequestedCapabilitiesIn(&write, allocator, stats);
276 try self.putSortedPostingsIn(&write, allocator, postings.items, postings.keys);
277 postings.deinit(allocator);
278 postings_live = false;
279 var lengths: std.ArrayList(posting_mod.DocumentLength) = .empty;
280 defer lengths.deinit(allocator);
281 for (documents) |document| {
282 const indexed_text = token_mod.documentText(document.text);
283 try self.putTextIn(&write, allocator, document.rowid, indexed_text);
284 const length = token_mod.documentTokenLength(indexed_text);
285 try lengths.append(allocator, .{ .rowid = document.rowid, .length = length });
286 stats = try posting_mod.addDocumentStats(stats, length);
287 }
288 try self.putSortedDocumentLengthsIn(&write, allocator, lengths.items);
289 try self.putCorpusStatsIn(&write, stats);
290 return try write.commit(options);
291 }
292
293 pub fn clear(self: *Search, allocator: Allocator, options: file.CommitOptions) Error!file.Commit {
294 const phase = trace.scope("search.clear");
295 defer phase.end();
296
297 var write = try self.space.beginWrite();
298 defer write.deinit();
299 try self.clearIn(&write, allocator);
300 try self.putRequestedCapabilitiesIn(&write);
301 return try write.commit(options);
302 }
303
304 pub fn delete(self: *Search, allocator: Allocator, rowid: i64, options: file.CommitOptions) Error!file.Commit {
305 const phase = trace.scope("search.delete");
306 defer phase.end();
307
308 const old = (try self.documents.get(allocator, rowid)) orelse return error.KeyNotFound;
309 defer allocator.free(old);
310 var write = try self.space.beginWrite();
311 defer write.deinit();
312 var stats = try self.corpusStatsOrEmpty(allocator);
313 try self.ensureRequestedCapabilitiesIn(&write, allocator, stats);
314 try self.deletePostingsIn(&write, allocator, rowid, try token_mod.textFromRow(old));
315 const old_length = try self.documentLength(allocator, rowid);
316 try self.deleteDocumentLengthIn(&write, rowid);
317 stats = try posting_mod.subtractDocumentStats(stats, old_length);
318 try self.putCorpusStatsIn(&write, stats);
319 try self.documents.deleteIn(&write, rowid);
320 return try write.commit(options);
321 }
322
323 fn getDocumentText(self: anytype, allocator: Allocator, rowid: i64) Error!?text_mod.DocumentText {
324 const bytes = (try self.documents.get(allocator, rowid)) orelse return null;
325 errdefer allocator.free(bytes);
326 return .{
327 .bytes = bytes,
328 .text = try token_mod.textFromRow(bytes),
329 };
330 }
331
332 pub fn getText(self: anytype, allocator: Allocator, rowid: i64) Error!?[]u8 {
333 var document = (try self.getDocumentText(allocator, rowid)) orelse return null;
334 defer document.deinit(allocator);
335 return try allocator.dupe(u8, document.text);
336 }
337
338 pub fn capabilities(self: anytype, allocator: Allocator) Error!posting_mod.Capabilities {
339 return try self.indexCapabilities(allocator);
340 }
341
342 pub fn indexedSource(self: anytype, allocator: Allocator) Error!?Source {
343 var lookup: index_mod.Scan = undefined;
344 try self.terms.lookupPayloads(&lookup, allocator, &posting_mod.sourceValues());
345 defer lookup.deinit();
346 const entry = (try lookup.next()) orelse return null;
347 if (entry.rowid != posting_mod.internal_meta_rowid) return error.InvalidSearchIndex;
348 const value = try posting_mod.sourcePayloadValue(entry.payload);
349 if (try lookup.next() != null) return error.InvalidSearchIndex;
350 return value;
351 }
352
353 pub fn sourceMatches(self: anytype, allocator: Allocator, expected: Source) Error!bool {
354 const current = (try self.indexedSource(allocator)) orelse return false;
355 return current.same(expected);
356 }
357
358 pub fn sourceSchemaMatches(self: anytype, allocator: Allocator, schema: u64) Error!bool {
359 const current = (try self.indexedSource(allocator)) orelse return false;
360 return current.schema == schema;
361 }
362
363 pub fn postingStats(self: anytype, allocator: Allocator) Error!posting_mod.PostingStats {
364 var scan: index_mod.Scan = undefined;
365 try self.terms.scanPayloads(&scan, allocator, null, null);
366 defer scan.deinit();
367 var stats: posting_mod.PostingStats = .{};
368 var previous_key: [posting_mod.phrase_pair_posting_key_max_bytes]u8 = undefined;
369 var previous_key_len: usize = 0;
370 var have_previous_key = false;
371 var values: [1]row.Value = undefined;
372 var scratch: [page.size]u8 = undefined;
373 while (try scan.next()) |entry| {
374 stats.term_index_entries += 1;
375 stats.payload_bytes += entry.payload.len;
376 const decoded = try key_mod.decodeIndex(&values, &scratch, entry.key);
377 if (decoded.values.len != 1) return error.InvalidSearchIndex;
378 const key_text = switch (decoded.values[0]) {
379 .text => |text| text,
380 else => return error.InvalidSearchIndex,
381 };
382 if (std.mem.eql(u8, key_text, posting_mod.document_length_key)) {
383 stats.document_length_entries += 1;
384 if (posting_mod.lengthSegmentPayloadCount(entry.payload)) |count| {
385 stats.document_length_segment_entries += 1;
386 stats.document_length_segment_postings += count;
387 } else {
388 _ = try posting_mod.documentLengthPayloadMaybe(entry.payload);
389 stats.document_length_direct_entries += 1;
390 }
391 continue;
392 }
393 if (std.mem.eql(u8, key_text, posting_mod.corpus_stats_key) or
394 std.mem.eql(u8, key_text, posting_mod.capabilities_key) or
395 std.mem.eql(u8, key_text, posting_mod.source_key))
396 {
397 stats.metadata_entries += 1;
398 continue;
399 }
400 if (key_text.len == 0 or key_text.len > previous_key.len) return error.InvalidSearchIndex;
401 if (!have_previous_key or !std.mem.eql(u8, previous_key[0..previous_key_len], key_text)) {
402 stats.posting_term_keys += 1;
403 stats.posting_term_key_text_bytes += key_text.len;
404 switch (key_text[0]) {
405 '=' => {
406 stats.exact_term_keys += 1;
407 stats.exact_term_key_text_bytes += key_text.len;
408 },
409 '*' => {
410 stats.prefix_term_keys += 1;
411 stats.prefix_term_key_text_bytes += key_text.len;
412 },
413 posting_mod.phrase_pair_posting_kind => {
414 stats.phrase_pair_term_keys += 1;
415 stats.phrase_pair_term_key_text_bytes += key_text.len;
416 },
417 else => return error.InvalidSearchIndex,
418 }
419 @memcpy(previous_key[0..key_text.len], key_text);
420 previous_key_len = key_text.len;
421 have_previous_key = true;
422 }
423 stats.posting_entries += 1;
424 stats.posting_payload_bytes += entry.payload.len;
425 stats.posting_key_text_bytes += key_text.len;
426 const phrase_pair = key_text[0] == posting_mod.phrase_pair_posting_kind;
427 switch (key_text[0]) {
428 '=' => {
429 stats.exact_posting_key_text_bytes += key_text.len;
430 stats.exact_posting_payload_bytes += entry.payload.len;
431 },
432 '*' => {
433 stats.prefix_posting_key_text_bytes += key_text.len;
434 stats.prefix_posting_payload_bytes += entry.payload.len;
435 },
436 posting_mod.phrase_pair_posting_kind => {
437 stats.phrase_pair_posting_key_text_bytes += key_text.len;
438 stats.phrase_pair_posting_payload_bytes += entry.payload.len;
439 },
440 else => return error.InvalidSearchIndex,
441 }
442 if (posting_mod.segmentPayloadCount(entry.payload)) |count| {
443 stats.segment_entries += 1;
444 stats.segment_postings += count;
445 if (phrase_pair) {
446 stats.phrase_pair_segment_entries += 1;
447 stats.phrase_pair_segment_postings += count;
448 }
449 } else {
450 const count = try posting_mod.postingPayloadCount(entry.payload);
451 if (count == 0) {
452 stats.direct_posting_tombstones += 1;
453 if (phrase_pair) stats.phrase_pair_direct_posting_tombstones += 1;
454 } else {
455 stats.direct_posting_entries += 1;
456 if (phrase_pair) stats.phrase_pair_direct_posting_entries += 1;
457 }
458 }
459 }
460 return stats;
461 }
462
463 fn putIn(self: *Search, write: *tree.Write, allocator: Allocator, rowid: i64, text: []const u8) Error!void {
464 const indexed_text = token_mod.documentText(text);
465 const old = try self.documents.get(allocator, rowid);
466 defer if (old) |bytes| allocator.free(bytes);
467 var stats = try self.corpusStatsOrEmpty(allocator);
468 try self.ensureRequestedCapabilitiesIn(write, allocator, stats);
469 if (old) |bytes| {
470 const old_text = try token_mod.textFromRow(bytes);
471 if (std.mem.eql(u8, old_text, indexed_text)) return;
472 const old_length = try self.documentLength(allocator, rowid);
473 try self.replacePostingsIn(write, allocator, rowid, old_text, indexed_text);
474 try self.putTextIn(write, allocator, rowid, indexed_text);
475 const length = token_mod.documentTokenLength(indexed_text);
476 if (old_length != length) {
477 try self.putDocumentLengthIn(write, rowid, length);
478 stats = try posting_mod.subtractDocumentStats(stats, old_length);
479 stats = try posting_mod.addDocumentStats(stats, length);
480 try self.putCorpusStatsIn(write, stats);
481 }
482 return;
483 }
484 try self.putTextIn(write, allocator, rowid, indexed_text);
485 try self.putPostingsIn(write, allocator, rowid, indexed_text);
486 const length = token_mod.documentTokenLength(indexed_text);
487 try self.putDocumentLengthIn(write, rowid, length);
488 stats = try posting_mod.addDocumentStats(stats, length);
489 try self.putCorpusStatsIn(write, stats);
490 }
491
492 fn requestedCapabilities(self: anytype) posting_mod.Capabilities {
493 return .{
494 .phrase_pair_postings = self.index_phrase_pairs,
495 .phrase_pair_singletons_omitted = self.index_phrase_pairs and self.omit_phrase_pair_singletons,
496 };
497 }
498
499 fn indexCapabilities(self: anytype, allocator: Allocator) Error!posting_mod.Capabilities {
500 var lookup: index_mod.Scan = undefined;
501 try self.terms.lookupPayloads(&lookup, allocator, &posting_mod.capabilitiesValues());
502 defer lookup.deinit();
503 const entry = (try lookup.next()) orelse return .{};
504 if (entry.rowid != posting_mod.internal_meta_rowid) return error.InvalidSearchIndex;
505 const value = try posting_mod.capabilitiesPayloadValue(entry.payload);
506 if (try lookup.next() != null) return error.InvalidSearchIndex;
507 return value;
508 }
509
510 fn canUsePhrasePairPostings(self: anytype, allocator: Allocator) Error!bool {
511 if (!self.index_phrase_pairs) return false;
512 return (try self.indexCapabilities(allocator)).phrase_pair_postings;
513 }
514
515 fn phrasePairPostingsArePartial(self: anytype, allocator: Allocator) Error!bool {
516 if (!self.index_phrase_pairs) return false;
517 return (try self.indexCapabilities(allocator)).phrase_pair_singletons_omitted;
518 }
519
520 fn ensureRequestedCapabilitiesIn(self: *Search, write: *tree.Write, allocator: Allocator, stats: posting_mod.CorpusStats) Error!void {
521 const requested = self.requestedCapabilities();
522 if (posting_mod.capabilitiesEmpty(requested)) return;
523 const current = try self.indexCapabilities(allocator);
524 if (posting_mod.capabilitiesInclude(current, requested)) return;
525 if (stats.documents != 0) return error.SearchIndexRebuildRequired;
526 try self.putCapabilitiesIn(write, posting_mod.mergeCapabilities(current, requested));
527 }
528
529 fn putRequestedCapabilitiesIn(self: *Search, write: *tree.Write) Error!void {
530 const requested = self.requestedCapabilities();
531 if (posting_mod.capabilitiesEmpty(requested)) return;
532 try self.putCapabilitiesIn(write, requested);
533 }
534
535 fn putCapabilitiesIn(self: *Search, write: *tree.Write, capabilities_value: posting_mod.Capabilities) Error!void {
536 var payload: [posting_mod.capabilities_payload_size]u8 = undefined;
537 try self.terms.putPayloadIn(write, posting_mod.internal_meta_rowid, &posting_mod.capabilitiesValues(), try posting_mod.capabilitiesPayload(&payload, capabilities_value));
538 }
539
540 fn putSourceIn(self: *Search, write: *tree.Write, source_value: Source) Error!void {
541 var payload: [posting_mod.source_payload_size]u8 = undefined;
542 try self.terms.putPayloadIn(write, posting_mod.internal_meta_rowid, &posting_mod.sourceValues(), try posting_mod.sourcePayload(&payload, source_value));
543 }
544
545 pub fn query(self: anytype, allocator: Allocator, text: []const u8, limit: usize) Error!Results {
546 const phase = trace.scope("search.query");
547 defer phase.end();
548
549 var parsed = try query_mod.parseQuery(allocator, text, self.field_names);
550 defer parsed.deinit(allocator);
551 return try self.queryParsed(allocator, parsed, limit);
552 }
553
554 pub fn queryScoreBand(self: anytype, allocator: Allocator, text: []const u8, limit: usize) Error!Results {
555 const phase = trace.scope("search.query_score_band");
556 defer phase.end();
557
558 var parsed = try query_mod.parseQuery(allocator, text, self.field_names);
559 defer parsed.deinit(allocator);
560 return try self.queryParsedScoreBand(allocator, parsed, limit);
561 }
562
563 pub fn queryRanked(self: anytype, allocator: Allocator, text: []const u8, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
564 const phase = trace.scope("search.query_ranked");
565 defer phase.end();
566
567 var parsed = try query_mod.parseQuery(allocator, text, self.field_names);
568 defer parsed.deinit(allocator);
569 return try self.queryParsedRanked(allocator, parsed, limit, options);
570 }
571
572 pub fn prepare(self: anytype, allocator: Allocator, text: []const u8) Error!Prepared {
573 const phase = trace.scope("search.prepare");
574 defer phase.end();
575
576 return .{ .parsed = try query_mod.parseQuery(allocator, text, self.field_names) };
577 }
578
579 fn queryParsed(self: anytype, allocator: Allocator, parsed: query_mod.Query, limit: usize) Error!Results {
580 if (parsed.clauses.len == 0) return .{ .allocator = allocator, .hits = &.{}, .total = 0 };
581
582 if (parsed.clauses.len == 1) {
583 if (limit != 0) {
584 if (try self.clauseBoundedIndexHits(allocator, parsed.clauses[0], limit)) |bounded_hits| {
585 std.mem.sort(hit_mod.Hit, bounded_hits.hits, {}, hit_mod.hitLess);
586 return .{ .allocator = allocator, .hits = bounded_hits.hits, .total = bounded_hits.total };
587 }
588 }
589 if (try self.clauseIndexHits(allocator, parsed.clauses[0])) |index_hits| {
590 const total = index_hits.len;
591 return .{ .allocator = allocator, .hits = try hit_mod.sortAndLimitOwnedHits(allocator, index_hits, limit), .total = total };
592 }
593 }
594
595 if (limit != 0) {
596 if (try self.queryBoundedExactOrHits(allocator, parsed, limit)) |bounded_hits| {
597 std.mem.sort(hit_mod.Hit, bounded_hits.hits, {}, hit_mod.hitLess);
598 return .{ .allocator = allocator, .hits = bounded_hits.hits, .total = bounded_hits.total };
599 }
600 }
601
602 var hits: []hit_mod.Hit = &.{};
603 var have_hits = false;
604 errdefer if (have_hits) allocator.free(hits);
605 for (parsed.clauses) |clause| {
606 const clause_hits = (try self.clauseIndexHits(allocator, clause)) orelse try self.scoredClauseHits(allocator, clause);
607 var clause_hits_owned = true;
608 errdefer if (clause_hits_owned) allocator.free(clause_hits);
609 if (!have_hits) {
610 hits = clause_hits;
611 have_hits = true;
612 clause_hits_owned = false;
613 continue;
614 }
615 const merged = try hit_mod.unionHits(allocator, hits, clause_hits);
616 allocator.free(hits);
617 allocator.free(clause_hits);
618 clause_hits_owned = false;
619 hits = merged;
620 }
621 const total = hits.len;
622 const owned = try hit_mod.sortAndLimitOwnedHits(allocator, hits, limit);
623 return .{ .allocator = allocator, .hits = owned, .total = total };
624 }
625
626 fn queryParsedScoreBand(self: anytype, allocator: Allocator, parsed: query_mod.Query, limit: usize) Error!Results {
627 if (limit == 0) return try self.queryParsed(allocator, parsed, 0);
628 if (parsed.clauses.len == 0) return .{ .allocator = allocator, .hits = &.{}, .total = 0 };
629
630 if (parsed.clauses.len == 1) {
631 if (try self.clauseScoreBandIndexHits(allocator, parsed.clauses[0], limit)) |band_hits| {
632 std.mem.sort(hit_mod.Hit, band_hits.hits, {}, hit_mod.hitLess);
633 return .{ .allocator = allocator, .hits = band_hits.hits, .total = band_hits.total };
634 }
635 if (try self.clauseIndexHits(allocator, parsed.clauses[0])) |index_hits| {
636 const total = index_hits.len;
637 return .{ .allocator = allocator, .hits = try hit_mod.sortAndScoreBandOwnedHits(allocator, index_hits, limit), .total = total };
638 }
639 }
640
641 if (try self.queryScoreBandExactOrHits(allocator, parsed, limit)) |band_hits| {
642 std.mem.sort(hit_mod.Hit, band_hits.hits, {}, hit_mod.hitLess);
643 return .{ .allocator = allocator, .hits = band_hits.hits, .total = band_hits.total };
644 }
645
646 var full = try self.queryParsed(allocator, parsed, 0);
647 errdefer full.deinit();
648 full.hits = try hit_mod.sortAndScoreBandOwnedHits(allocator, full.hits, limit);
649 return full;
650 }
651
652 fn queryParsedRanked(self: anytype, allocator: Allocator, parsed: query_mod.Query, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
653 if (try self.queryParsedRankedExactOr(allocator, parsed, limit, options)) |ranked| return ranked;
654
655 if (query_mod.singlePositivePrefixRankingAtom(parsed)) |atom| {
656 return try self.rankPrefixAtom(allocator, atom, limit, options);
657 }
658
659 if (query_mod.singlePositivePhrasePairRankingAtom(parsed)) |atom| {
660 return try self.rankPhrasePairAtom(allocator, atom, limit, options);
661 }
662
663 const rows = try self.queryParsedCandidateRows(allocator, parsed);
664 defer if (rows.len != 0) allocator.free(rows);
665 if (rows.len == 0) return .{ .allocator = allocator, .hits = &.{}, .total = 0 };
666
667 const stats = try self.corpusStats(allocator);
668 if (query_mod.singlePositiveNonExactRankingAtom(parsed)) |atom| {
669 return try self.rankSingleAtomCandidateRows(allocator, rows, atom, stats, limit, options);
670 }
671 const terms = try self.collectRankingTerms(allocator, parsed, stats.documents);
672 defer rank_mod.deinitRankingTerms(allocator, terms);
673 return try self.rankCandidateRows(allocator, rows, terms, stats, limit, options);
674 }
675
676 fn queryParsedCandidateRows(self: anytype, allocator: Allocator, parsed: query_mod.Query) Error![]i64 {
677 var rows: []i64 = &.{};
678 var have_rows = false;
679 errdefer if (have_rows) allocator.free(rows);
680 for (parsed.clauses) |clause| {
681 const clause_rows = try self.clauseCandidates(allocator, clause);
682 var clause_rows_owned = true;
683 errdefer if (clause_rows_owned) allocator.free(clause_rows);
684 if (!have_rows) {
685 rows = clause_rows;
686 have_rows = true;
687 clause_rows_owned = false;
688 continue;
689 }
690 const merged = try hit_mod.unionRows(allocator, rows, clause_rows);
691 allocator.free(rows);
692 allocator.free(clause_rows);
693 clause_rows_owned = false;
694 rows = merged;
695 }
696 return if (have_rows) rows else &.{};
697 }
698
699 fn rankSingleAtomCandidateRows(self: anytype, allocator: Allocator, rows: []const i64, atom: *const query_mod.Atom, stats: posting_mod.CorpusStats, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
700 var lengths = try rank_mod.DocumentLengthCursor.init(self, allocator);
701 defer lengths.deinit();
702
703 var candidates: std.ArrayList(rank_mod.SingleAtomCandidate) = .empty;
704 defer candidates.deinit(allocator);
705
706 var document_frequency: usize = 0;
707 if (text_mod.useDocumentTextScanRows(rows)) {
708 var documents: text_mod.DocumentTextCursor = undefined;
709 try documents.init(self, allocator, rows[0]);
710 defer documents.deinit();
711 for (rows) |rowid| {
712 const frequency = (try documents.atomFrequencyFor(rowid, atom.*)) orelse continue;
713 try rank_mod.appendSingleAtomCandidate(allocator, &candidates, &lengths, &document_frequency, rowid, frequency);
714 }
715 } else {
716 for (rows) |rowid| {
717 var document = (try self.getDocumentText(allocator, rowid)) orelse continue;
718 const frequency = text_mod.atomFrequency(document.text, atom.*);
719 document.deinit(allocator);
720 try rank_mod.appendSingleAtomCandidate(allocator, &candidates, &lengths, &document_frequency, rowid, frequency);
721 }
722 }
723
724 const idf = try rank_mod.bm25Idf(stats.documents, document_frequency);
725 var hits: std.ArrayList(hit_mod.RankedHit) = .empty;
726 errdefer hits.deinit(allocator);
727 for (candidates.items) |candidate| {
728 try hit_mod.appendRankedHit(allocator, &hits, limit, .{
729 .rowid = candidate.rowid,
730 .rank = -rank_mod.bm25Score(idf, candidate.frequency, candidate.length, stats, options),
731 });
732 }
733
734 std.mem.sort(hit_mod.RankedHit, hits.items, {}, hit_mod.rankedHitLess);
735 return .{
736 .allocator = allocator,
737 .hits = try hits.toOwnedSlice(allocator),
738 .total = document_frequency,
739 };
740 }
741
742 fn rankPrefixAtom(self: anytype, allocator: Allocator, atom: *const query_mod.Atom, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
743 const hits = try self.prefixHits(allocator, atom.tokens[0]);
744 defer if (hits.len != 0) allocator.free(hits);
745 if (hits.len == 0) return .{ .allocator = allocator, .hits = &.{}, .total = 0 };
746
747 const stats = try self.corpusStats(allocator);
748 const idf = try rank_mod.bm25Idf(stats.documents, hits.len);
749 var lengths = try rank_mod.DocumentLengthCursor.init(self, allocator);
750 defer lengths.deinit();
751
752 var ranked: std.ArrayList(hit_mod.RankedHit) = .empty;
753 errdefer ranked.deinit(allocator);
754 if (text_mod.useDocumentTextScanHits(hits)) {
755 var documents: text_mod.DocumentTextCursor = undefined;
756 try documents.init(self, allocator, hits[0].rowid);
757 defer documents.deinit();
758 for (hits) |hit| {
759 const frequency = (try documents.atomFrequencyFor(hit.rowid, atom.*)) orelse continue;
760 try rank_mod.appendPrefixRankedHit(allocator, &ranked, &lengths, hit.rowid, frequency, idf, stats, options, limit);
761 }
762 } else {
763 for (hits) |hit| {
764 var document = (try self.getDocumentText(allocator, hit.rowid)) orelse continue;
765 const frequency = text_mod.atomFrequency(document.text, atom.*);
766 document.deinit(allocator);
767 try rank_mod.appendPrefixRankedHit(allocator, &ranked, &lengths, hit.rowid, frequency, idf, stats, options, limit);
768 }
769 }
770
771 std.mem.sort(hit_mod.RankedHit, ranked.items, {}, hit_mod.rankedHitLess);
772 return .{
773 .allocator = allocator,
774 .hits = try ranked.toOwnedSlice(allocator),
775 .total = hits.len,
776 };
777 }
778
779 fn rankPhrasePairAtom(self: anytype, allocator: Allocator, atom: *const query_mod.Atom, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
780 std.debug.assert(query_mod.isPhrasePairAtom(atom.*));
781 const hits = if (try self.canUsePhrasePairPostings(allocator) and !try self.phrasePairPostingsArePartial(allocator))
782 try self.phrasePairHits(allocator, atom.tokens[0], atom.tokens[1])
783 else
784 try self.phrasePairVerifiedHits(allocator, atom.*, 1);
785 defer if (hits.len != 0) allocator.free(hits);
786 if (hits.len == 0) return .{ .allocator = allocator, .hits = &.{}, .total = 0 };
787
788 const stats = try self.corpusStats(allocator);
789 const idf = try rank_mod.bm25Idf(stats.documents, hits.len);
790 var lengths = try rank_mod.DocumentLengthCursor.init(self, allocator);
791 defer lengths.deinit();
792
793 var ranked: std.ArrayList(hit_mod.RankedHit) = .empty;
794 errdefer ranked.deinit(allocator);
795 for (hits) |hit| {
796 try hit_mod.appendRankedHit(allocator, &ranked, limit, .{
797 .rowid = hit.rowid,
798 .rank = -rank_mod.bm25Score(idf, hit.score, try lengths.lengthFor(hit.rowid), stats, options),
799 });
800 }
801
802 std.mem.sort(hit_mod.RankedHit, ranked.items, {}, hit_mod.rankedHitLess);
803 return .{
804 .allocator = allocator,
805 .hits = try ranked.toOwnedSlice(allocator),
806 .total = hits.len,
807 };
808 }
809
810 fn queryParsedRankedExactOr(self: anytype, allocator: Allocator, parsed: query_mod.Query, limit: usize, options: rank_mod.RankingOptions) Error!?RankedResults {
811 if (parsed.clauses.len == 0) return .{ .allocator = allocator, .hits = &.{}, .total = 0 };
812 for (parsed.clauses) |clause| _ = query_mod.exactClauseTerm(clause) orelse return null;
813
814 const stats = try self.corpusStats(allocator);
815 const cursors = try allocator.alloc(rank_mod.ExactRankingCursor, parsed.clauses.len);
816 var cursor_count: usize = 0;
817 defer {
818 for (cursors[0..cursor_count]) |*cursor| cursor.deinit();
819 allocator.free(cursors);
820 }
821 for (parsed.clauses) |clause| {
822 const term_text = query_mod.exactClauseTerm(clause).?;
823 cursors[cursor_count] = .{
824 .cursor = try rank_mod.ExactTermCursor.init(self, allocator, term_text),
825 .idf = 0.0,
826 };
827 cursor_count += 1;
828 const document_frequency = try cursors[cursor_count - 1].cursor.documentFrequency();
829 cursors[cursor_count - 1].idf = try rank_mod.bm25Idf(stats.documents, document_frequency);
830 try cursors[cursor_count - 1].cursor.advance();
831 }
832
833 var lengths = try rank_mod.DocumentLengthCursor.init(self, allocator);
834 defer lengths.deinit();
835
836 var hits: std.ArrayList(hit_mod.RankedHit) = .empty;
837 errdefer hits.deinit(allocator);
838 var total: usize = 0;
839 while (try rank_mod.nextExactRankedOrHit(cursors[0..cursor_count], &lengths, stats, options)) |hit| {
840 total += 1;
841 try hit_mod.appendRankedHit(allocator, &hits, limit, hit);
842 }
843 std.mem.sort(hit_mod.RankedHit, hits.items, {}, hit_mod.rankedHitLess);
844 return .{
845 .allocator = allocator,
846 .hits = try hits.toOwnedSlice(allocator),
847 .total = total,
848 };
849 }
850
851 fn rankCandidateRows(self: anytype, allocator: Allocator, rows: []const i64, terms: []const rank_mod.RankingTerm, stats: posting_mod.CorpusStats, limit: usize, options: rank_mod.RankingOptions) Error!RankedResults {
852 var lengths = try rank_mod.DocumentLengthCursor.init(self, allocator);
853 defer lengths.deinit();
854
855 var hits: std.ArrayList(hit_mod.RankedHit) = .empty;
856 errdefer hits.deinit(allocator);
857
858 var document_text: ?text_mod.DocumentText = null;
859 defer text_mod.deinitDocumentText(allocator, &document_text);
860 var document_text_rowid: ?i64 = null;
861
862 var total: usize = 0;
863 for (rows) |rowid| {
864 const length = try lengths.lengthFor(rowid);
865 var score: f64 = 0.0;
866 for (terms) |term| {
867 const frequency = if (term.posting_hits.len != 0)
868 hit_mod.postingHitFrequency(term.posting_hits, rowid)
869 else frequency: {
870 if (document_text_rowid == null or document_text_rowid.? != rowid) {
871 text_mod.deinitDocumentText(allocator, &document_text);
872 document_text_rowid = null;
873 document_text = (try self.getDocumentText(allocator, rowid)) orelse break :frequency 0;
874 document_text_rowid = rowid;
875 }
876 break :frequency text_mod.atomFrequency(document_text.?.text, term.atom.*);
877 };
878 if (frequency == 0) continue;
879 score += rank_mod.bm25Score(term.idf, frequency, length, stats, options);
880 }
881 total += 1;
882 try hit_mod.appendRankedHit(allocator, &hits, limit, .{ .rowid = rowid, .rank = -score });
883 }
884
885 std.mem.sort(hit_mod.RankedHit, hits.items, {}, hit_mod.rankedHitLess);
886 return .{
887 .allocator = allocator,
888 .hits = try hits.toOwnedSlice(allocator),
889 .total = total,
890 };
891 }
892
893 fn putPostingsIn(self: *Search, write: *tree.Write, allocator: Allocator, rowid: i64, text: []const u8) Error!void {
894 const counts = try posting_mod.documentTermCounts(allocator, text, self.index_prefixes, self.index_phrase_pairs and !self.omit_phrase_pair_singletons);
895 defer posting_mod.freeTermCounts(allocator, counts);
896 for (counts) |term_count| try self.putPostingIn(write, rowid, term_count);
897 }
898
899 fn replacePostingsIn(self: *Search, write: *tree.Write, allocator: Allocator, rowid: i64, old_text: []const u8, new_text: []const u8) Error!void {
900 const old_counts = try posting_mod.documentTermCounts(allocator, old_text, self.index_prefixes, self.index_phrase_pairs);
901 defer posting_mod.freeTermCounts(allocator, old_counts);
902 const new_counts = try posting_mod.documentTermCounts(allocator, new_text, self.index_prefixes, self.index_phrase_pairs and !self.omit_phrase_pair_singletons);
903 defer posting_mod.freeTermCounts(allocator, new_counts);
904 for (old_counts) |old_count| {
905 const new_count = termCount(new_counts, old_count.key) orelse {
906 try self.deletePostingIn(write, rowid, old_count.key);
907 continue;
908 };
909 if (old_count.count != new_count.count) try self.putPostingIn(write, rowid, new_count);
910 }
911 for (new_counts) |new_count| {
912 if (termCount(old_counts, new_count.key) == null) try self.putPostingIn(write, rowid, new_count);
913 }
914 }
915
916 fn putPostingIn(self: *Search, write: *tree.Write, rowid: i64, term_count: posting_mod.TermCount) Error!void {
917 var payload: [page.size]u8 = undefined;
918 try self.terms.putPayloadIn(write, rowid, &posting_mod.postingValues(term_count.key), try posting_mod.postingPayload(&payload, term_count.count));
919 }
920
921 fn deletePostingIn(self: *Search, write: *tree.Write, rowid: i64, key: []const u8) Error!void {
922 self.terms.deleteIn(write, rowid, &posting_mod.postingValues(key)) catch |err| switch (err) {
923 error.KeyNotFound => {},
924 else => return err,
925 };
926 var payload: [page.size]u8 = undefined;
927 try self.terms.putPayloadIn(write, rowid, &posting_mod.postingValues(key), try posting_mod.postingPayload(&payload, 0));
928 }
929
930 fn putDocumentLengthIn(self: *Search, write: *tree.Write, rowid: i64, length: usize) Error!void {
931 var payload: [posting_mod.document_length_payload_size]u8 = undefined;
932 try self.terms.putPayloadIn(write, rowid, &posting_mod.documentLengthValues(), try posting_mod.documentLengthPayload(&payload, length));
933 }
934
935 fn deleteDocumentLengthIn(self: *Search, write: *tree.Write, rowid: i64) Error!void {
936 var payload: [posting_mod.document_length_payload_size]u8 = undefined;
937 try self.terms.putPayloadIn(write, rowid, &posting_mod.documentLengthValues(), posting_mod.documentLengthTombstonePayload(&payload));
938 }
939
940 fn putCorpusStatsIn(self: *Search, write: *tree.Write, stats: posting_mod.CorpusStats) Error!void {
941 var payload: [posting_mod.corpus_stats_payload_size]u8 = undefined;
942 try self.terms.putPayloadIn(write, posting_mod.internal_meta_rowid, &posting_mod.corpusStatsValues(), try posting_mod.corpusStatsPayload(&payload, stats));
943 }
944
945 fn putSortedDocumentLengthsIn(self: *Search, write: *tree.Write, allocator: Allocator, lengths: []posting_mod.DocumentLength) Error!void {
946 std.mem.sort(posting_mod.DocumentLength, lengths, {}, posting_mod.documentLengthLess);
947 if (posting_mod.canSegmentDocumentLengths(lengths)) {
948 const payload = try posting_mod.documentLengthSegmentPayload(allocator, lengths);
949 defer allocator.free(payload);
950 try self.terms.putPayloadIn(write, try posting_mod.segmentRowid(lengths[0].rowid), &posting_mod.documentLengthValues(), payload);
951 return;
952 }
953 for (lengths) |document_length| try self.putDocumentLengthIn(write, document_length.rowid, document_length.length);
954 }
955
956 fn putSortedPostingsIn(self: *Search, write: *tree.Write, allocator: Allocator, postings: []const posting_mod.Posting, keys: []const []const u8) Error!void {
957 var start: usize = 0;
958 while (start < postings.len) {
959 var end = start + 1;
960 while (end < postings.len and postings[start].key_index == postings[end].key_index) : (end += 1) {}
961 const group = postings[start..end];
962 if (group[0].key_index >= keys.len) return error.InvalidSearchIndex;
963 const key = keys[group[0].key_index];
964 if (posting_mod.canSegmentPostings(group)) {
965 const payload = try posting_mod.postingSegmentPayload(allocator, group);
966 defer allocator.free(payload);
967 try self.terms.putPayloadIn(write, try posting_mod.segmentRowid(group[0].rowid), &posting_mod.postingValues(key), payload);
968 } else if (self.omit_phrase_pair_singletons and posting_mod.isPhrasePairPostingKey(key)) {
969 start = end;
970 continue;
971 } else {
972 for (group) |posting| {
973 var payload: [page.size]u8 = undefined;
974 try self.terms.putPayloadIn(write, posting.rowid, &posting_mod.postingValues(key), try posting_mod.postingPayload(&payload, posting.count));
975 }
976 }
977 start = end;
978 }
979 }
980
981 fn putTextIn(self: *Search, write: *tree.Write, allocator: Allocator, rowid: i64, text: []const u8) Error!void {
982 const values = [_]row.Value{.{ .text = text }};
983 const size = try row.encodedSize(&values);
984 if (size <= page.size) {
985 var row_bytes: [page.size]u8 = undefined;
986 try self.documents.putEncodedIn(write, rowid, try row.encode(&row_bytes, &values));
987 return;
988 }
989 const row_bytes = try allocator.alloc(u8, size);
990 defer allocator.free(row_bytes);
991 try self.documents.putEncodedIn(write, rowid, try row.encode(row_bytes, &values));
992 }
993
994 fn deletePostingsIn(self: *Search, write: *tree.Write, allocator: Allocator, rowid: i64, text: []const u8) Error!void {
995 const counts = try posting_mod.documentTermCounts(allocator, text, self.index_prefixes, self.index_phrase_pairs);
996 defer posting_mod.freeTermCounts(allocator, counts);
997 for (counts) |term_count| try self.deletePostingIn(write, rowid, term_count.key);
998 }
999
1000 fn clauseIndexHits(self: anytype, allocator: Allocator, clause: query_mod.Clause) Error!?[]hit_mod.Hit {
1001 if (clause.positive.len == 0) return null;
1002 const first = try self.atomIndexHits(allocator, clause.positive[0]);
1003 var current = first orelse return null;
1004 errdefer allocator.free(current);
1005 var index: usize = 1;
1006 while (index < clause.positive.len) : (index += 1) {
1007 const next_maybe = try self.atomIndexHits(allocator, clause.positive[index]);
1008 const next = next_maybe orelse {
1009 allocator.free(current);
1010 return null;
1011 };
1012 defer allocator.free(next);
1013 const merged = try hit_mod.intersectHits(allocator, current, next);
1014 allocator.free(current);
1015 current = merged;
1016 }
1017 for (clause.negative) |atom| {
1018 const excluded = try self.atomDocuments(allocator, atom);
1019 defer allocator.free(excluded);
1020 const filtered = try hit_mod.subtractHits(allocator, current, excluded);
1021 allocator.free(current);
1022 current = filtered;
1023 }
1024 return current;
1025 }
1026
1027 fn clauseBoundedIndexHits(self: anytype, allocator: Allocator, clause: query_mod.Clause, limit: usize) Error!?hit_mod.HitSet {
1028 if (clause.negative.len != 0 or clause.positive.len != 1) return null;
1029 return try self.atomBoundedIndexHits(allocator, clause.positive[0], limit);
1030 }
1031
1032 fn clauseScoreBandIndexHits(self: anytype, allocator: Allocator, clause: query_mod.Clause, limit: usize) Error!?hit_mod.HitSet {
1033 if (clause.negative.len != 0 or clause.positive.len != 1) return null;
1034 return try self.atomScoreBandIndexHits(allocator, clause.positive[0], limit);
1035 }
1036
1037 fn queryBoundedExactOrHits(self: anytype, allocator: Allocator, parsed: query_mod.Query, limit: usize) Error!?hit_mod.HitSet {
1038 if (parsed.clauses.len < 2) return null;
1039 for (parsed.clauses) |clause| _ = query_mod.exactClauseTerm(clause) orelse return null;
1040
1041 const cursors = try allocator.alloc(rank_mod.ExactTermCursor, parsed.clauses.len);
1042 var cursor_count: usize = 0;
1043 defer {
1044 for (cursors[0..cursor_count]) |*cursor| cursor.deinit();
1045 allocator.free(cursors);
1046 }
1047
1048 for (parsed.clauses) |clause| {
1049 const term = query_mod.exactClauseTerm(clause).?;
1050 cursors[cursor_count] = try rank_mod.ExactTermCursor.init(self, allocator, term);
1051 cursor_count += 1;
1052 try cursors[cursor_count - 1].advance();
1053 }
1054
1055 var hits: std.ArrayList(hit_mod.Hit) = .empty;
1056 errdefer hits.deinit(allocator);
1057 var total: usize = 0;
1058 while (try rank_mod.nextExactOrHit(cursors)) |hit| {
1059 total += 1;
1060 try hit_mod.appendBoundedHit(allocator, &hits, limit, hit);
1061 }
1062 return .{ .hits = try hits.toOwnedSlice(allocator), .total = total };
1063 }
1064
1065 fn queryScoreBandExactOrHits(self: anytype, allocator: Allocator, parsed: query_mod.Query, limit: usize) Error!?hit_mod.HitSet {
1066 var bounded_hits = (try self.queryBoundedExactOrHits(allocator, parsed, limit)) orelse return null;
1067 errdefer if (bounded_hits.hits.len != 0) allocator.free(bounded_hits.hits);
1068 if (bounded_hits.total <= limit) return bounded_hits;
1069 const cutoff = hit_mod.minHitScore(bounded_hits.hits);
1070 allocator.free(bounded_hits.hits);
1071 bounded_hits.hits = &.{};
1072 bounded_hits.hits = try self.queryExactOrHitsAtLeastScore(allocator, parsed, cutoff);
1073 return bounded_hits;
1074 }
1075
1076 fn queryExactOrHitsAtLeastScore(self: anytype, allocator: Allocator, parsed: query_mod.Query, cutoff: usize) Error![]hit_mod.Hit {
1077 if (parsed.clauses.len < 2) return &.{};
1078
1079 const cursors = try allocator.alloc(rank_mod.ExactTermCursor, parsed.clauses.len);
1080 var cursor_count: usize = 0;
1081 defer {
1082 for (cursors[0..cursor_count]) |*cursor| cursor.deinit();
1083 allocator.free(cursors);
1084 }
1085
1086 for (parsed.clauses) |clause| {
1087 const term = query_mod.exactClauseTerm(clause).?;
1088 cursors[cursor_count] = try rank_mod.ExactTermCursor.init(self, allocator, term);
1089 cursor_count += 1;
1090 try cursors[cursor_count - 1].advance();
1091 }
1092
1093 var hits: std.ArrayList(hit_mod.Hit) = .empty;
1094 errdefer hits.deinit(allocator);
1095 while (try rank_mod.nextExactOrHit(cursors)) |hit| {
1096 if (hit.score >= cutoff) try hits.append(allocator, hit);
1097 }
1098 return try hits.toOwnedSlice(allocator);
1099 }
1100
1101 fn atomIndexHits(self: anytype, allocator: Allocator, atom: query_mod.Atom) Error!?[]hit_mod.Hit {
1102 if (atom.tokens.len == 0 or atom.prefix or atom.near_window != null or atom.field != null) return null;
1103 if (atom.phrase) {
1104 if (!query_mod.isPhrasePairAtom(atom)) return null;
1105 const hits = if (try self.canUsePhrasePairPostings(allocator) and !try self.phrasePairPostingsArePartial(allocator))
1106 try self.phrasePairHits(allocator, atom.tokens[0], atom.tokens[1])
1107 else
1108 try self.phrasePairVerifiedHits(allocator, atom, 1);
1109 errdefer if (hits.len != 0) allocator.free(hits);
1110 for (hits) |*hit| hit.score = std.math.mul(usize, hit.score, atom.tokens.len) catch return error.InvalidSearchIndex;
1111 return hits;
1112 }
1113 var current = try self.termHits(allocator, atom.tokens[0]);
1114 errdefer allocator.free(current);
1115 var index: usize = 1;
1116 while (index < atom.tokens.len) : (index += 1) {
1117 const next = try self.termHits(allocator, atom.tokens[index]);
1118 defer allocator.free(next);
1119 const merged = try hit_mod.intersectHits(allocator, current, next);
1120 allocator.free(current);
1121 current = merged;
1122 }
1123 return current;
1124 }
1125
1126 fn atomBoundedIndexHits(self: anytype, allocator: Allocator, atom: query_mod.Atom, limit: usize) Error!?hit_mod.HitSet {
1127 if (atom.tokens.len != 1 or atom.prefix or atom.phrase or atom.near_window != null or atom.field != null) return null;
1128 return try self.termHitsBounded(allocator, atom.tokens[0], limit);
1129 }
1130
1131 fn atomScoreBandIndexHits(self: anytype, allocator: Allocator, atom: query_mod.Atom, limit: usize) Error!?hit_mod.HitSet {
1132 if (atom.tokens.len != 1 or atom.prefix or atom.phrase or atom.near_window != null or atom.field != null) return null;
1133 return try self.termHitsScoreBand(allocator, atom.tokens[0], limit);
1134 }
1135
1136 fn clauseCandidates(self: anytype, allocator: Allocator, clause: query_mod.Clause) Error![]i64 {
1137 var current = if (clause.positive.len == 0) try self.allDocuments(allocator) else try self.atomDocuments(allocator, clause.positive[0]);
1138 errdefer allocator.free(current);
1139 var index: usize = if (clause.positive.len == 0) 0 else 1;
1140 while (index < clause.positive.len) : (index += 1) {
1141 const next = try self.atomDocuments(allocator, clause.positive[index]);
1142 defer allocator.free(next);
1143 const merged = try hit_mod.intersectRows(allocator, current, next);
1144 allocator.free(current);
1145 current = merged;
1146 }
1147 for (clause.negative) |atom| {
1148 const excluded = try self.atomDocuments(allocator, atom);
1149 defer allocator.free(excluded);
1150 const filtered = try hit_mod.subtractRows(allocator, current, excluded);
1151 allocator.free(current);
1152 current = filtered;
1153 }
1154 return current;
1155 }
1156
1157 fn scoredClauseHits(self: anytype, allocator: Allocator, clause: query_mod.Clause) Error![]hit_mod.Hit {
1158 const candidates = try self.clauseCandidates(allocator, clause);
1159 defer allocator.free(candidates);
1160 var hits: std.ArrayList(hit_mod.Hit) = .empty;
1161 errdefer hits.deinit(allocator);
1162 for (candidates) |rowid| {
1163 const doc = (try self.documents.get(allocator, rowid)) orelse continue;
1164 defer allocator.free(doc);
1165 const score = text_mod.clauseScore(try token_mod.textFromRow(doc), clause);
1166 if (score == 0) continue;
1167 try hits.append(allocator, .{ .rowid = rowid, .score = score });
1168 }
1169 return try hits.toOwnedSlice(allocator);
1170 }
1171
1172 fn atomDocuments(self: anytype, allocator: Allocator, atom: query_mod.Atom) Error![]i64 {
1173 if (atom.tokens.len == 0) return &.{};
1174 if (query_mod.isPhrasePairAtom(atom)) {
1175 const hits = if (try self.canUsePhrasePairPostings(allocator) and !try self.phrasePairPostingsArePartial(allocator))
1176 try self.phrasePairHits(allocator, atom.tokens[0], atom.tokens[1])
1177 else
1178 try self.phrasePairVerifiedHits(allocator, atom, 1);
1179 defer if (hits.len != 0) allocator.free(hits);
1180 var rows: std.ArrayList(i64) = .empty;
1181 errdefer rows.deinit(allocator);
1182 try rows.ensureUnusedCapacity(allocator, hits.len);
1183 for (hits) |hit| rows.appendAssumeCapacity(hit.rowid);
1184 return try rows.toOwnedSlice(allocator);
1185 }
1186 var current = try self.termDocuments(allocator, atom.tokens[0], atom.prefix);
1187 errdefer allocator.free(current);
1188 if (atom.prefix) {
1189 if (atom.field == null) return current;
1190 const filtered = try self.verifiedRowsFromCandidates(allocator, current, atom);
1191 allocator.free(current);
1192 return filtered;
1193 }
1194 var index: usize = 1;
1195 while (index < atom.tokens.len) : (index += 1) {
1196 const next = try self.termDocuments(allocator, atom.tokens[index], false);
1197 defer allocator.free(next);
1198 const merged = try hit_mod.intersectRows(allocator, current, next);
1199 allocator.free(current);
1200 current = merged;
1201 }
1202 if (atom.phrase or atom.near_window != null or atom.field != null) {
1203 const filtered = try self.verifiedRowsFromCandidates(allocator, current, atom);
1204 allocator.free(current);
1205 return filtered;
1206 }
1207 return current;
1208 }
1209
1210 fn verifiedRowsFromCandidates(self: anytype, allocator: Allocator, rows: []const i64, atom: query_mod.Atom) Error![]i64 {
1211 var out: std.ArrayList(i64) = .empty;
1212 errdefer out.deinit(allocator);
1213 for (rows) |rowid| {
1214 var document = (try self.getDocumentText(allocator, rowid)) orelse continue;
1215 const frequency = text_mod.atomFrequency(document.text, atom);
1216 document.deinit(allocator);
1217 if (frequency != 0) try out.append(allocator, rowid);
1218 }
1219 return try out.toOwnedSlice(allocator);
1220 }
1221
1222 fn phrasePairVerifiedHits(self: anytype, allocator: Allocator, atom: query_mod.Atom, score_multiplier: usize) Error![]hit_mod.Hit {
1223 std.debug.assert(query_mod.isPhrasePairAtom(atom));
1224 var left = try rank_mod.ExactTermCursor.init(self, allocator, atom.tokens[0]);
1225 defer left.deinit();
1226 try left.advance();
1227 var right = try rank_mod.ExactTermCursor.init(self, allocator, atom.tokens[1]);
1228 defer right.deinit();
1229 try right.advance();
1230 var indexed: ?rank_mod.ExactTermCursor = if (try self.canUsePhrasePairPostings(allocator))
1231 try rank_mod.ExactTermCursor.initPhrasePair(self, allocator, atom.tokens[0], atom.tokens[1])
1232 else
1233 null;
1234 defer if (indexed) |*cursor| cursor.deinit();
1235 if (indexed) |*cursor| try cursor.advance();
1236 var hits: std.ArrayList(hit_mod.Hit) = .empty;
1237 errdefer hits.deinit(allocator);
1238 var documents: ?text_mod.DocumentTextCursor = null;
1239 defer if (documents) |*cursor| cursor.deinit();
1240 while (try rank_mod.nextPhrasePairCandidateRow(&left, &right)) |rowid| {
1241 const indexed_frequency = if (indexed) |*cursor| try rank_mod.postingCursorFrequencyAt(cursor, rowid) else 0;
1242 const frequency = if (indexed_frequency != 0) indexed_frequency else frequency: {
1243 if (documents == null) {
1244 documents = @as(text_mod.DocumentTextCursor, undefined);
1245 errdefer documents = null;
1246 try documents.?.init(self, allocator, rowid);
1247 }
1248 if (documents) |*cursor| break :frequency (try cursor.atomFrequencyFor(rowid, atom)) orelse 0;
1249 unreachable;
1250 };
1251 if (frequency != 0) try hits.append(allocator, .{
1252 .rowid = rowid,
1253 .score = std.math.mul(usize, frequency, score_multiplier) catch return error.InvalidSearchIndex,
1254 });
1255 }
1256 return try hits.toOwnedSlice(allocator);
1257 }
1258
1259 fn termDocuments(self: anytype, allocator: Allocator, term: []const u8, prefix: bool) Error![]i64 {
1260 if (prefix and !self.index_prefixes) return try self.prefixDocuments(allocator, term);
1261 const hits = try self.termHitsForKind(allocator, if (prefix) '*' else '=', term);
1262 defer allocator.free(hits);
1263 var rows: std.ArrayList(i64) = .empty;
1264 errdefer rows.deinit(allocator);
1265 try rows.ensureUnusedCapacity(allocator, hits.len);
1266 for (hits) |hit| rows.appendAssumeCapacity(hit.rowid);
1267 return try hit_mod.sortedUniqueRows(allocator, &rows);
1268 }
1269
1270 fn termHits(self: anytype, allocator: Allocator, term: []const u8) Error![]hit_mod.Hit {
1271 return try self.termHitsForKind(allocator, '=', term);
1272 }
1273
1274 fn termHitsForKind(self: anytype, allocator: Allocator, kind: u8, term: []const u8) Error![]hit_mod.Hit {
1275 var key_buffer: [token_mod.max_token_bytes + 1]u8 = undefined;
1276 const key_text = posting_mod.postingKeyBuffer(&key_buffer, kind, term);
1277 return try self.postingHits(allocator, key_text);
1278 }
1279
1280 fn postingHits(self: anytype, allocator: Allocator, key_text: []const u8) Error![]hit_mod.Hit {
1281 var lookup: index_mod.Scan = undefined;
1282 try self.terms.lookupPayloads(&lookup, allocator, &.{.{ .text = key_text }});
1283 defer lookup.deinit();
1284 var raw: std.ArrayList(hit_mod.RawHit) = .empty;
1285 errdefer raw.deinit(allocator);
1286 while (try lookup.next()) |entry| try hit_mod.appendRawHits(allocator, &raw, entry.rowid, entry.payload);
1287 return try hit_mod.coalescedHits(allocator, &raw);
1288 }
1289
1290 fn phrasePairHits(self: anytype, allocator: Allocator, first: []const u8, second: []const u8) Error![]hit_mod.Hit {
1291 var key_buffer: [posting_mod.phrase_pair_posting_key_max_bytes]u8 = undefined;
1292 const key_text = posting_mod.phrasePairPostingKeyBuffer(&key_buffer, first, second);
1293 return try self.postingHits(allocator, key_text);
1294 }
1295
1296 fn termHitsBounded(self: anytype, allocator: Allocator, term: []const u8, limit: usize) Error!hit_mod.HitSet {
1297 const hits = try self.termHits(allocator, term);
1298 errdefer if (hits.len != 0) allocator.free(hits);
1299 const total = hits.len;
1300 return .{ .hits = try hit_mod.sortAndLimitOwnedHits(allocator, hits, limit), .total = total };
1301 }
1302
1303 fn termHitsScoreBand(self: anytype, allocator: Allocator, term: []const u8, limit: usize) Error!hit_mod.HitSet {
1304 if (limit == 0) {
1305 const hits = try self.termHits(allocator, term);
1306 return .{ .hits = hits, .total = hits.len };
1307 }
1308 var cursor = try rank_mod.ExactTermCursor.init(self, allocator, term);
1309 defer cursor.deinit();
1310 var hits: std.ArrayList(hit_mod.Hit) = .empty;
1311 errdefer hits.deinit(allocator);
1312 var total: usize = 0;
1313 while (true) {
1314 try cursor.advance();
1315 const hit = cursor.current orelse break;
1316 total += 1;
1317 try hit_mod.appendBoundedHit(allocator, &hits, limit, hit);
1318 }
1319 if (total <= limit) {
1320 const owned = try hits.toOwnedSlice(allocator);
1321 std.mem.sort(hit_mod.Hit, owned, {}, hit_mod.hitLess);
1322 return .{ .hits = owned, .total = total };
1323 }
1324 const cutoff = hit_mod.minHitScore(hits.items);
1325 hits.clearRetainingCapacity();
1326 cursor.reset();
1327 while (true) {
1328 try cursor.advance();
1329 const hit = cursor.current orelse break;
1330 if (hit.score >= cutoff) try hits.append(allocator, hit);
1331 }
1332 const owned = try hits.toOwnedSlice(allocator);
1333 std.mem.sort(hit_mod.Hit, owned, {}, hit_mod.hitLess);
1334 return .{ .hits = owned, .total = total };
1335 }
1336
1337 fn termHitsAtLeastScore(self: anytype, allocator: Allocator, term: []const u8, cutoff: usize) Error![]hit_mod.Hit {
1338 const source = try self.termHits(allocator, term);
1339 defer allocator.free(source);
1340 var hits: std.ArrayList(hit_mod.Hit) = .empty;
1341 errdefer hits.deinit(allocator);
1342 for (source) |hit| {
1343 const score = hit.score;
1344 if (score < cutoff) continue;
1345 try hits.append(allocator, hit);
1346 }
1347 return try hits.toOwnedSlice(allocator);
1348 }
1349
1350 fn prefixHits(self: anytype, allocator: Allocator, term: []const u8) Error![]hit_mod.Hit {
1351 if (self.index_prefixes) return try self.termHitsForKind(allocator, '*', term);
1352 var start_buffer: [token_mod.max_token_bytes + 1]u8 = undefined;
1353 const start = posting_mod.postingKeyBuffer(&start_buffer, '=', term);
1354 var end_buffer: [token_mod.max_token_bytes + 2]u8 = undefined;
1355 const end = posting_mod.prefixEndBuffer(&end_buffer, start);
1356 var lookup: index_mod.Scan = undefined;
1357 try self.terms.scanPayloads(
1358 &lookup,
1359 allocator,
1360 &.{.{ .text = start }},
1361 &.{.{ .text = end }},
1362 );
1363 defer lookup.deinit();
1364 var raw: std.ArrayList(hit_mod.RawHit) = .empty;
1365 errdefer raw.deinit(allocator);
1366 while (try lookup.next()) |entry| try hit_mod.appendRawHits(allocator, &raw, entry.rowid, entry.payload);
1367 const hits = try hit_mod.coalescedHits(allocator, &raw);
1368 return hits;
1369 }
1370
1371 fn prefixDocuments(self: anytype, allocator: Allocator, term: []const u8) Error![]i64 {
1372 const hits = try self.prefixHits(allocator, term);
1373 defer allocator.free(hits);
1374 var rows: std.ArrayList(i64) = .empty;
1375 errdefer rows.deinit(allocator);
1376 try rows.ensureUnusedCapacity(allocator, hits.len);
1377 for (hits) |hit| rows.appendAssumeCapacity(hit.rowid);
1378 return try hit_mod.sortedUniqueRows(allocator, &rows);
1379 }
1380
1381 fn allDocuments(self: anytype, allocator: Allocator) Error![]i64 {
1382 var scan: table_mod.Scan = undefined;
1383 try self.documents.scan(&scan, allocator, null, null);
1384 defer scan.deinit();
1385 var rows: std.ArrayList(i64) = .empty;
1386 errdefer rows.deinit(allocator);
1387 while (try scan.next()) |entry| try rows.append(allocator, entry.rowid);
1388 return try hit_mod.sortedUniqueRows(allocator, &rows);
1389 }
1390
1391 fn corpusStats(self: anytype, allocator: Allocator) Error!posting_mod.CorpusStats {
1392 var lookup: index_mod.Scan = undefined;
1393 try self.terms.lookupPayloads(&lookup, allocator, &posting_mod.corpusStatsValues());
1394 defer lookup.deinit();
1395 const entry = (try lookup.next()) orelse return error.InvalidSearchIndex;
1396 if (entry.rowid != posting_mod.internal_meta_rowid) return error.InvalidSearchIndex;
1397 const stats = try posting_mod.corpusStatsPayloadValue(entry.payload);
1398 if (try lookup.next() != null) return error.InvalidSearchIndex;
1399 return stats;
1400 }
1401
1402 fn corpusStatsOrEmpty(self: anytype, allocator: Allocator) Error!posting_mod.CorpusStats {
1403 return self.corpusStats(allocator) catch |err| switch (err) {
1404 error.InvalidSearchIndex => stats: {
1405 var scan: table_mod.Scan = undefined;
1406 try self.documents.scan(&scan, allocator, null, null);
1407 defer scan.deinit();
1408 if (try scan.next() != null) return error.InvalidSearchIndex;
1409 break :stats .{ .documents = 0, .total_tokens = 0 };
1410 },
1411 else => return err,
1412 };
1413 }
1414
1415 fn documentLength(self: anytype, allocator: Allocator, rowid: i64) Error!usize {
1416 var cursor = try rank_mod.DocumentLengthCursor.init(self, allocator);
1417 defer cursor.deinit();
1418 return try cursor.lengthFor(rowid);
1419 }
1420
1421 fn documentLengths(self: anytype, allocator: Allocator) Error![]posting_mod.DocumentLength {
1422 var cursor = try rank_mod.DocumentLengthCursor.init(self, allocator);
1423 defer cursor.deinit();
1424 var lengths: std.ArrayList(posting_mod.DocumentLength) = .empty;
1425 errdefer lengths.deinit(allocator);
1426 while (cursor.rowid) |rowid| {
1427 try lengths.append(allocator, .{
1428 .rowid = rowid,
1429 .length = cursor.length,
1430 });
1431 try cursor.advance();
1432 }
1433 return try lengths.toOwnedSlice(allocator);
1434 }
1435
1436 fn collectRankingTerms(self: anytype, allocator: Allocator, parsed: query_mod.Query, document_count: usize) Error![]rank_mod.RankingTerm {
1437 var terms: std.ArrayList(rank_mod.RankingTerm) = .empty;
1438 errdefer {
1439 for (terms.items) |*term| term.deinit(allocator);
1440 terms.deinit(allocator);
1441 }
1442 for (parsed.clauses) |*clause| {
1443 for (clause.positive) |*atom| {
1444 if (atom.tokens.len == 0) continue;
1445 if (query_mod.isExactRankingAtom(atom.*)) {
1446 const hits = try self.termHits(allocator, atom.tokens[0]);
1447 const idf = rank_mod.bm25Idf(document_count, hits.len) catch |err| {
1448 if (hits.len != 0) allocator.free(hits);
1449 return err;
1450 };
1451 terms.append(allocator, .{
1452 .atom = atom,
1453 .posting_hits = hits,
1454 .document_frequency = hits.len,
1455 .idf = idf,
1456 }) catch |err| {
1457 if (hits.len != 0) allocator.free(hits);
1458 return err;
1459 };
1460 } else if (query_mod.isPhrasePairAtom(atom.*)) {
1461 const hits = if (try self.canUsePhrasePairPostings(allocator) and !try self.phrasePairPostingsArePartial(allocator))
1462 try self.phrasePairHits(allocator, atom.tokens[0], atom.tokens[1])
1463 else
1464 try self.phrasePairVerifiedHits(allocator, atom.*, 1);
1465 const idf = rank_mod.bm25Idf(document_count, hits.len) catch |err| {
1466 if (hits.len != 0) allocator.free(hits);
1467 return err;
1468 };
1469 terms.append(allocator, .{
1470 .atom = atom,
1471 .posting_hits = hits,
1472 .document_frequency = hits.len,
1473 .idf = idf,
1474 }) catch |err| {
1475 if (hits.len != 0) allocator.free(hits);
1476 return err;
1477 };
1478 } else {
1479 const frequency = try self.atomDocumentFrequency(allocator, atom.*);
1480 try terms.append(allocator, .{
1481 .atom = atom,
1482 .posting_hits = &.{},
1483 .document_frequency = frequency,
1484 .idf = try rank_mod.bm25Idf(document_count, frequency),
1485 });
1486 }
1487 }
1488 }
1489 return try terms.toOwnedSlice(allocator);
1490 }
1491
1492 fn atomDocumentFrequency(self: anytype, allocator: Allocator, atom: query_mod.Atom) Error!usize {
1493 const rows = try self.atomDocuments(allocator, atom);
1494 defer allocator.free(rows);
1495 if (!atom.phrase) return rows.len;
1496 var count: usize = 0;
1497 for (rows) |rowid| {
1498 var document = (try self.getDocumentText(allocator, rowid)) orelse continue;
1499 const frequency = text_mod.atomFrequency(document.text, atom);
1500 document.deinit(allocator);
1501 if (frequency != 0) count += 1;
1502 }
1503 return count;
1504 }
1505
1506 fn clearIn(self: *Search, write: *tree.Write, allocator: Allocator) Error!void {
1507 _ = allocator;
1508 try write.clear(&self.terms.entries);
1509 try write.clear(&self.documents.rows);
1510 }
1511
1512 fn collectTermKeys(self: anytype, allocator: Allocator) Error![]const []const u8 {
1513 var scan: index_mod.Scan = undefined;
1514 try self.terms.scan(&scan, allocator, null, null);
1515 defer scan.deinit();
1516 var keys: std.ArrayList([]const u8) = .empty;
1517 errdefer posting_mod.freeStringSlice(allocator, keys.items);
1518 while (try scan.next()) |entry| try keys.append(allocator, try allocator.dupe(u8, entry.key));
1519 return try keys.toOwnedSlice(allocator);
1520 }
1521
1522 fn collectPostings(self: anytype, allocator: Allocator, documents: []const Document) Error!posting_mod.PostingSet {
1523 var postings: std.ArrayList(posting_mod.Posting) = .empty;
1524 errdefer postings.deinit(allocator);
1525 var keys: std.ArrayList([]const u8) = .empty;
1526 errdefer posting_mod.deinitStringList(allocator, &keys);
1527 var key_map = posting_mod.PostingKeyMap.init(allocator);
1528 defer key_map.deinit();
1529 const phrase_pairs = self.index_phrase_pairs and !self.omit_phrase_pair_singletons;
1530 if (!self.index_prefixes) {
1531 const postings_capacity = bulkCapacity(documents.len, if (phrase_pairs) 32 else 16, 0);
1532 try postings.ensureTotalCapacity(allocator, postings_capacity);
1533 const key_capacity = bulkCapacity(documents.len, if (phrase_pairs) 2 else 1, 64);
1534 try keys.ensureTotalCapacity(allocator, key_capacity);
1535 try key_map.ensureTotalCapacity(hashCapacityHint(key_capacity));
1536 }
1537 const term_count_capacity = try documentTermCountScratchCapacity(documents, self.index_prefixes, phrase_pairs);
1538 const term_counts: []posting_mod.TermCount = if (term_count_capacity == 0)
1539 &.{}
1540 else
1541 try allocator.alloc(posting_mod.TermCount, term_count_capacity);
1542 defer if (term_counts.len != 0) allocator.free(term_counts);
1543 for (documents) |document| {
1544 const counts = try posting_mod.documentInternedTermCounts(allocator, token_mod.documentText(document.text), self.index_prefixes, phrase_pairs, &key_map, &keys, term_counts);
1545 try postings.ensureUnusedCapacity(allocator, counts.len);
1546 for (counts) |term_count| {
1547 const key_index = key_map.get(term_count.key) orelse return error.InvalidSearchIndex;
1548 const count = std.math.cast(u32, term_count.count) orelse return error.InvalidSearchIndex;
1549 postings.appendAssumeCapacity(.{ .rowid = document.rowid, .key_index = key_index, .count = count });
1550 }
1551 }
1552 if (self.index_phrase_pairs and self.omit_phrase_pair_singletons) try posting_mod.collectPartialPhrasePairPostings(allocator, documents, &postings, &keys);
1553 const items = try postings.toOwnedSlice(allocator);
1554 errdefer if (items.len != 0) allocator.free(items);
1555 return .{
1556 .items = items,
1557 .keys = try keys.toOwnedSlice(allocator),
1558 };
1559 }
1560 };
1561
1562 pub const Reader = struct {
1563 documents: table_mod.Reader,
1564 terms: index_mod.Reader,
1565 index_prefixes: bool,
1566 index_phrase_pairs: bool,
1567 omit_phrase_pair_singletons: bool,
1568 field_names: []const []const u8,
1569
1570 pub fn open(snapshot: file.Snapshot, options: Options) Error!Reader {
1571 const roots = [_]space_mod.RootSpec{
1572 .{ .root_page = options.documents_root },
1573 .{ .root_page = options.terms_root },
1574 };
1575 const read_space = try space_mod.Reader.open(snapshot, .{
1576 .meta_page = options.meta_page,
1577 .roots = &roots,
1578 .reserved_page_max = options.reserved_page_max,
1579 });
1580 return .{
1581 .documents = try read_space.rowidTable(options.documents_root),
1582 .terms = try read_space.index(options.terms_root, &.{.{ .collation = .binary }}),
1583 .index_prefixes = options.index_prefixes,
1584 .index_phrase_pairs = options.index_phrase_pairs,
1585 .omit_phrase_pair_singletons = options.omit_phrase_pair_singletons,
1586 .field_names = options.field_names,
1587 };
1588 }
1589
1590 pub const getText = Search.getText;
1591 pub const capabilities = Search.capabilities;
1592 pub const indexedSource = Search.indexedSource;
1593 pub const sourceMatches = Search.sourceMatches;
1594 pub const sourceSchemaMatches = Search.sourceSchemaMatches;
1595 pub const postingStats = Search.postingStats;
1596 pub const query = Search.query;
1597 pub const queryScoreBand = Search.queryScoreBand;
1598 pub const queryRanked = Search.queryRanked;
1599 pub const prepare = Search.prepare;
1600
1601 const getDocumentText = Search.getDocumentText;
1602 const requestedCapabilities = Search.requestedCapabilities;
1603 const indexCapabilities = Search.indexCapabilities;
1604 const canUsePhrasePairPostings = Search.canUsePhrasePairPostings;
1605 const phrasePairPostingsArePartial = Search.phrasePairPostingsArePartial;
1606 const queryParsed = Search.queryParsed;
1607 const queryParsedScoreBand = Search.queryParsedScoreBand;
1608 const queryParsedRanked = Search.queryParsedRanked;
1609 const queryParsedCandidateRows = Search.queryParsedCandidateRows;
1610 const rankSingleAtomCandidateRows = Search.rankSingleAtomCandidateRows;
1611 const rankPrefixAtom = Search.rankPrefixAtom;
1612 const rankPhrasePairAtom = Search.rankPhrasePairAtom;
1613 const queryParsedRankedExactOr = Search.queryParsedRankedExactOr;
1614 const rankCandidateRows = Search.rankCandidateRows;
1615 const clauseIndexHits = Search.clauseIndexHits;
1616 const clauseBoundedIndexHits = Search.clauseBoundedIndexHits;
1617 const clauseScoreBandIndexHits = Search.clauseScoreBandIndexHits;
1618 const queryBoundedExactOrHits = Search.queryBoundedExactOrHits;
1619 const queryScoreBandExactOrHits = Search.queryScoreBandExactOrHits;
1620 const queryExactOrHitsAtLeastScore = Search.queryExactOrHitsAtLeastScore;
1621 const atomIndexHits = Search.atomIndexHits;
1622 const atomBoundedIndexHits = Search.atomBoundedIndexHits;
1623 const atomScoreBandIndexHits = Search.atomScoreBandIndexHits;
1624 const clauseCandidates = Search.clauseCandidates;
1625 const scoredClauseHits = Search.scoredClauseHits;
1626 const atomDocuments = Search.atomDocuments;
1627 const verifiedRowsFromCandidates = Search.verifiedRowsFromCandidates;
1628 const phrasePairVerifiedHits = Search.phrasePairVerifiedHits;
1629 const termDocuments = Search.termDocuments;
1630 const termHits = Search.termHits;
1631 const termHitsForKind = Search.termHitsForKind;
1632 const postingHits = Search.postingHits;
1633 const phrasePairHits = Search.phrasePairHits;
1634 const termHitsBounded = Search.termHitsBounded;
1635 const termHitsScoreBand = Search.termHitsScoreBand;
1636 const termHitsAtLeastScore = Search.termHitsAtLeastScore;
1637 const prefixHits = Search.prefixHits;
1638 const prefixDocuments = Search.prefixDocuments;
1639 const allDocuments = Search.allDocuments;
1640 const corpusStats = Search.corpusStats;
1641 const corpusStatsOrEmpty = Search.corpusStatsOrEmpty;
1642 const documentLength = Search.documentLength;
1643 const documentLengths = Search.documentLengths;
1644 const collectRankingTerms = Search.collectRankingTerms;
1645 const atomDocumentFrequency = Search.atomDocumentFrequency;
1646 const collectTermKeys = Search.collectTermKeys;
1647 const collectPostings = Search.collectPostings;
1648 };
1649
1650 fn documentTermCountScratchCapacity(documents: []const Document, index_prefixes: bool, index_phrase_pairs: bool) Allocator.Error!usize {
1651 var capacity: usize = 0;
1652 for (documents) |document| {
1653 capacity = @max(capacity, try posting_mod.documentTermCountCapacity(token_mod.documentText(document.text), index_prefixes, index_phrase_pairs));
1654 }
1655 return capacity;
1656 }
1657
1658 fn termCount(counts: []const posting_mod.TermCount, key: []const u8) ?posting_mod.TermCount {
1659 for (counts) |count| if (std.mem.eql(u8, count.key, key)) return count;
1660 return null;
1661 }
1662
1663 pub fn bulkCapacity(count: usize, multiplier: usize, slack: usize) usize {
1664 const scaled = std.math.mul(usize, count, multiplier) catch return std.math.maxInt(usize);
1665 return std.math.add(usize, scaled, slack) catch std.math.maxInt(usize);
1666 }
1667
1668 pub fn hashCapacityHint(capacity: usize) u32 {
1669 return @intCast(@min(capacity, std.math.maxInt(u32)));
1670 }
1671
1672 pub fn testingHeader() wal.Header {
1673 return .{
1674 .sequence = 1701,
1675 .salt = .{ .first = 0x7171_2323, .second = 0x4545_8989 },
1676 };
1677 }
1678
1679 pub fn recoveredHeader() wal.Header {
1680 return .{
1681 .sequence = 1702,
1682 .salt = .{ .first = 0x8888_aaaa, .second = 0x9999_bbbb },
1683 };
1684 }
1685
1686 test "search posting scratch capacity follows largest actual document" {
1687 const documents = [_]Document{
1688 .{ .rowid = 1, .text = "" },
1689 .{ .rowid = 2, .text = "alpha beta gamma" },
1690 .{ .rowid = 3, .text = "delta" },
1691 };
1692 try std.testing.expectEqual(@as(usize, 3), try documentTermCountScratchCapacity(&documents, false, false));
1693 try std.testing.expectEqual(@as(usize, 5), try documentTermCountScratchCapacity(&documents, false, true));
1694 try std.testing.expectEqual(@as(usize, 17), try documentTermCountScratchCapacity(&documents, true, false));
1695 try std.testing.expectEqual(@as(usize, 0), try documentTermCountScratchCapacity(&.{}, true, true));
1696 }
1697
1698 test "search read-only reader queries a fixed file snapshot without write declarations" {
1699 var tmp = std.testing.tmpDir(.{});
1700 defer tmp.cleanup();
1701
1702 const source = Source{ .schema = 7, .head = @splat(0x5a) };
1703 {
1704 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1705 .paths = .{ .database = "search.db", .wal = "search.wal" },
1706 .header = testingHeader(),
1707 });
1708 defer database.deinit();
1709 try database.reserve(.{ .wal_frames = 256 });
1710 var search = try Search.open(&database, .{ .index_prefixes = false });
1711 _ = try search.loadAllForSource(std.testing.allocator, &.{
1712 .{ .rowid = 4, .text = "alpha needle" },
1713 .{ .rowid = 9, .text = "beta" },
1714 }, source, .{ .durability = .buffered });
1715 try database.syncWal();
1716 }
1717
1718 const opened = try file.ReadOnlyDatabase.openForTesting(std.testing.allocator, tmp.dir, .{
1719 .paths = .{ .database = "search.db", .wal = "search.wal" },
1720 .header = testingHeader(),
1721 });
1722 var database = switch (opened) {
1723 .ready => |ready| ready,
1724 .repair_required => return error.UnexpectedRepairRequired,
1725 };
1726 defer database.deinit();
1727 const search = try Reader.open(database.snapshot(), .{ .index_prefixes = false });
1728 try std.testing.expect(try search.sourceMatches(std.testing.allocator, source));
1729 var results = try search.query(std.testing.allocator, "needle", 0);
1730 defer results.deinit();
1731 try expectHits(results.hits, &.{4});
1732 const indexed_source = (try search.indexedSource(std.testing.allocator)) orelse
1733 return error.ExpectedSearchSource;
1734 try std.testing.expect(indexed_source.same(source));
1735 try std.testing.expect(try search.sourceSchemaMatches(std.testing.allocator, source.schema));
1736 const stored_text = (try search.getText(std.testing.allocator, 4)) orelse
1737 return error.ExpectedSearchDocument;
1738 defer std.testing.allocator.free(stored_text);
1739 try std.testing.expectEqualStrings("alpha needle", stored_text);
1740 _ = try search.capabilities(std.testing.allocator);
1741 try std.testing.expect((try search.postingStats(std.testing.allocator)).term_index_entries > 0);
1742 var score_band = try search.queryScoreBand(std.testing.allocator, "needle", 1);
1743 defer score_band.deinit();
1744 try expectHits(score_band.hits, &.{4});
1745 var ranked = try search.queryRanked(std.testing.allocator, "needle", 1, .{});
1746 defer ranked.deinit();
1747 try std.testing.expectEqual(@as(usize, 1), ranked.hits.len);
1748 try std.testing.expectEqual(@as(i64, 4), ranked.hits[0].rowid);
1749 var prepared = try search.prepare(std.testing.allocator, "needle");
1750 defer prepared.deinit(std.testing.allocator);
1751 var prepared_results = try prepared.execute(&search, std.testing.allocator, 1);
1752 defer prepared_results.deinit();
1753 try expectHits(prepared_results.hits, &.{4});
1754 var prepared_band = try prepared.executeScoreBand(&search, std.testing.allocator, 1);
1755 defer prepared_band.deinit();
1756 try expectHits(prepared_band.hits, &.{4});
1757 var prepared_ranked = try prepared.executeRanked(&search, std.testing.allocator, 1, .{});
1758 defer prepared_ranked.deinit();
1759 try std.testing.expectEqual(@as(usize, 1), prepared_ranked.hits.len);
1760 try std.testing.expectEqual(@as(i64, 4), prepared_ranked.hits[0].rowid);
1761 try std.testing.expect(!@hasDecl(Reader, "put"));
1762 try std.testing.expect(!@hasDecl(Reader, "loadAllForSource"));
1763 try std.testing.expect(!@hasDecl(Reader, "clear"));
1764 }
1765
1766 test "search supports exact prefix phrase boolean and reopen" {
1767 var tmp = std.testing.tmpDir(.{});
1768 defer tmp.cleanup();
1769
1770 {
1771 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1772 .paths = .{ .database = "search.db", .wal = "search.wal" },
1773 .header = testingHeader(),
1774 });
1775 defer database.deinit();
1776 try database.reserve(.{ .wal_frames = 512 });
1777
1778 var search = try Search.open(&database, .{});
1779 _ = try search.put(std.testing.allocator, 1, "Fix authentication bug login fails", .{ .durability = .buffered });
1780 _ = try search.put(std.testing.allocator, 2, "Add authorization check logout", .{ .durability = .buffered });
1781 _ = try search.put(std.testing.allocator, 3, "Cache invalidation", .{ .durability = .buffered });
1782
1783 var exact = try search.query(std.testing.allocator, "authentication", 0);
1784 defer exact.deinit();
1785 try expectHits(exact.hits, &.{1});
1786
1787 var prefix = try search.query(std.testing.allocator, "auth*", 0);
1788 defer prefix.deinit();
1789 try expectHits(prefix.hits, &.{ 1, 2 });
1790
1791 var disjunction = try search.query(std.testing.allocator, "login OR logout", 0);
1792 defer disjunction.deinit();
1793 try expectHits(disjunction.hits, &.{ 1, 2 });
1794
1795 var negated = try search.query(std.testing.allocator, "auth* NOT authorization", 0);
1796 defer negated.deinit();
1797 try expectHits(negated.hits, &.{1});
1798
1799 var phrase = try search.query(std.testing.allocator, "\"authentication bug\"", 0);
1800 defer phrase.deinit();
1801 try expectHits(phrase.hits, &.{1});
1802
1803 var conjunction = try search.query(std.testing.allocator, "authentication AND bug", 0);
1804 defer conjunction.deinit();
1805 try expectHits(conjunction.hits, &.{1});
1806
1807 var lower_or = try search.query(std.testing.allocator, "login or logout", 0);
1808 defer lower_or.deinit();
1809 try expectHits(lower_or.hits, &.{});
1810
1811 var lower_and = try search.query(std.testing.allocator, "authentication and bug", 0);
1812 defer lower_and.deinit();
1813 try expectHits(lower_and.hits, &.{});
1814
1815 var lower_not = try search.query(std.testing.allocator, "auth* not authorization", 0);
1816 defer lower_not.deinit();
1817 try expectHits(lower_not.hits, &.{});
1818
1819 var prepared = try search.prepare(std.testing.allocator, "\"authentication bug\" OR auth* NOT authorization");
1820 defer prepared.deinit(std.testing.allocator);
1821 try std.testing.expect(prepared.matches("Fix authentication bug"));
1822 try std.testing.expect(prepared.matches("Authentication failure"));
1823 try std.testing.expect(!prepared.matches("Authorization failure"));
1824 try std.testing.expect(!prepared.matches("Authentication authorization"));
1825
1826 try database.syncWal();
1827 }
1828
1829 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1830 .paths = .{ .database = "search.db", .wal = "search.wal" },
1831 .header = recoveredHeader(),
1832 });
1833 defer reopened.deinit();
1834 var search = try Search.open(&reopened, .{});
1835 var prefix = try search.query(std.testing.allocator, "auth*", 0);
1836 defer prefix.deinit();
1837 try expectHits(prefix.hits, &.{ 1, 2 });
1838 }
1839
1840 test "search supports fts near expressions" {
1841 var tmp = std.testing.tmpDir(.{});
1842 defer tmp.cleanup();
1843
1844 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1845 .paths = .{ .database = "search.db", .wal = "search.wal" },
1846 .header = testingHeader(),
1847 });
1848 defer database.deinit();
1849 try database.reserve(.{ .wal_frames = 512 });
1850
1851 var search = try Search.open(&database, .{});
1852 _ = try search.put(std.testing.allocator, 1, "smg cli helpers truncate Trim a string to width with an ellipsis", .{ .durability = .buffered });
1853 _ = try search.put(std.testing.allocator, 2, "smg graph sem graph truncate path Drop path components after max depth", .{ .durability = .buffered });
1854 _ = try search.put(std.testing.allocator, 3, "alpha truncate one two three four five six path", .{ .durability = .buffered });
1855 _ = try search.put(std.testing.allocator, 4, "NEAR token", .{ .durability = .buffered });
1856
1857 var explicit_window = try search.query(std.testing.allocator, "NEAR(truncate path, 5)", 0);
1858 defer explicit_window.deinit();
1859 try expectHits(explicit_window.hits, &.{2});
1860
1861 var default_window = try search.query(std.testing.allocator, "NEAR(truncate path)", 0);
1862 defer default_window.deinit();
1863 try expectHits(default_window.hits, &.{ 2, 3 });
1864
1865 var adjacent = try search.query(std.testing.allocator, "NEAR(truncate path, 0)", 0);
1866 defer adjacent.deinit();
1867 try expectHits(adjacent.hits, &.{2});
1868
1869 var reversed = try search.query(std.testing.allocator, "NEAR(path truncate, 0)", 0);
1870 defer reversed.deinit();
1871 try expectHits(reversed.hits, &.{2});
1872
1873 var narrow = try search.query(std.testing.allocator, "NEAR(smg truncate, 2)", 0);
1874 defer narrow.deinit();
1875 try expectHits(narrow.hits, &.{1});
1876
1877 var wider = try search.query(std.testing.allocator, "NEAR(smg truncate, 3)", 0);
1878 defer wider.deinit();
1879 try expectHits(wider.hits, &.{ 1, 2 });
1880
1881 var bare = try search.query(std.testing.allocator, "NEAR", 0);
1882 defer bare.deinit();
1883 try expectHits(bare.hits, &.{4});
1884
1885 try std.testing.expectError(error.InvalidQuery, search.query(std.testing.allocator, "NEAR(truncate path, bad)", 0));
1886 try std.testing.expectError(error.InvalidQuery, search.query(std.testing.allocator, "NEAR(truncate path, )", 0));
1887 }
1888
1889 test "search field separators bound phrase and near matches" {
1890 var tmp = std.testing.tmpDir(.{});
1891 defer tmp.cleanup();
1892
1893 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1894 .paths = .{ .database = "search.db", .wal = "search.wal" },
1895 .header = testingHeader(),
1896 });
1897 defer database.deinit();
1898 try database.reserve(.{ .wal_frames = 512 });
1899
1900 var search = try Search.open(&database, .{ .index_phrase_pairs = true });
1901 const separated = try std.fmt.allocPrint(std.testing.allocator, "name token{c}doc body", .{token_mod.field_separator});
1902 defer std.testing.allocator.free(separated);
1903 _ = try search.put(std.testing.allocator, 1, separated, .{ .durability = .buffered });
1904 _ = try search.put(std.testing.allocator, 2, "name token doc body", .{ .durability = .buffered });
1905
1906 var conjunction = try search.query(std.testing.allocator, "token AND doc", 0);
1907 defer conjunction.deinit();
1908 try expectHits(conjunction.hits, &.{ 1, 2 });
1909
1910 var phrase = try search.query(std.testing.allocator, "\"token doc\"", 0);
1911 defer phrase.deinit();
1912 try expectHits(phrase.hits, &.{2});
1913
1914 var near = try search.query(std.testing.allocator, "NEAR(token doc, 100)", 0);
1915 defer near.deinit();
1916 try expectHits(near.hits, &.{2});
1917
1918 var same_field_phrase = try search.query(std.testing.allocator, "\"doc body\"", 0);
1919 defer same_field_phrase.deinit();
1920 try expectHits(same_field_phrase.hits, &.{ 1, 2 });
1921
1922 var same_field_near = try search.query(std.testing.allocator, "NEAR(doc body, 0)", 0);
1923 defer same_field_near.deinit();
1924 try expectHits(same_field_near.hits, &.{ 1, 2 });
1925 }
1926
1927 test "search supports grouped boolean and field filters" {
1928 var tmp = std.testing.tmpDir(.{});
1929 defer tmp.cleanup();
1930
1931 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1932 .paths = .{ .database = "search.db", .wal = "search.wal" },
1933 .header = testingHeader(),
1934 });
1935 defer database.deinit();
1936 try database.reserve(.{ .wal_frames = 512 });
1937
1938 const fields = [_][]const u8{ "name_tokens", "docstring" };
1939 var search = try Search.open(&database, .{ .field_names = &fields });
1940 const truncate_doc = try std.fmt.allocPrint(std.testing.allocator, "truncate{c}doc body", .{token_mod.field_separator});
1941 defer std.testing.allocator.free(truncate_doc);
1942 const format_doc = try std.fmt.allocPrint(std.testing.allocator, "format output{c}Format output for display", .{token_mod.field_separator});
1943 defer std.testing.allocator.free(format_doc);
1944 const path_doc = try std.fmt.allocPrint(std.testing.allocator, "truncate path{c}Drop path components after max depth", .{token_mod.field_separator});
1945 defer std.testing.allocator.free(path_doc);
1946 const helpers_doc = try std.fmt.allocPrint(std.testing.allocator, "helpers truncate{c}Trim a string", .{token_mod.field_separator});
1947 defer std.testing.allocator.free(helpers_doc);
1948 _ = try search.put(std.testing.allocator, 1, truncate_doc, .{ .durability = .buffered });
1949 _ = try search.put(std.testing.allocator, 2, format_doc, .{ .durability = .buffered });
1950 _ = try search.put(std.testing.allocator, 3, path_doc, .{ .durability = .buffered });
1951 _ = try search.put(std.testing.allocator, 4, helpers_doc, .{ .durability = .buffered });
1952
1953 var name_filter = try search.query(std.testing.allocator, "name_tokens:truncate", 0);
1954 defer name_filter.deinit();
1955 try expectHits(name_filter.hits, &.{ 1, 3, 4 });
1956
1957 var doc_filter = try search.query(std.testing.allocator, "docstring:max", 0);
1958 defer doc_filter.deinit();
1959 try expectHits(doc_filter.hits, &.{3});
1960
1961 var group_left = try search.query(std.testing.allocator, "(truncate OR output) AND format", 0);
1962 defer group_left.deinit();
1963 try expectHits(group_left.hits, &.{2});
1964
1965 var group_right = try search.query(std.testing.allocator, "truncate AND (path OR helpers)", 0);
1966 defer group_right.deinit();
1967 try expectHits(group_right.hits, &.{ 3, 4 });
1968
1969 var field_group = try search.query(std.testing.allocator, "name_tokens:(path OR helpers) AND docstring:Trim", 0);
1970 defer field_group.deinit();
1971 try expectHits(field_group.hits, &.{4});
1972 }
1973
1974 test "search negative field atoms subtract only field matches" {
1975 var tmp = std.testing.tmpDir(.{});
1976 defer tmp.cleanup();
1977
1978 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
1979 .paths = .{ .database = "search.db", .wal = "search.wal" },
1980 .header = testingHeader(),
1981 });
1982 defer database.deinit();
1983 try database.reserve(.{ .wal_frames = 512 });
1984
1985 const fields = [_][]const u8{ "title", "body" };
1986 var search = try Search.open(&database, .{ .field_names = &fields });
1987 const beta_body = try std.fmt.allocPrint(std.testing.allocator, "alpha{c}beta", .{token_mod.field_separator});
1988 defer std.testing.allocator.free(beta_body);
1989 const beta_title = try std.fmt.allocPrint(std.testing.allocator, "alpha beta{c}", .{token_mod.field_separator});
1990 defer std.testing.allocator.free(beta_title);
1991 const betray_body = try std.fmt.allocPrint(std.testing.allocator, "alpha{c}betray", .{token_mod.field_separator});
1992 defer std.testing.allocator.free(betray_body);
1993 _ = try search.put(std.testing.allocator, 1, beta_body, .{ .durability = .buffered });
1994 _ = try search.put(std.testing.allocator, 2, beta_title, .{ .durability = .buffered });
1995 _ = try search.put(std.testing.allocator, 3, betray_body, .{ .durability = .buffered });
1996
1997 var negative_term = try search.query(std.testing.allocator, "alpha NOT title:beta", 0);
1998 defer negative_term.deinit();
1999 try expectHits(negative_term.hits, &.{ 1, 3 });
2000
2001 var negative_prefix = try search.query(std.testing.allocator, "alpha NOT body:bet*", 0);
2002 defer negative_prefix.deinit();
2003 try expectHits(negative_prefix.hits, &.{2});
2004
2005 var pure_negative = try search.query(std.testing.allocator, "NOT body:beta", 0);
2006 defer pure_negative.deinit();
2007 try expectHits(pure_negative.hits, &.{ 2, 3 });
2008 }
2009
2010 test "search ranked field clauses verify candidates and keep negative matches" {
2011 var tmp = std.testing.tmpDir(.{});
2012 defer tmp.cleanup();
2013
2014 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2015 .paths = .{ .database = "search.db", .wal = "search.wal" },
2016 .header = testingHeader(),
2017 });
2018 defer database.deinit();
2019 try database.reserve(.{ .wal_frames = 512 });
2020
2021 const fields = [_][]const u8{ "title", "body" };
2022 var search = try Search.open(&database, .{ .field_names = &fields });
2023 const title_pair = try std.fmt.allocPrint(std.testing.allocator, "Alpha Beta{c}gamma", .{token_mod.field_separator});
2024 defer std.testing.allocator.free(title_pair);
2025 const split_pair = try std.fmt.allocPrint(std.testing.allocator, "alpha{c}beta", .{token_mod.field_separator});
2026 defer std.testing.allocator.free(split_pair);
2027 const gamma_only = try std.fmt.allocPrint(std.testing.allocator, "gamma{c}", .{token_mod.field_separator});
2028 defer std.testing.allocator.free(gamma_only);
2029 _ = try search.put(std.testing.allocator, 1, title_pair, .{ .durability = .buffered });
2030 _ = try search.put(std.testing.allocator, 2, split_pair, .{ .durability = .buffered });
2031 _ = try search.put(std.testing.allocator, 3, gamma_only, .{ .durability = .buffered });
2032
2033 var field_clause = try search.queryRanked(std.testing.allocator, "title:alpha body:beta", 0, .{});
2034 defer field_clause.deinit();
2035 try expectRankedHits(field_clause.hits, &.{2});
2036 try std.testing.expectEqual(@as(usize, 1), field_clause.total);
2037
2038 var field_prefix = try search.queryRanked(std.testing.allocator, "title:bet*", 0, .{});
2039 defer field_prefix.deinit();
2040 try expectRankedHits(field_prefix.hits, &.{1});
2041 try std.testing.expectEqual(@as(usize, 1), field_prefix.total);
2042
2043 var negative_clause = try search.queryRanked(std.testing.allocator, "beta OR NOT alpha", 0, .{});
2044 defer negative_clause.deinit();
2045 try expectRankedHits(negative_clause.hits, &.{ 2, 1, 3 });
2046 try std.testing.expectEqual(@as(usize, 3), negative_clause.total);
2047 try std.testing.expectEqual(@as(f64, 0.0), negative_clause.hits[2].rank);
2048 }
2049
2050 test "search prepared query reuses parsed exact clauses" {
2051 var tmp = std.testing.tmpDir(.{});
2052 defer tmp.cleanup();
2053
2054 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2055 .paths = .{ .database = "search.db", .wal = "search.wal" },
2056 .header = testingHeader(),
2057 });
2058 defer database.deinit();
2059 try database.reserve(.{ .wal_frames = 512 });
2060
2061 var search = try Search.open(&database, .{ .index_prefixes = false });
2062 _ = try search.put(std.testing.allocator, 1, "sql cache cache", .{ .durability = .buffered });
2063 _ = try search.put(std.testing.allocator, 2, "sql query", .{ .durability = .buffered });
2064 _ = try search.put(std.testing.allocator, 3, "cache only", .{ .durability = .buffered });
2065
2066 var prepared = try search.prepare(std.testing.allocator, "sql AND cache");
2067 defer prepared.deinit(std.testing.allocator);
2068
2069 var first = try prepared.execute(&search, std.testing.allocator, 0);
2070 defer first.deinit();
2071 try expectHits(first.hits, &.{1});
2072 try std.testing.expectEqual(@as(usize, 3), first.hits[0].score);
2073
2074 var second = try prepared.execute(&search, std.testing.allocator, 1);
2075 defer second.deinit();
2076 try expectHits(second.hits, &.{1});
2077 }
2078
2079 test "search limited results keep total" {
2080 var tmp = std.testing.tmpDir(.{});
2081 defer tmp.cleanup();
2082
2083 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2084 .paths = .{ .database = "search.db", .wal = "search.wal" },
2085 .header = testingHeader(),
2086 });
2087 defer database.deinit();
2088 try database.reserve(.{ .wal_frames = 512 });
2089
2090 var search = try Search.open(&database, .{ .index_prefixes = false });
2091 _ = try search.put(std.testing.allocator, 1, "cache cache alpha", .{ .durability = .buffered });
2092 _ = try search.put(std.testing.allocator, 2, "cache beta", .{ .durability = .buffered });
2093 _ = try search.put(std.testing.allocator, 3, "gamma", .{ .durability = .buffered });
2094
2095 var limited = try search.query(std.testing.allocator, "cache", 1);
2096 defer limited.deinit();
2097 try std.testing.expectEqual(@as(usize, 2), limited.total);
2098 try expectHits(limited.hits, &.{1});
2099
2100 var full = try search.query(std.testing.allocator, "cache", 0);
2101 defer full.deinit();
2102 try std.testing.expectEqual(@as(usize, 2), full.total);
2103 try expectHits(full.hits, &.{ 1, 2 });
2104 }
2105
2106 test "search subtracts negative doclists before scoring" {
2107 var tmp = std.testing.tmpDir(.{});
2108 defer tmp.cleanup();
2109
2110 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2111 .paths = .{ .database = "search.db", .wal = "search.wal" },
2112 .header = testingHeader(),
2113 });
2114 defer database.deinit();
2115 try database.reserve(.{ .wal_frames = 512 });
2116
2117 var search = try Search.open(&database, .{});
2118 _ = try search.put(std.testing.allocator, 1, "alpha beta", .{ .durability = .buffered });
2119 _ = try search.put(std.testing.allocator, 2, "alpha gamma", .{ .durability = .buffered });
2120 _ = try search.put(std.testing.allocator, 3, "delta gamma", .{ .durability = .buffered });
2121
2122 var positive = try search.query(std.testing.allocator, "alpha NOT beta", 0);
2123 defer positive.deinit();
2124 try expectHits(positive.hits, &.{2});
2125
2126 var only_negative = try search.query(std.testing.allocator, "NOT gamma", 0);
2127 defer only_negative.deinit();
2128 try expectHits(only_negative.hits, &.{1});
2129 }
2130
2131 test "search exact-only prefix rows remain sorted for conjunctions" {
2132 var tmp = std.testing.tmpDir(.{});
2133 defer tmp.cleanup();
2134
2135 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2136 .paths = .{ .database = "search.db", .wal = "search.wal" },
2137 .header = testingHeader(),
2138 });
2139 defer database.deinit();
2140 try database.reserve(.{ .wal_frames = 512 });
2141
2142 var search = try Search.open(&database, .{ .index_prefixes = false });
2143 _ = try search.put(std.testing.allocator, 1, "authz check", .{ .durability = .buffered });
2144 _ = try search.put(std.testing.allocator, 2, "authn skip", .{ .durability = .buffered });
2145 _ = try search.put(std.testing.allocator, 3, "check only", .{ .durability = .buffered });
2146 _ = try search.put(std.testing.allocator, 4, "authn check", .{ .durability = .buffered });
2147
2148 var results = try search.query(std.testing.allocator, "auth* AND check", 0);
2149 defer results.deinit();
2150 try expectHits(results.hits, &.{ 1, 4 });
2151 }
2152
2153 test "search phrase matching preserves skipped token gaps" {
2154 var tmp = std.testing.tmpDir(.{});
2155 defer tmp.cleanup();
2156
2157 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2158 .paths = .{ .database = "search.db", .wal = "search.wal" },
2159 .header = testingHeader(),
2160 });
2161 defer database.deinit();
2162 try database.reserve(.{ .wal_frames = 512 });
2163
2164 var text: std.ArrayList(u8) = .empty;
2165 defer text.deinit(std.testing.allocator);
2166 try text.appendSlice(std.testing.allocator, "alpha ");
2167 for (0..token_mod.max_token_bytes + 1) |_| try text.append(std.testing.allocator, 'x');
2168 try text.appendSlice(std.testing.allocator, " beta");
2169
2170 var search = try Search.open(&database, .{ .index_phrase_pairs = true });
2171 _ = try search.put(std.testing.allocator, 1, text.items, .{ .durability = .buffered });
2172
2173 var exact = try search.query(std.testing.allocator, "alpha beta", 0);
2174 defer exact.deinit();
2175 try expectHits(exact.hits, &.{1});
2176
2177 var phrase = try search.query(std.testing.allocator, "\"alpha beta\"", 0);
2178 defer phrase.deinit();
2179 try expectHits(phrase.hits, &.{});
2180 }
2181
2182 test "search replacement and delete update postings atomically" {
2183 var tmp = std.testing.tmpDir(.{});
2184 defer tmp.cleanup();
2185
2186 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2187 .paths = .{ .database = "search.db", .wal = "search.wal" },
2188 .header = testingHeader(),
2189 });
2190 defer database.deinit();
2191 try database.reserve(.{ .wal_frames = 512 });
2192
2193 var search = try Search.open(&database, .{ .index_phrase_pairs = true });
2194 _ = try search.put(std.testing.allocator, 1, "old token", .{ .durability = .buffered });
2195 _ = try search.put(std.testing.allocator, 1, "new token", .{ .durability = .buffered });
2196
2197 var old = try search.query(std.testing.allocator, "old", 0);
2198 defer old.deinit();
2199 try expectHits(old.hits, &.{});
2200
2201 var old_phrase = try search.query(std.testing.allocator, "\"old token\"", 0);
2202 defer old_phrase.deinit();
2203 try expectHits(old_phrase.hits, &.{});
2204
2205 var new = try search.query(std.testing.allocator, "new", 0);
2206 defer new.deinit();
2207 try expectHits(new.hits, &.{1});
2208
2209 var new_phrase = try search.query(std.testing.allocator, "\"new token\"", 0);
2210 defer new_phrase.deinit();
2211 try expectHits(new_phrase.hits, &.{1});
2212
2213 _ = try search.delete(std.testing.allocator, 1, .{ .durability = .buffered });
2214 var after = try search.query(std.testing.allocator, "new", 0);
2215 defer after.deinit();
2216 try expectHits(after.hits, &.{});
2217
2218 var after_phrase = try search.query(std.testing.allocator, "\"new token\"", 0);
2219 defer after_phrase.deinit();
2220 try expectHits(after_phrase.hits, &.{});
2221 }
2222
2223 test "search replacement tolerates missing old postings" {
2224 var tmp = std.testing.tmpDir(.{});
2225 defer tmp.cleanup();
2226
2227 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2228 .paths = .{ .database = "search.db", .wal = "search.wal" },
2229 .header = testingHeader(),
2230 });
2231 defer database.deinit();
2232 try database.reserve(.{ .wal_frames = 512 });
2233
2234 var search = try Search.open(&database, .{});
2235 _ = try search.put(std.testing.allocator, 1, "old token", .{ .durability = .buffered });
2236 const key_text = try posting_mod.postingKey(std.testing.allocator, '=', "old");
2237 defer std.testing.allocator.free(key_text);
2238 _ = try search.terms.delete(1, &posting_mod.postingValues(key_text), .{ .durability = .buffered });
2239 _ = try search.put(std.testing.allocator, 1, "new token", .{ .durability = .buffered });
2240
2241 var old = try search.query(std.testing.allocator, "old", 0);
2242 defer old.deinit();
2243 try expectHits(old.hits, &.{});
2244
2245 var new = try search.query(std.testing.allocator, "new", 0);
2246 defer new.deinit();
2247 try expectHits(new.hits, &.{1});
2248 }
2249
2250 test "search exact ranking uses posting term frequencies" {
2251 var tmp = std.testing.tmpDir(.{});
2252 defer tmp.cleanup();
2253
2254 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2255 .paths = .{ .database = "search.db", .wal = "search.wal" },
2256 .header = testingHeader(),
2257 });
2258 defer database.deinit();
2259 try database.reserve(.{ .wal_frames = 512 });
2260
2261 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
2262 _ = try search.put(std.testing.allocator, 1, "alpha beta", .{ .durability = .buffered });
2263 _ = try search.put(std.testing.allocator, 2, "alpha alpha beta", .{ .durability = .buffered });
2264
2265 var exact = try search.query(std.testing.allocator, "alpha", 0);
2266 defer exact.deinit();
2267 try expectHits(exact.hits, &.{ 2, 1 });
2268 try std.testing.expectEqual(@as(usize, 2), exact.hits[0].score);
2269
2270 var conjunction = try search.query(std.testing.allocator, "alpha AND beta", 0);
2271 defer conjunction.deinit();
2272 try expectHits(conjunction.hits, &.{ 2, 1 });
2273 try std.testing.expectEqual(@as(usize, 3), conjunction.hits[0].score);
2274 }
2275
2276 test "search ranked query boosts rare exact terms over common frequency" {
2277 var tmp = std.testing.tmpDir(.{});
2278 defer tmp.cleanup();
2279
2280 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2281 .paths = .{ .database = "search.db", .wal = "search.wal" },
2282 .header = testingHeader(),
2283 });
2284 defer database.deinit();
2285 try database.reserve(.{ .wal_frames = 512 });
2286
2287 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
2288 _ = try search.put(std.testing.allocator, 1, "common common common common common common common common", .{ .durability = .buffered });
2289 _ = try search.put(std.testing.allocator, 2, "common common", .{ .durability = .buffered });
2290 _ = try search.put(std.testing.allocator, 3, "common rare", .{ .durability = .buffered });
2291
2292 var ranked = try search.queryRanked(std.testing.allocator, "common OR rare", 0, .{});
2293 defer ranked.deinit();
2294 try std.testing.expectEqual(@as(usize, 3), ranked.total);
2295 try expectRankedHits(ranked.hits, &.{ 3, 1, 2 });
2296
2297 var prepared = try search.prepare(std.testing.allocator, "common OR rare");
2298 defer prepared.deinit(std.testing.allocator);
2299 var prepared_ranked = try prepared.executeRanked(&search, std.testing.allocator, 2, .{});
2300 defer prepared_ranked.deinit();
2301 try std.testing.expectEqual(@as(usize, 3), prepared_ranked.total);
2302 try expectRankedHits(prepared_ranked.hits, &.{ 3, 1 });
2303 }
2304
2305 test "search ranked query normalizes document length" {
2306 var tmp = std.testing.tmpDir(.{});
2307 defer tmp.cleanup();
2308
2309 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2310 .paths = .{ .database = "search.db", .wal = "search.wal" },
2311 .header = testingHeader(),
2312 });
2313 defer database.deinit();
2314 try database.reserve(.{ .wal_frames = 512 });
2315
2316 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
2317 _ = try search.put(std.testing.allocator, 1, "needle", .{ .durability = .buffered });
2318 _ = try search.put(std.testing.allocator, 2, "needle filler filler filler filler filler filler filler", .{ .durability = .buffered });
2319
2320 var ranked = try search.queryRanked(std.testing.allocator, "needle", 0, .{});
2321 defer ranked.deinit();
2322 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2323 try expectRankedHits(ranked.hits, &.{ 1, 2 });
2324 try std.testing.expect(ranked.hits[0].rank < ranked.hits[1].rank);
2325 }
2326
2327 test "search ranked phrase excludes token intersection false positives" {
2328 var tmp = std.testing.tmpDir(.{});
2329 defer tmp.cleanup();
2330
2331 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2332 .paths = .{ .database = "search.db", .wal = "search.wal" },
2333 .header = testingHeader(),
2334 });
2335 defer database.deinit();
2336 try database.reserve(.{ .wal_frames = 512 });
2337
2338 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
2339 _ = try search.put(std.testing.allocator, 1, "before alpha beta after", .{ .durability = .buffered });
2340 _ = try search.put(std.testing.allocator, 2, "before alpha gap beta after", .{ .durability = .buffered });
2341
2342 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2343 defer ranked.deinit();
2344 try std.testing.expectEqual(@as(usize, 1), ranked.total);
2345 try expectRankedHits(ranked.hits, &.{1});
2346 }
2347
2348 test "search ranked phrase pair uses posting frequency" {
2349 var tmp = std.testing.tmpDir(.{});
2350 defer tmp.cleanup();
2351
2352 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2353 .paths = .{ .database = "search.db", .wal = "search.wal" },
2354 .header = testingHeader(),
2355 });
2356 defer database.deinit();
2357 try database.reserve(.{ .wal_frames = 512 });
2358
2359 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
2360 _ = try search.putAll(std.testing.allocator, &.{
2361 .{ .rowid = 1, .text = "alpha beta" },
2362 .{ .rowid = 2, .text = "alpha beta alpha beta" },
2363 .{ .rowid = 3, .text = "alpha slow beta" },
2364 }, .{ .durability = .buffered });
2365
2366 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2367 defer ranked.deinit();
2368 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2369 try expectRankedHits(ranked.hits, &.{ 2, 1 });
2370 }
2371
2372 test "search phrase pair singleton omission verifies candidates" {
2373 var tmp = std.testing.tmpDir(.{});
2374 defer tmp.cleanup();
2375
2376 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2377 .paths = .{ .database = "search.db", .wal = "search.wal" },
2378 .header = testingHeader(),
2379 });
2380 defer database.deinit();
2381 try database.reserve(.{ .wal_frames = 512 });
2382
2383 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true, .omit_phrase_pair_singletons = true });
2384 _ = try search.loadAllNew(std.testing.allocator, &.{
2385 .{ .rowid = 1, .text = "alpha beta" },
2386 .{ .rowid = 2, .text = "alpha gap beta" },
2387 .{ .rowid = 3, .text = "beta alpha" },
2388 }, .{ .durability = .buffered });
2389
2390 const stats = try search.postingStats(std.testing.allocator);
2391 try std.testing.expect((try search.capabilities(std.testing.allocator)).phrase_pair_singletons_omitted);
2392 try std.testing.expectEqual(@as(usize, 0), stats.phrase_pair_direct_posting_entries);
2393 try std.testing.expectEqual(@as(usize, 0), stats.phrase_pair_segment_entries);
2394
2395 var phrase = try search.query(std.testing.allocator, "\"alpha beta\"", 0);
2396 defer phrase.deinit();
2397 try std.testing.expectEqual(@as(usize, 1), phrase.total);
2398 try expectHits(phrase.hits, &.{1});
2399
2400 var negative = try search.query(std.testing.allocator, "alpha NOT \"alpha beta\"", 0);
2401 defer negative.deinit();
2402 try std.testing.expectEqual(@as(usize, 2), negative.total);
2403 try expectHits(negative.hits, &.{ 2, 3 });
2404
2405 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2406 defer ranked.deinit();
2407 try std.testing.expectEqual(@as(usize, 1), ranked.total);
2408 try expectRankedHits(ranked.hits, &.{1});
2409 }
2410
2411 test "search phrase pair singleton omission segments repeated pairs" {
2412 var tmp = std.testing.tmpDir(.{});
2413 defer tmp.cleanup();
2414
2415 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2416 .paths = .{ .database = "search.db", .wal = "search.wal" },
2417 .header = testingHeader(),
2418 });
2419 defer database.deinit();
2420 try database.reserve(.{ .wal_frames = 512 });
2421
2422 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true, .omit_phrase_pair_singletons = true });
2423 _ = try search.loadAllNew(std.testing.allocator, &.{
2424 .{ .rowid = 1, .text = "alpha beta once" },
2425 .{ .rowid = 2, .text = "alpha beta alpha beta twice" },
2426 .{ .rowid = 3, .text = "alpha gap beta" },
2427 }, .{ .durability = .buffered });
2428
2429 const stats = try search.postingStats(std.testing.allocator);
2430 try std.testing.expectEqual(@as(usize, 0), stats.phrase_pair_direct_posting_entries);
2431 try std.testing.expectEqual(@as(usize, 1), stats.phrase_pair_segment_entries);
2432 try std.testing.expectEqual(@as(usize, 2), stats.phrase_pair_segment_postings);
2433
2434 var phrase = try search.query(std.testing.allocator, "\"alpha beta\"", 0);
2435 defer phrase.deinit();
2436 try std.testing.expectEqual(@as(usize, 2), phrase.total);
2437 try expectHits(phrase.hits, &.{ 2, 1 });
2438
2439 var negative = try search.query(std.testing.allocator, "alpha NOT \"alpha beta\"", 0);
2440 defer negative.deinit();
2441 try std.testing.expectEqual(@as(usize, 1), negative.total);
2442 try expectHits(negative.hits, &.{3});
2443
2444 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2445 defer ranked.deinit();
2446 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2447 try expectRankedHits(ranked.hits, &.{ 2, 1 });
2448 }
2449
2450 test "search phrase pair capability persists across empty index opt in" {
2451 var tmp = std.testing.tmpDir(.{});
2452 defer tmp.cleanup();
2453
2454 {
2455 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2456 .paths = .{ .database = "search.db", .wal = "search.wal" },
2457 .header = testingHeader(),
2458 });
2459 defer database.deinit();
2460 try database.reserve(.{ .wal_frames = 512 });
2461
2462 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
2463 try std.testing.expect(!(try search.capabilities(std.testing.allocator)).phrase_pair_postings);
2464 _ = try search.putAll(std.testing.allocator, &.{
2465 .{ .rowid = 1, .text = "alpha beta" },
2466 .{ .rowid = 2, .text = "alpha beta alpha beta" },
2467 .{ .rowid = 3, .text = "alpha slow beta" },
2468 }, .{ .durability = .buffered });
2469 try std.testing.expect((try search.capabilities(std.testing.allocator)).phrase_pair_postings);
2470
2471 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2472 defer ranked.deinit();
2473 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2474 try expectRankedHits(ranked.hits, &.{ 2, 1 });
2475 try database.syncWal();
2476 }
2477
2478 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2479 .paths = .{ .database = "search.db", .wal = "search.wal" },
2480 .header = recoveredHeader(),
2481 });
2482 defer reopened.deinit();
2483 var search = try Search.open(&reopened, .{ .index_prefixes = false, .index_phrase_pairs = true });
2484 try std.testing.expect((try search.capabilities(std.testing.allocator)).phrase_pair_postings);
2485 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2486 defer ranked.deinit();
2487 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2488 try expectRankedHits(ranked.hits, &.{ 2, 1 });
2489 }
2490
2491 test "search phrase pair incremental opt in requires rebuild" {
2492 var tmp = std.testing.tmpDir(.{});
2493 defer tmp.cleanup();
2494
2495 {
2496 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2497 .paths = .{ .database = "search.db", .wal = "search.wal" },
2498 .header = testingHeader(),
2499 });
2500 defer database.deinit();
2501 try database.reserve(.{ .wal_frames = 512 });
2502
2503 var search = try Search.open(&database, .{ .index_prefixes = false });
2504 _ = try search.put(std.testing.allocator, 1, "alpha beta", .{ .durability = .buffered });
2505 try database.syncWal();
2506 }
2507
2508 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2509 .paths = .{ .database = "search.db", .wal = "search.wal" },
2510 .header = recoveredHeader(),
2511 });
2512 defer reopened.deinit();
2513 var search = try Search.open(&reopened, .{ .index_prefixes = false, .index_phrase_pairs = true });
2514 try std.testing.expectError(error.SearchIndexRebuildRequired, search.put(std.testing.allocator, 2, "alpha beta alpha beta", .{ .durability = .buffered }));
2515 try std.testing.expect(!(try search.capabilities(std.testing.allocator)).phrase_pair_postings);
2516
2517 _ = try search.loadAll(std.testing.allocator, &.{
2518 .{ .rowid = 1, .text = "alpha beta" },
2519 .{ .rowid = 2, .text = "alpha beta alpha beta" },
2520 }, .{ .durability = .buffered });
2521 try std.testing.expect((try search.capabilities(std.testing.allocator)).phrase_pair_postings);
2522 var ranked = try search.queryRanked(std.testing.allocator, "\"alpha beta\"", 0, .{});
2523 defer ranked.deinit();
2524 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2525 try expectRankedHits(ranked.hits, &.{ 2, 1 });
2526 }
2527
2528 test "search ranked prefix uses prefix posting frequency" {
2529 var tmp = std.testing.tmpDir(.{});
2530 defer tmp.cleanup();
2531
2532 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2533 .paths = .{ .database = "search.db", .wal = "search.wal" },
2534 .header = testingHeader(),
2535 });
2536 defer database.deinit();
2537 try database.reserve(.{ .wal_frames = 512 });
2538
2539 var search = try Search.open(&database, .{ .index_prefixes = false });
2540 _ = try search.put(std.testing.allocator, 1, "authentication filler", .{ .durability = .buffered });
2541 _ = try search.put(std.testing.allocator, 2, "authentication authorization", .{ .durability = .buffered });
2542 _ = try search.put(std.testing.allocator, 3, "allocator filler", .{ .durability = .buffered });
2543
2544 var ranked = try search.queryRanked(std.testing.allocator, "auth*", 0, .{});
2545 defer ranked.deinit();
2546 try std.testing.expectEqual(@as(usize, 2), ranked.total);
2547 try expectRankedHits(ranked.hits, &.{ 2, 1 });
2548 try std.testing.expect(ranked.hits[0].rank < ranked.hits[1].rank);
2549 }
2550
2551 test "search ranked query keeps stats valid across load replace and delete" {
2552 var tmp = std.testing.tmpDir(.{});
2553 defer tmp.cleanup();
2554
2555 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2556 .paths = .{ .database = "search.db", .wal = "search.wal" },
2557 .header = testingHeader(),
2558 });
2559 defer database.deinit();
2560 try database.reserve(.{ .wal_frames = 512 });
2561
2562 var search = try Search.open(&database, .{ .index_prefixes = false });
2563 _ = try search.loadAll(std.testing.allocator, &.{
2564 .{ .rowid = 1, .text = "needle" },
2565 .{ .rowid = 2, .text = "needle filler filler filler" },
2566 }, .{ .durability = .buffered });
2567 const loaded_lengths = try search.documentLengths(std.testing.allocator);
2568 defer std.testing.allocator.free(loaded_lengths);
2569 try std.testing.expectEqual(@as(usize, 2), loaded_lengths.len);
2570 try std.testing.expectEqual(@as(i64, 1), loaded_lengths[0].rowid);
2571 try std.testing.expectEqual(@as(usize, 1), loaded_lengths[0].length);
2572 try std.testing.expectEqual(@as(i64, 2), loaded_lengths[1].rowid);
2573 try std.testing.expectEqual(@as(usize, 4), loaded_lengths[1].length);
2574
2575 var initial = try search.queryRanked(std.testing.allocator, "needle", 0, .{});
2576 defer initial.deinit();
2577 try expectRankedHits(initial.hits, &.{ 1, 2 });
2578
2579 _ = try search.put(std.testing.allocator, 1, "filler only", .{ .durability = .buffered });
2580 try std.testing.expectEqual(@as(usize, 2), try search.documentLength(std.testing.allocator, 1));
2581 var replaced = try search.queryRanked(std.testing.allocator, "needle", 0, .{});
2582 defer replaced.deinit();
2583 try expectRankedHits(replaced.hits, &.{2});
2584
2585 _ = try search.delete(std.testing.allocator, 2, .{ .durability = .buffered });
2586 try std.testing.expectError(error.InvalidSearchIndex, search.documentLength(std.testing.allocator, 2));
2587 var deleted = try search.queryRanked(std.testing.allocator, "needle", 0, .{});
2588 defer deleted.deinit();
2589 try std.testing.expectEqual(@as(usize, 0), deleted.total);
2590 try std.testing.expectEqual(@as(usize, 0), deleted.hits.len);
2591 }
2592
2593 test "search score band keeps exact term cutoff ties" {
2594 var tmp = std.testing.tmpDir(.{});
2595 defer tmp.cleanup();
2596
2597 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2598 .paths = .{ .database = "search.db", .wal = "search.wal" },
2599 .header = testingHeader(),
2600 });
2601 defer database.deinit();
2602 try database.reserve(.{ .wal_frames = 512 });
2603
2604 var search = try Search.open(&database, .{ .index_prefixes = false });
2605 _ = try search.put(std.testing.allocator, 1, "cache", .{ .durability = .buffered });
2606 _ = try search.put(std.testing.allocator, 2, "cache cache", .{ .durability = .buffered });
2607 _ = try search.put(std.testing.allocator, 3, "cache", .{ .durability = .buffered });
2608 _ = try search.put(std.testing.allocator, 4, "cache cache cache", .{ .durability = .buffered });
2609 _ = try search.put(std.testing.allocator, 5, "cache cache", .{ .durability = .buffered });
2610
2611 var band = try search.queryScoreBand(std.testing.allocator, "cache", 2);
2612 defer band.deinit();
2613 try std.testing.expectEqual(@as(usize, 5), band.total);
2614 try expectHits(band.hits, &.{ 4, 2, 5 });
2615 try std.testing.expectEqual(@as(usize, 3), band.hits[0].score);
2616 try std.testing.expectEqual(@as(usize, 2), band.hits[1].score);
2617 try std.testing.expectEqual(@as(usize, 2), band.hits[2].score);
2618
2619 var prepared = try search.prepare(std.testing.allocator, "cache");
2620 defer prepared.deinit(std.testing.allocator);
2621 var prepared_band = try prepared.executeScoreBand(&search, std.testing.allocator, 2);
2622 defer prepared_band.deinit();
2623 try expectHits(prepared_band.hits, &.{ 4, 2, 5 });
2624 }
2625
2626 test "search score band streams segmented exact term cutoff ties" {
2627 var tmp = std.testing.tmpDir(.{});
2628 defer tmp.cleanup();
2629
2630 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2631 .paths = .{ .database = "search.db", .wal = "search.wal" },
2632 .header = testingHeader(),
2633 });
2634 defer database.deinit();
2635 try database.reserve(.{ .wal_frames = 512 });
2636
2637 var search = try Search.open(&database, .{ .index_prefixes = false });
2638 _ = try search.loadAll(std.testing.allocator, &.{
2639 .{ .rowid = 1, .text = "cache" },
2640 .{ .rowid = 2, .text = "cache cache" },
2641 .{ .rowid = 3, .text = "cache" },
2642 .{ .rowid = 4, .text = "cache cache cache" },
2643 .{ .rowid = 5, .text = "cache cache" },
2644 }, .{ .durability = .buffered });
2645
2646 var band = try search.queryScoreBand(std.testing.allocator, "cache", 2);
2647 defer band.deinit();
2648 try std.testing.expectEqual(@as(usize, 5), band.total);
2649 try expectHits(band.hits, &.{ 4, 2, 5 });
2650 try std.testing.expectEqual(@as(usize, 3), band.hits[0].score);
2651 try std.testing.expectEqual(@as(usize, 2), band.hits[1].score);
2652 try std.testing.expectEqual(@as(usize, 2), band.hits[2].score);
2653 }
2654
2655 test "search OR unions exact hits and sums duplicate scores" {
2656 var tmp = std.testing.tmpDir(.{});
2657 defer tmp.cleanup();
2658
2659 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2660 .paths = .{ .database = "search.db", .wal = "search.wal" },
2661 .header = testingHeader(),
2662 });
2663 defer database.deinit();
2664 try database.reserve(.{ .wal_frames = 512 });
2665
2666 var search = try Search.open(&database, .{ .index_prefixes = false });
2667 _ = try search.put(std.testing.allocator, 1, "alpha beta beta", .{ .durability = .buffered });
2668 _ = try search.put(std.testing.allocator, 2, "alpha", .{ .durability = .buffered });
2669 _ = try search.put(std.testing.allocator, 3, "beta", .{ .durability = .buffered });
2670
2671 var disjunction = try search.query(std.testing.allocator, "alpha OR beta", 0);
2672 defer disjunction.deinit();
2673 try std.testing.expectEqual(@as(usize, 3), disjunction.total);
2674 try expectHits(disjunction.hits, &.{ 1, 2, 3 });
2675 try std.testing.expectEqual(@as(usize, 3), disjunction.hits[0].score);
2676
2677 var limited = try search.query(std.testing.allocator, "alpha OR beta", 2);
2678 defer limited.deinit();
2679 try std.testing.expectEqual(@as(usize, 3), limited.total);
2680 try expectHits(limited.hits, &.{ 1, 2 });
2681 try std.testing.expectEqual(@as(usize, 3), limited.hits[0].score);
2682
2683 var duplicate = try search.query(std.testing.allocator, "alpha OR alpha", 0);
2684 defer duplicate.deinit();
2685 try std.testing.expectEqual(@as(usize, 2), duplicate.total);
2686 try expectHits(duplicate.hits, &.{ 1, 2 });
2687 try std.testing.expectEqual(@as(usize, 2), duplicate.hits[0].score);
2688 }
2689
2690 test "search score band keeps OR cutoff ties" {
2691 var tmp = std.testing.tmpDir(.{});
2692 defer tmp.cleanup();
2693
2694 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2695 .paths = .{ .database = "search.db", .wal = "search.wal" },
2696 .header = testingHeader(),
2697 });
2698 defer database.deinit();
2699 try database.reserve(.{ .wal_frames = 512 });
2700
2701 var search = try Search.open(&database, .{ .index_prefixes = false });
2702 _ = try search.put(std.testing.allocator, 1, "alpha", .{ .durability = .buffered });
2703 _ = try search.put(std.testing.allocator, 2, "alpha alpha", .{ .durability = .buffered });
2704 _ = try search.put(std.testing.allocator, 3, "beta beta", .{ .durability = .buffered });
2705 _ = try search.put(std.testing.allocator, 4, "alpha beta beta", .{ .durability = .buffered });
2706
2707 var band = try search.queryScoreBand(std.testing.allocator, "alpha OR beta", 2);
2708 defer band.deinit();
2709 try std.testing.expectEqual(@as(usize, 4), band.total);
2710 try expectHits(band.hits, &.{ 4, 2, 3 });
2711 try std.testing.expectEqual(@as(usize, 3), band.hits[0].score);
2712 try std.testing.expectEqual(@as(usize, 2), band.hits[1].score);
2713 try std.testing.expectEqual(@as(usize, 2), band.hits[2].score);
2714 }
2715
2716 test "search limited OR keeps best hits before rowid order" {
2717 var tmp = std.testing.tmpDir(.{});
2718 defer tmp.cleanup();
2719
2720 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2721 .paths = .{ .database = "search.db", .wal = "search.wal" },
2722 .header = testingHeader(),
2723 });
2724 defer database.deinit();
2725 try database.reserve(.{ .wal_frames = 512 });
2726
2727 var search = try Search.open(&database, .{ .index_prefixes = false });
2728 _ = try search.put(std.testing.allocator, 1, "alpha", .{ .durability = .buffered });
2729 _ = try search.put(std.testing.allocator, 2, "beta beta beta", .{ .durability = .buffered });
2730 _ = try search.put(std.testing.allocator, 3, "alpha alpha", .{ .durability = .buffered });
2731
2732 var limited = try search.query(std.testing.allocator, "alpha OR beta", 2);
2733 defer limited.deinit();
2734 try std.testing.expectEqual(@as(usize, 3), limited.total);
2735 try expectHits(limited.hits, &.{ 2, 3 });
2736 try std.testing.expectEqual(@as(usize, 3), limited.hits[0].score);
2737 try std.testing.expectEqual(@as(usize, 2), limited.hits[1].score);
2738 }
2739
2740 test "search limited OR streams multiple exact terms" {
2741 var tmp = std.testing.tmpDir(.{});
2742 defer tmp.cleanup();
2743
2744 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2745 .paths = .{ .database = "search.db", .wal = "search.wal" },
2746 .header = testingHeader(),
2747 });
2748 defer database.deinit();
2749 try database.reserve(.{ .wal_frames = 512 });
2750
2751 var search = try Search.open(&database, .{ .index_prefixes = false });
2752 _ = try search.put(std.testing.allocator, 1, "alpha gamma gamma", .{ .durability = .buffered });
2753 _ = try search.put(std.testing.allocator, 2, "beta beta beta", .{ .durability = .buffered });
2754 _ = try search.put(std.testing.allocator, 3, "alpha alpha", .{ .durability = .buffered });
2755 _ = try search.put(std.testing.allocator, 4, "delta delta delta delta", .{ .durability = .buffered });
2756 _ = try search.put(std.testing.allocator, 5, "gamma beta", .{ .durability = .buffered });
2757
2758 var limited = try search.query(std.testing.allocator, "alpha OR beta OR gamma OR delta", 3);
2759 defer limited.deinit();
2760 try std.testing.expectEqual(@as(usize, 5), limited.total);
2761 try expectHits(limited.hits, &.{ 4, 1, 2 });
2762 try std.testing.expectEqual(@as(usize, 4), limited.hits[0].score);
2763 try std.testing.expectEqual(@as(usize, 3), limited.hits[1].score);
2764 try std.testing.expectEqual(@as(usize, 3), limited.hits[2].score);
2765
2766 var duplicate = try search.query(std.testing.allocator, "alpha OR beta OR alpha", 2);
2767 defer duplicate.deinit();
2768 try std.testing.expectEqual(@as(usize, 4), duplicate.total);
2769 try expectHits(duplicate.hits, &.{ 3, 2 });
2770 try std.testing.expectEqual(@as(usize, 4), duplicate.hits[0].score);
2771 try std.testing.expectEqual(@as(usize, 3), duplicate.hits[1].score);
2772 }
2773
2774 test "search limited OR streams segmented terms with direct tombstones" {
2775 var tmp = std.testing.tmpDir(.{});
2776 defer tmp.cleanup();
2777
2778 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2779 .paths = .{ .database = "search.db", .wal = "search.wal" },
2780 .header = testingHeader(),
2781 });
2782 defer database.deinit();
2783 try database.reserve(.{ .wal_frames = 1024 });
2784
2785 var search = try Search.open(&database, .{ .index_prefixes = false });
2786 _ = try search.loadAllNew(std.testing.allocator, &.{
2787 .{ .rowid = 1, .text = "alpha alpha" },
2788 .{ .rowid = 2, .text = "beta beta beta" },
2789 .{ .rowid = 3, .text = "alpha beta" },
2790 }, .{ .durability = .buffered });
2791 _ = try search.loadAllNew(std.testing.allocator, &.{
2792 .{ .rowid = 4, .text = "gamma gamma gamma gamma" },
2793 .{ .rowid = 5, .text = "alpha gamma" },
2794 }, .{ .durability = .buffered });
2795 _ = try search.delete(std.testing.allocator, 3, .{ .durability = .buffered });
2796 _ = try search.put(std.testing.allocator, 2, "alpha alpha alpha replacement", .{ .durability = .buffered });
2797
2798 var limited = try search.query(std.testing.allocator, "alpha OR beta OR gamma", 2);
2799 defer limited.deinit();
2800 try std.testing.expectEqual(@as(usize, 4), limited.total);
2801 try expectHits(limited.hits, &.{ 4, 2 });
2802 try std.testing.expectEqual(@as(usize, 4), limited.hits[0].score);
2803 try std.testing.expectEqual(@as(usize, 3), limited.hits[1].score);
2804
2805 var duplicate = try search.query(std.testing.allocator, "alpha OR alpha", 2);
2806 defer duplicate.deinit();
2807 try std.testing.expectEqual(@as(usize, 3), duplicate.total);
2808 try expectHits(duplicate.hits, &.{ 2, 1 });
2809 try std.testing.expectEqual(@as(usize, 6), duplicate.hits[0].score);
2810 try std.testing.expectEqual(@as(usize, 4), duplicate.hits[1].score);
2811
2812 var band = try search.queryScoreBand(std.testing.allocator, "alpha OR beta OR gamma", 3);
2813 defer band.deinit();
2814 try std.testing.expectEqual(@as(usize, 4), band.total);
2815 try expectHits(band.hits, &.{ 4, 2, 1, 5 });
2816 try std.testing.expectEqual(@as(usize, 4), band.hits[0].score);
2817 try std.testing.expectEqual(@as(usize, 3), band.hits[1].score);
2818 try std.testing.expectEqual(@as(usize, 2), band.hits[2].score);
2819 try std.testing.expectEqual(@as(usize, 2), band.hits[3].score);
2820 }
2821
2822 test "search rejects stale posting payloads" {
2823 var tmp = std.testing.tmpDir(.{});
2824 defer tmp.cleanup();
2825
2826 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2827 .paths = .{ .database = "search.db", .wal = "search.wal" },
2828 .header = testingHeader(),
2829 });
2830 defer database.deinit();
2831 try database.reserve(.{ .wal_frames = 512 });
2832
2833 var search = try Search.open(&database, .{ .index_prefixes = false });
2834 _ = try search.put(std.testing.allocator, 1, "alpha", .{ .durability = .buffered });
2835 const key_text = try posting_mod.postingKey(std.testing.allocator, '=', "alpha");
2836 defer std.testing.allocator.free(key_text);
2837 _ = try search.terms.delete(1, &posting_mod.postingValues(key_text), .{ .durability = .buffered });
2838 _ = try search.terms.put(1, &.{ .{ .text = key_text }, .{ .integer = 1 }, .{ .integer = 1 } }, .{ .durability = .buffered });
2839
2840 try std.testing.expectError(error.InvalidSearchIndex, search.query(std.testing.allocator, "alpha", 0));
2841 }
2842
2843 test "search identical put commits no pages" {
2844 var tmp = std.testing.tmpDir(.{});
2845 defer tmp.cleanup();
2846
2847 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2848 .paths = .{ .database = "search.db", .wal = "search.wal" },
2849 .header = testingHeader(),
2850 });
2851 defer database.deinit();
2852 try database.reserve(.{ .wal_frames = 512 });
2853
2854 var search = try Search.open(&database, .{ .index_prefixes = false });
2855 const first = try search.put(std.testing.allocator, 1, "alpha beta", .{ .durability = .buffered });
2856 try std.testing.expect(first.pages > 0);
2857 const before = try search.postingStats(std.testing.allocator);
2858
2859 const second = try search.put(std.testing.allocator, 1, "alpha beta", .{ .durability = .buffered });
2860 try std.testing.expectEqual(@as(usize, 0), second.frames);
2861 try std.testing.expectEqual(@as(usize, 0), second.pages);
2862 try std.testing.expect(!second.synced);
2863 try std.testing.expectEqualDeep(before, try search.postingStats(std.testing.allocator));
2864 }
2865
2866 test "search changed put updates exact and phrase posting counts" {
2867 var tmp = std.testing.tmpDir(.{});
2868 defer tmp.cleanup();
2869
2870 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2871 .paths = .{ .database = "search.db", .wal = "search.wal" },
2872 .header = testingHeader(),
2873 });
2874 defer database.deinit();
2875 try database.reserve(.{ .wal_frames = 512 });
2876
2877 var search = try Search.open(&database, .{
2878 .index_prefixes = false,
2879 .index_phrase_pairs = true,
2880 });
2881 _ = try search.put(std.testing.allocator, 1, "alpha beta beta", .{ .durability = .buffered });
2882 _ = try search.put(std.testing.allocator, 2, "alpha beta", .{ .durability = .buffered });
2883 _ = try search.put(std.testing.allocator, 1, "alpha gamma gamma", .{ .durability = .buffered });
2884
2885 var beta = try search.query(std.testing.allocator, "beta", 0);
2886 defer beta.deinit();
2887 try expectHits(beta.hits, &.{2});
2888 var gamma = try search.query(std.testing.allocator, "gamma", 0);
2889 defer gamma.deinit();
2890 try expectHits(gamma.hits, &.{1});
2891 try std.testing.expectEqual(@as(usize, 2), gamma.hits[0].score);
2892 var phrase = try search.query(std.testing.allocator, "\"alpha gamma\"", 0);
2893 defer phrase.deinit();
2894 try expectHits(phrase.hits, &.{1});
2895 }
2896
2897 test "search bulk put stores documents in one commit" {
2898 var tmp = std.testing.tmpDir(.{});
2899 defer tmp.cleanup();
2900
2901 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2902 .paths = .{ .database = "search.db", .wal = "search.wal" },
2903 .header = testingHeader(),
2904 });
2905 defer database.deinit();
2906 try database.reserve(.{ .wal_frames = 512 });
2907
2908 var search = try Search.open(&database, .{});
2909 _ = try search.putAll(std.testing.allocator, &.{
2910 .{ .rowid = 1, .text = "alpha beta" },
2911 .{ .rowid = 2, .text = "alpha gamma" },
2912 .{ .rowid = 3, .text = "delta" },
2913 }, .{ .durability = .buffered });
2914
2915 var alpha = try search.query(std.testing.allocator, "alpha", 0);
2916 defer alpha.deinit();
2917 try expectHits(alpha.hits, &.{ 1, 2 });
2918
2919 var conjunction = try search.query(std.testing.allocator, "alpha AND beta", 0);
2920 defer conjunction.deinit();
2921 try expectHits(conjunction.hits, &.{1});
2922 }
2923
2924 test "search bulk load stores fresh documents in sorted posting order" {
2925 var tmp = std.testing.tmpDir(.{});
2926 defer tmp.cleanup();
2927
2928 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2929 .paths = .{ .database = "search.db", .wal = "search.wal" },
2930 .header = testingHeader(),
2931 });
2932 defer database.deinit();
2933 try database.reserve(.{ .wal_frames = 512 });
2934
2935 var search = try Search.open(&database, .{});
2936 _ = try search.loadAll(std.testing.allocator, &.{
2937 .{ .rowid = 3, .text = "zeta beta" },
2938 .{ .rowid = 1, .text = "alpha beta" },
2939 .{ .rowid = 2, .text = "alpha gamma" },
2940 }, .{ .durability = .buffered });
2941
2942 var alpha = try search.query(std.testing.allocator, "alpha", 0);
2943 defer alpha.deinit();
2944 try expectHits(alpha.hits, &.{ 1, 2 });
2945
2946 var beta = try search.query(std.testing.allocator, "beta", 0);
2947 defer beta.deinit();
2948 try expectHits(beta.hits, &.{ 1, 3 });
2949 }
2950
2951 test "search bulk load stores repeated terms in immutable segments" {
2952 var tmp = std.testing.tmpDir(.{});
2953 defer tmp.cleanup();
2954
2955 {
2956 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
2957 .paths = .{ .database = "search.db", .wal = "search.wal" },
2958 .header = testingHeader(),
2959 });
2960 defer database.deinit();
2961 try database.reserve(.{ .wal_frames = 1024 });
2962
2963 var search = try Search.open(&database, .{ .index_prefixes = false });
2964 _ = try search.loadAllNew(std.testing.allocator, &.{
2965 .{ .rowid = 1, .text = "common alpha" },
2966 .{ .rowid = 2, .text = "common beta" },
2967 .{ .rowid = 3, .text = "common gamma" },
2968 }, .{ .durability = .buffered });
2969 _ = try search.loadAllNew(std.testing.allocator, &.{
2970 .{ .rowid = 4, .text = "common common delta" },
2971 .{ .rowid = 5, .text = "common epsilon" },
2972 }, .{ .durability = .buffered });
2973
2974 var key_buffer: [token_mod.max_token_bytes + 1]u8 = undefined;
2975 const key_text = posting_mod.postingKeyBuffer(&key_buffer, '=', "common");
2976 var lookup: index_mod.Scan = undefined;
2977 try search.terms.lookupPayloads(&lookup, std.testing.allocator, &.{.{ .text = key_text }});
2978 defer lookup.deinit();
2979 var entries: usize = 0;
2980 while (try lookup.next()) |entry| {
2981 entries += 1;
2982 try std.testing.expect(posting_mod.segmentPayloadCount(entry.payload) != null);
2983 }
2984 try std.testing.expectEqual(@as(usize, 2), entries);
2985
2986 var common = try search.query(std.testing.allocator, "common", 0);
2987 defer common.deinit();
2988 try std.testing.expectEqual(@as(usize, 5), common.total);
2989 try expectHits(common.hits, &.{ 4, 1, 2, 3, 5 });
2990 try std.testing.expectEqual(@as(usize, 2), common.hits[0].score);
2991
2992 _ = try search.delete(std.testing.allocator, 2, .{ .durability = .buffered });
2993 var deleted = try search.query(std.testing.allocator, "common", 0);
2994 defer deleted.deinit();
2995 try std.testing.expectEqual(@as(usize, 4), deleted.total);
2996 try expectHits(deleted.hits, &.{ 4, 1, 3, 5 });
2997
2998 _ = try search.put(std.testing.allocator, 2, "common common common replacement", .{ .durability = .buffered });
2999 var replaced = try search.query(std.testing.allocator, "common", 0);
3000 defer replaced.deinit();
3001 try std.testing.expectEqual(@as(usize, 5), replaced.total);
3002 try expectHits(replaced.hits, &.{ 2, 4, 1, 3, 5 });
3003 try std.testing.expectEqual(@as(usize, 3), replaced.hits[0].score);
3004
3005 _ = try search.delete(std.testing.allocator, 2, .{ .durability = .buffered });
3006 var deleted_again = try search.query(std.testing.allocator, "common", 0);
3007 defer deleted_again.deinit();
3008 try std.testing.expectEqual(@as(usize, 4), deleted_again.total);
3009 try expectHits(deleted_again.hits, &.{ 4, 1, 3, 5 });
3010
3011 try database.syncWal();
3012 }
3013
3014 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3015 .paths = .{ .database = "search.db", .wal = "search.wal" },
3016 .header = recoveredHeader(),
3017 });
3018 defer reopened.deinit();
3019 var search = try Search.open(&reopened, .{ .index_prefixes = false });
3020 var common = try search.query(std.testing.allocator, "common", 0);
3021 defer common.deinit();
3022 try std.testing.expectEqual(@as(usize, 4), common.total);
3023 try expectHits(common.hits, &.{ 4, 1, 3, 5 });
3024 }
3025
3026 test "search posting stats classify phrase pair segments" {
3027 var tmp = std.testing.tmpDir(.{});
3028 defer tmp.cleanup();
3029
3030 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3031 .paths = .{ .database = "search.db", .wal = "search.wal" },
3032 .header = testingHeader(),
3033 });
3034 defer database.deinit();
3035 try database.reserve(.{ .wal_frames = 1024 });
3036
3037 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
3038 _ = try search.loadAllNew(std.testing.allocator, &.{
3039 .{ .rowid = 1, .text = "alpha beta common" },
3040 .{ .rowid = 2, .text = "alpha beta common" },
3041 .{ .rowid = 3, .text = "alpha gamma common" },
3042 }, .{ .durability = .buffered });
3043
3044 const stats = try search.postingStats(std.testing.allocator);
3045 try std.testing.expectEqual(@as(usize, 8), stats.posting_term_keys);
3046 try std.testing.expectEqual(@as(usize, 72), stats.posting_term_key_text_bytes);
3047 try std.testing.expectEqual(@as(usize, 4), stats.exact_term_keys);
3048 try std.testing.expectEqual(@as(usize, 24), stats.exact_term_key_text_bytes);
3049 try std.testing.expectEqual(@as(usize, 4), stats.phrase_pair_term_keys);
3050 try std.testing.expectEqual(@as(usize, 48), stats.phrase_pair_term_key_text_bytes);
3051 try std.testing.expectEqual(@as(usize, 8), stats.posting_entries);
3052 try std.testing.expectEqual(@as(usize, 128), stats.posting_payload_bytes);
3053 try std.testing.expectEqual(@as(usize, 72), stats.posting_key_text_bytes);
3054 try std.testing.expectEqual(@as(usize, 24), stats.exact_posting_key_text_bytes);
3055 try std.testing.expectEqual(@as(usize, 48), stats.phrase_pair_posting_key_text_bytes);
3056 try std.testing.expectEqual(@as(usize, 76), stats.exact_posting_payload_bytes);
3057 try std.testing.expectEqual(@as(usize, 52), stats.phrase_pair_posting_payload_bytes);
3058 try std.testing.expectEqual(@as(usize, 3), stats.direct_posting_entries);
3059 try std.testing.expectEqual(@as(usize, 0), stats.direct_posting_tombstones);
3060 try std.testing.expectEqual(@as(usize, 5), stats.segment_entries);
3061 try std.testing.expectEqual(@as(usize, 12), stats.segment_postings);
3062 try std.testing.expectEqual(@as(usize, 15), stats.livePostingEntries());
3063 try std.testing.expectEqual(@as(usize, 2), stats.phrase_pair_direct_posting_entries);
3064 try std.testing.expectEqual(@as(usize, 2), stats.phrase_pair_segment_entries);
3065 try std.testing.expectEqual(@as(usize, 4), stats.phrase_pair_segment_postings);
3066 try std.testing.expectEqual(@as(usize, 6), stats.phrasePairLivePostingEntries());
3067 try std.testing.expectEqual(@as(usize, 1), stats.document_length_entries);
3068 try std.testing.expectEqual(@as(usize, 1), stats.document_length_segment_entries);
3069 try std.testing.expectEqual(@as(usize, 3), stats.document_length_segment_postings);
3070 try std.testing.expectEqual(@as(usize, 2), stats.metadata_entries);
3071 try std.testing.expectEqual(stats.term_index_entries, stats.posting_entries + stats.document_length_entries + stats.metadata_entries);
3072 }
3073
3074 test "search bulk load replaces existing documents" {
3075 var tmp = std.testing.tmpDir(.{});
3076 defer tmp.cleanup();
3077
3078 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3079 .paths = .{ .database = "search.db", .wal = "search.wal" },
3080 .header = testingHeader(),
3081 });
3082 defer database.deinit();
3083 try database.reserve(.{ .wal_frames = 512 });
3084
3085 var search = try Search.open(&database, .{ .index_prefixes = false });
3086 _ = try search.putAll(std.testing.allocator, &.{
3087 .{ .rowid = 1, .text = "alpha stale" },
3088 .{ .rowid = 2, .text = "orphan stale" },
3089 }, .{ .durability = .buffered });
3090 _ = try search.loadAll(std.testing.allocator, &.{
3091 .{ .rowid = 1, .text = "beta fresh" },
3092 .{ .rowid = 3, .text = "gamma fresh" },
3093 }, .{ .durability = .buffered });
3094
3095 var stale = try search.query(std.testing.allocator, "stale", 0);
3096 defer stale.deinit();
3097 try expectHits(stale.hits, &.{});
3098
3099 var alpha = try search.query(std.testing.allocator, "alpha", 0);
3100 defer alpha.deinit();
3101 try expectHits(alpha.hits, &.{});
3102
3103 var beta = try search.query(std.testing.allocator, "beta", 0);
3104 defer beta.deinit();
3105 try expectHits(beta.hits, &.{1});
3106
3107 var gamma = try search.query(std.testing.allocator, "gamma", 0);
3108 defer gamma.deinit();
3109 try expectHits(gamma.hits, &.{3});
3110 }
3111
3112 test "search clear removes indexed documents and postings" {
3113 var tmp = std.testing.tmpDir(.{});
3114 defer tmp.cleanup();
3115
3116 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3117 .paths = .{ .database = "search.db", .wal = "search.wal" },
3118 .header = testingHeader(),
3119 });
3120 defer database.deinit();
3121 try database.reserve(.{ .wal_frames = 512 });
3122
3123 var search = try Search.open(&database, .{ .index_prefixes = false });
3124 _ = try search.putAll(std.testing.allocator, &.{
3125 .{ .rowid = 1, .text = "alpha beta" },
3126 .{ .rowid = 2, .text = "gamma delta" },
3127 }, .{ .durability = .buffered });
3128 _ = try search.clear(std.testing.allocator, .{ .durability = .buffered });
3129
3130 var alpha = try search.query(std.testing.allocator, "alpha", 0);
3131 defer alpha.deinit();
3132 try expectHits(alpha.hits, &.{});
3133
3134 _ = try search.put(std.testing.allocator, 3, "epsilon", .{ .durability = .buffered });
3135 var epsilon = try search.query(std.testing.allocator, "epsilon", 0);
3136 defer epsilon.deinit();
3137 try expectHits(epsilon.hits, &.{3});
3138 }
3139
3140 test "search loadAllNew loads cleared sorted batches across reopen" {
3141 var tmp = std.testing.tmpDir(.{});
3142 defer tmp.cleanup();
3143
3144 {
3145 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3146 .paths = .{ .database = "search.db", .wal = "search.wal" },
3147 .header = testingHeader(),
3148 });
3149 defer database.deinit();
3150 try database.reserve(.{ .wal_frames = 512 });
3151
3152 var search = try Search.open(&database, .{ .index_prefixes = false });
3153 _ = try search.put(std.testing.allocator, 1, "stale alpha", .{ .durability = .buffered });
3154 _ = try search.clear(std.testing.allocator, .{ .durability = .buffered });
3155 _ = try search.loadAllNew(std.testing.allocator, &.{
3156 .{ .rowid = 2, .text = "fresh beta" },
3157 .{ .rowid = 3, .text = "fresh gamma" },
3158 }, .{ .durability = .buffered });
3159
3160 var stale = try search.query(std.testing.allocator, "stale", 0);
3161 defer stale.deinit();
3162 try expectHits(stale.hits, &.{});
3163
3164 var beta = try search.query(std.testing.allocator, "beta", 0);
3165 defer beta.deinit();
3166 try expectHits(beta.hits, &.{2});
3167
3168 try database.syncWal();
3169 }
3170
3171 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3172 .paths = .{ .database = "search.db", .wal = "search.wal" },
3173 .header = recoveredHeader(),
3174 });
3175 defer reopened.deinit();
3176 var search = try Search.open(&reopened, .{ .index_prefixes = false });
3177 var gamma = try search.query(std.testing.allocator, "gamma", 0);
3178 defer gamma.deinit();
3179 try expectHits(gamma.hits, &.{3});
3180 }
3181
3182 test "search bulk load can rebuild identical exact postings across reopen" {
3183 var tmp = std.testing.tmpDir(.{});
3184 defer tmp.cleanup();
3185
3186 {
3187 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3188 .paths = .{ .database = "search.db", .wal = "search.wal" },
3189 .header = testingHeader(),
3190 });
3191 defer database.deinit();
3192 try database.reserve(.{ .wal_frames = 512 });
3193
3194 var search = try Search.open(&database, .{ .index_prefixes = false });
3195 _ = try search.put(std.testing.allocator, 1, "alpha beta", .{ .durability = .buffered });
3196 _ = try search.loadAll(std.testing.allocator, &.{
3197 .{ .rowid = 1, .text = "alpha beta" },
3198 }, .{ .durability = .buffered });
3199
3200 var conjunction = try search.query(std.testing.allocator, "alpha AND beta", 0);
3201 defer conjunction.deinit();
3202 try expectHits(conjunction.hits, &.{1});
3203 try database.syncWal();
3204 }
3205
3206 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3207 .paths = .{ .database = "search.db", .wal = "search.wal" },
3208 .header = recoveredHeader(),
3209 });
3210 defer reopened.deinit();
3211 var search = try Search.open(&reopened, .{ .index_prefixes = false });
3212 var conjunction = try search.query(std.testing.allocator, "alpha AND beta", 0);
3213 defer conjunction.deinit();
3214 try expectHits(conjunction.hits, &.{1});
3215 }
3216
3217 test "search bulk load rebuilds changed phrase pair postings" {
3218 var tmp = std.testing.tmpDir(.{});
3219 defer tmp.cleanup();
3220
3221 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3222 .paths = .{ .database = "search.db", .wal = "search.wal" },
3223 .header = testingHeader(),
3224 });
3225 defer database.deinit();
3226 try database.reserve(.{ .wal_frames = 512 });
3227
3228 var search = try Search.open(&database, .{ .index_prefixes = false, .index_phrase_pairs = true });
3229 _ = try search.put(std.testing.allocator, 1, "alpha alpha", .{ .durability = .buffered });
3230 _ = try search.loadAll(std.testing.allocator, &.{
3231 .{ .rowid = 1, .text = "alpha beta" },
3232 }, .{ .durability = .buffered });
3233
3234 var phrase = try search.query(std.testing.allocator, "\"alpha beta\"", 0);
3235 defer phrase.deinit();
3236 try expectHits(phrase.hits, &.{1});
3237 }
3238
3239 test "search stores large documents through overflow rows" {
3240 var tmp = std.testing.tmpDir(.{});
3241 defer tmp.cleanup();
3242
3243 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3244 .paths = .{ .database = "search.db", .wal = "search.wal" },
3245 .header = testingHeader(),
3246 });
3247 defer database.deinit();
3248 try database.reserve(.{ .wal_frames = 1024 });
3249
3250 var text: std.ArrayList(u8) = .empty;
3251 defer text.deinit(std.testing.allocator);
3252 for (0..1024) |_| try text.appendSlice(std.testing.allocator, "large native search document needle ");
3253
3254 var search = try Search.open(&database, .{});
3255 _ = try search.put(std.testing.allocator, 1, text.items, .{ .durability = .buffered });
3256
3257 var results = try search.query(std.testing.allocator, "needle", 0);
3258 defer results.deinit();
3259 try expectHits(results.hits, &.{1});
3260 }
3261
3262 test "search bounds oversized documents to page safe text" {
3263 var tmp = std.testing.tmpDir(.{});
3264 defer tmp.cleanup();
3265
3266 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3267 .paths = .{ .database = "search.db", .wal = "search.wal" },
3268 .header = testingHeader(),
3269 });
3270 defer database.deinit();
3271 try database.reserve(.{ .wal_frames = 1024 });
3272
3273 var text: std.ArrayList(u8) = .empty;
3274 defer text.deinit(std.testing.allocator);
3275 try text.appendSlice(std.testing.allocator, "needle ");
3276 for (0..token_mod.max_document_bytes + 4096) |_| try text.append(std.testing.allocator, 'x');
3277 try text.appendSlice(std.testing.allocator, " tailunique");
3278
3279 var search = try Search.open(&database, .{});
3280 _ = try search.put(std.testing.allocator, 1, text.items, .{ .durability = .buffered });
3281
3282 const stored = (try search.getText(std.testing.allocator, 1)).?;
3283 defer std.testing.allocator.free(stored);
3284 try std.testing.expectEqual(@as(usize, token_mod.max_document_bytes), stored.len);
3285
3286 var head = try search.query(std.testing.allocator, "needle", 0);
3287 defer head.deinit();
3288 try expectHits(head.hits, &.{1});
3289
3290 var tail = try search.query(std.testing.allocator, "tailunique", 0);
3291 defer tail.deinit();
3292 try expectHits(tail.hits, &.{});
3293 }
3294
3295 test "search can skip prefix postings for exact workloads" {
3296 var tmp = std.testing.tmpDir(.{});
3297 defer tmp.cleanup();
3298
3299 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3300 .paths = .{ .database = "search.db", .wal = "search.wal" },
3301 .header = testingHeader(),
3302 });
3303 defer database.deinit();
3304
3305 var search = try Search.open(&database, .{ .index_prefixes = false });
3306 _ = try search.putAll(std.testing.allocator, &.{
3307 .{ .rowid = 1, .text = "authentication bug" },
3308 }, .{ .durability = .buffered });
3309
3310 var exact = try search.query(std.testing.allocator, "authentication", 0);
3311 defer exact.deinit();
3312 try expectHits(exact.hits, &.{1});
3313
3314 var prefix = try search.query(std.testing.allocator, "auth*", 0);
3315 defer prefix.deinit();
3316 try expectHits(prefix.hits, &.{1});
3317 }
3318
3319 test "search exact-only index reopens" {
3320 var tmp = std.testing.tmpDir(.{});
3321 defer tmp.cleanup();
3322
3323 {
3324 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3325 .paths = .{ .database = "search.db", .wal = "search.wal" },
3326 .header = testingHeader(),
3327 });
3328 defer database.deinit();
3329 try database.reserve(.{ .wal_frames = 512 });
3330 var search = try Search.open(&database, .{ .index_prefixes = false });
3331 _ = try search.put(std.testing.allocator, 1, "needle exact reopen", .{ .durability = .buffered });
3332 try database.syncWal();
3333 }
3334
3335 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3336 .paths = .{ .database = "search.db", .wal = "search.wal" },
3337 .header = recoveredHeader(),
3338 });
3339 defer reopened.deinit();
3340 var search = try Search.open(&reopened, .{ .index_prefixes = false });
3341 var exact = try search.query(std.testing.allocator, "needle", 0);
3342 defer exact.deinit();
3343 try expectHits(exact.hits, &.{1});
3344 }
3345
3346 test "search shares tablespace with rowid table roots" {
3347 var tmp = std.testing.tmpDir(.{});
3348 defer tmp.cleanup();
3349
3350 {
3351 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3352 .paths = .{ .database = "search.db", .wal = "search.wal" },
3353 .header = testingHeader(),
3354 });
3355 defer database.deinit();
3356 try database.reserve(.{ .wal_frames = 512 });
3357 const shared = try space_mod.Space.open(&database, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 5 }, .{ .root_page = 6 } }, .reserved_page_max = 6 });
3358 var table = try shared.rowidTable(2);
3359 _ = try table.put(1, &.{.{ .text = "metadata" }}, .{ .durability = .buffered });
3360 var search = try Search.open(&database, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 6, .index_prefixes = false });
3361 _ = try search.put(std.testing.allocator, 1, "needle shared reopen", .{ .durability = .buffered });
3362 try database.syncWal();
3363 }
3364
3365 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3366 .paths = .{ .database = "search.db", .wal = "search.wal" },
3367 .header = recoveredHeader(),
3368 });
3369 defer reopened.deinit();
3370 var search = try Search.open(&reopened, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 6, .index_prefixes = false });
3371 var exact = try search.query(std.testing.allocator, "needle", 0);
3372 defer exact.deinit();
3373 try expectHits(exact.hits, &.{1});
3374 }
3375
3376 test "search shares tablespace with two search roots" {
3377 var tmp = std.testing.tmpDir(.{});
3378 defer tmp.cleanup();
3379
3380 {
3381 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3382 .paths = .{ .database = "search.db", .wal = "search.wal" },
3383 .header = testingHeader(),
3384 });
3385 defer database.deinit();
3386 try database.reserve(.{ .wal_frames = 512 });
3387 const shared = try space_mod.Space.open(&database, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3388 var table = try shared.rowidTable(2);
3389 _ = try table.put(1, &.{.{ .text = "metadata" }}, .{ .durability = .buffered });
3390 var first = try Search.open(&database, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3391 _ = try first.put(std.testing.allocator, 1, "needle first reopen", .{ .durability = .buffered });
3392 _ = try Search.open(&database, .{ .documents_root = 7, .terms_root = 8, .reserved_page_max = 8, .index_prefixes = false });
3393 try database.syncWal();
3394 }
3395
3396 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3397 .paths = .{ .database = "search.db", .wal = "search.wal" },
3398 .header = recoveredHeader(),
3399 });
3400 defer reopened.deinit();
3401 var search = try Search.open(&reopened, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3402 var exact = try search.query(std.testing.allocator, "needle", 0);
3403 defer exact.deinit();
3404 try expectHits(exact.hits, &.{1});
3405 }
3406
3407 test "search reopens large rowids" {
3408 var tmp = std.testing.tmpDir(.{});
3409 defer tmp.cleanup();
3410 const rowid: i64 = 0x7fff_ffff_ffff_f123;
3411
3412 {
3413 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3414 .paths = .{ .database = "search.db", .wal = "search.wal" },
3415 .header = testingHeader(),
3416 });
3417 defer database.deinit();
3418 try database.reserve(.{ .wal_frames = 512 });
3419 var search = try Search.open(&database, .{ .index_prefixes = false });
3420 _ = try search.put(std.testing.allocator, rowid, "needle large rowid", .{ .durability = .buffered });
3421 try database.syncWal();
3422 }
3423
3424 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3425 .paths = .{ .database = "search.db", .wal = "search.wal" },
3426 .header = recoveredHeader(),
3427 });
3428 defer reopened.deinit();
3429 var search = try Search.open(&reopened, .{ .index_prefixes = false });
3430 var exact = try search.query(std.testing.allocator, "needle", 0);
3431 defer exact.deinit();
3432 try expectHits(exact.hits, &.{rowid});
3433 }
3434
3435 test "search reopens glom shaped document text" {
3436 var tmp = std.testing.tmpDir(.{});
3437 defer tmp.cleanup();
3438 const rowid: i64 = 0x6123_4567_89ab_cdef;
3439 const text = "reopen\ncodex\n/tmp/reopen/doc.md\nmemory\nreopen\nneedle survives reopen";
3440
3441 {
3442 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3443 .paths = .{ .database = "search.db", .wal = "search.wal" },
3444 .header = testingHeader(),
3445 });
3446 defer database.deinit();
3447 try database.reserve(.{ .wal_frames = 512 });
3448 const shared = try space_mod.Space.open(&database, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3449 var table = try shared.rowidTable(2);
3450 _ = try table.put(rowid, &.{ .{ .text = "codex" }, .{ .text = "/tmp/reopen/doc.md" }, .{ .text = "memory" }, .{ .text = text } }, .{ .durability = .buffered });
3451 var first = try Search.open(&database, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3452 _ = try first.put(std.testing.allocator, rowid, text, .{ .durability = .buffered });
3453 _ = try Search.open(&database, .{ .documents_root = 7, .terms_root = 8, .reserved_page_max = 8, .index_prefixes = false });
3454 try database.syncWal();
3455 }
3456
3457 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3458 .paths = .{ .database = "search.db", .wal = "search.wal" },
3459 .header = recoveredHeader(),
3460 });
3461 defer reopened.deinit();
3462 var search = try Search.open(&reopened, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3463 var exact = try search.query(std.testing.allocator, "needle", 0);
3464 defer exact.deinit();
3465 try expectHits(exact.hits, &.{rowid});
3466 }
3467
3468 test "search shared tablespace writes later reserved table root" {
3469 var tmp = std.testing.tmpDir(.{});
3470 defer tmp.cleanup();
3471
3472 {
3473 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3474 .paths = .{ .database = "search.db", .wal = "search.wal" },
3475 .header = testingHeader(),
3476 });
3477 defer database.deinit();
3478 try database.reserve(.{ .wal_frames = 512 });
3479 const shared = try space_mod.Space.open(&database, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3480 var documents = try shared.rowidTable(2);
3481 var refs = try shared.rowidTable(4);
3482 _ = try documents.put(1, &.{ .{ .text = "codex" }, .{ .text = "needle document" } }, .{ .durability = .buffered });
3483 var first = try Search.open(&database, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3484 _ = try first.put(std.testing.allocator, 1, "needle document", .{ .durability = .buffered });
3485 _ = try refs.put(1_000_000, &.{ .{ .text = "documents" }, .{ .integer = 0 }, .{ .text = "1" } }, .{ .durability = .buffered });
3486 try database.syncWal();
3487 }
3488
3489 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3490 .paths = .{ .database = "search.db", .wal = "search.wal" },
3491 .header = recoveredHeader(),
3492 });
3493 defer reopened.deinit();
3494 const shared = try space_mod.Space.open(&reopened, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3495 var documents = try shared.rowidTable(2);
3496 const bytes = (try documents.get(std.testing.allocator, 1)).?;
3497 defer std.testing.allocator.free(bytes);
3498 const view = try row.View.init(bytes);
3499 try std.testing.expectEqualStrings("needle document", (try view.column(1)).text);
3500 }
3501
3502 test "search shared tablespace writes refs after reopen query" {
3503 var tmp = std.testing.tmpDir(.{});
3504 defer tmp.cleanup();
3505
3506 {
3507 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3508 .paths = .{ .database = "search.db", .wal = "search.wal" },
3509 .header = testingHeader(),
3510 });
3511 defer database.deinit();
3512 try database.reserve(.{ .wal_frames = 512 });
3513 const shared = try space_mod.Space.open(&database, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3514 var documents = try shared.rowidTable(2);
3515 _ = try documents.put(1, &.{ .{ .text = "codex" }, .{ .text = "needle document" } }, .{ .durability = .buffered });
3516 var first = try Search.open(&database, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3517 _ = try first.put(std.testing.allocator, 1, "needle document", .{ .durability = .buffered });
3518 try database.syncWal();
3519 }
3520
3521 {
3522 var reopened = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3523 .paths = .{ .database = "search.db", .wal = "search.wal" },
3524 .header = recoveredHeader(),
3525 });
3526 defer reopened.deinit();
3527 var search = try Search.open(&reopened, .{ .documents_root = 5, .terms_root = 6, .reserved_page_max = 8, .index_prefixes = false });
3528 var exact = try search.query(std.testing.allocator, "needle", 0);
3529 defer exact.deinit();
3530 try expectHits(exact.hits, &.{1});
3531 const shared = try space_mod.Space.open(&reopened, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3532 var refs = try shared.rowidTable(4);
3533 _ = try refs.put(1_000_000, &.{ .{ .text = "documents" }, .{ .integer = 0 }, .{ .text = "1" } }, .{ .durability = .buffered });
3534 try reopened.syncWal();
3535 }
3536
3537 var final = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3538 .paths = .{ .database = "search.db", .wal = "search.wal" },
3539 .header = recoveredHeader(),
3540 });
3541 defer final.deinit();
3542 const shared = try space_mod.Space.open(&final, .{ .meta_page = 1, .roots = &.{ .{ .root_page = 2 }, .{ .root_page = 3 }, .{ .root_page = 4 }, .{ .root_page = 5 }, .{ .root_page = 6 }, .{ .root_page = 7 }, .{ .root_page = 8 } }, .reserved_page_max = 8 });
3543 var documents = try shared.rowidTable(2);
3544 const bytes = (try documents.get(std.testing.allocator, 1)).?;
3545 defer std.testing.allocator.free(bytes);
3546 const view = try row.View.init(bytes);
3547 try std.testing.expectEqualStrings("needle document", (try view.column(1)).text);
3548 }
3549
3550 test "search source revisions gate document updates" {
3551 var tmp = std.testing.tmpDir(.{});
3552 defer tmp.cleanup();
3553
3554 var database = try file.Database.openForTesting(std.testing.allocator, tmp.dir, .{
3555 .paths = .{ .database = "search.db", .wal = "search.wal" },
3556 .header = testingHeader(),
3557 });
3558 defer database.deinit();
3559 try database.reserve(.{ .wal_frames = 512 });
3560
3561 const first = Source{ .schema = 7, .head = @as([32]u8, @splat(1)) };
3562 const second = Source{ .schema = 7, .head = @as([32]u8, @splat(2)) };
3563 const third = Source{ .schema = 7, .head = @as([32]u8, @splat(3)) };
3564 var search = try Search.open(&database, .{ .index_prefixes = false });
3565 try std.testing.expect(!try search.sourceSchemaMatches(std.testing.allocator, first.schema));
3566 _ = try search.loadAllForSource(std.testing.allocator, &.{.{ .rowid = 1, .text = "first term" }}, first, .{ .durability = .buffered });
3567 try std.testing.expect(try search.sourceMatches(std.testing.allocator, first));
3568 try std.testing.expect(try search.sourceSchemaMatches(std.testing.allocator, first.schema));
3569 try std.testing.expect(!try search.sourceSchemaMatches(std.testing.allocator, first.schema + 1));
3570
3571 try std.testing.expect((try search.putForSource(std.testing.allocator, 1, "stale term", second, third, .{ .durability = .buffered })) == null);
3572 try std.testing.expect(try search.sourceMatches(std.testing.allocator, first));
3573 var stale = try search.query(std.testing.allocator, "stale", 0);
3574 defer stale.deinit();
3575 try std.testing.expectEqual(@as(usize, 0), stale.total);
3576
3577 _ = (try search.putForSource(std.testing.allocator, 1, "second term", first, second, .{ .durability = .buffered })).?;
3578 try std.testing.expect(try search.sourceMatches(std.testing.allocator, second));
3579 var replaced = try search.query(std.testing.allocator, "first", 0);
3580 defer replaced.deinit();
3581 try std.testing.expectEqual(@as(usize, 0), replaced.total);
3582 var current = try search.query(std.testing.allocator, "second", 0);
3583 defer current.deinit();
3584 try expectHits(current.hits, &.{1});
3585
3586 _ = (try search.advanceSource(std.testing.allocator, second, third, .{ .durability = .buffered })).?;
3587 try std.testing.expect(try search.sourceMatches(std.testing.allocator, third));
3588 const stored = (try search.indexedSource(std.testing.allocator)).?;
3589 try std.testing.expect(stored.same(third));
3590 }
3591
3592 pub fn expectHits(hits: []const hit_mod.Hit, expected: []const i64) !void {
3593 try std.testing.expectEqual(expected.len, hits.len);
3594 for (expected, 0..) |rowid, index| try std.testing.expectEqual(rowid, hits[index].rowid);
3595 }
3596
3597 pub fn expectRankedHits(hits: []const hit_mod.RankedHit, expected: []const i64) !void {
3598 try std.testing.expectEqual(expected.len, hits.len);
3599 for (expected, 0..) |rowid, index| try std.testing.expectEqual(rowid, hits[index].rowid);
3600 }