lib/sql/src/search/token.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

 1 const std = @import("std");
 2 const sql = @import("../root.zig");
 3 const engine_mod = @import("engine.zig");
 4 const text_mod = @import("text.zig");
 5 const row = sql.row;
 6 
 7 const Allocator = std.mem.Allocator;
 8 
 9 pub const max_document_bytes = 60 * 1024;
10 
11 pub const max_token_bytes = 512;
12 
13 pub const field_separator: u8 = 0x1f;
14 
15 pub const Token = struct {
16     text: []u8,
17     position: usize,
18 };
19 
20 pub fn documentText(text: []const u8) []const u8 {
21     return text[0..@min(text.len, max_document_bytes)];
22 }
23 
24 pub fn tokenize(allocator: Allocator, text: []const u8) engine_mod.Error![]Token {
25     var tokens: std.ArrayList(Token) = .empty;
26     errdefer freeTokens(allocator, tokens.items);
27     var start: ?usize = null;
28     var position: usize = 0;
29     for (text, 0..) |byte, index| {
30         if (std.ascii.isAlphanumeric(byte)) {
31             if (start == null) start = index;
32         } else if (start) |begin| {
33             try appendToken(allocator, &tokens, text[begin..index], position);
34             position += 1;
35             start = null;
36         }
37     }
38     if (start) |begin| try appendToken(allocator, &tokens, text[begin..], position);
39     return try tokens.toOwnedSlice(allocator);
40 }
41 
42 pub fn freeTokens(allocator: Allocator, tokens: []Token) void {
43     for (tokens) |token| allocator.free(token.text);
44     allocator.free(tokens);
45 }
46 
47 pub fn appendToken(allocator: Allocator, tokens: *std.ArrayList(Token), raw: []const u8, position: usize) engine_mod.Error!void {
48     if (raw.len == 0) return;
49     if (raw.len > max_token_bytes) return;
50     const out = try allocator.alloc(u8, raw.len);
51     errdefer allocator.free(out);
52     for (raw, 0..) |byte, index| out[index] = std.ascii.toLower(byte);
53     try tokens.append(allocator, .{ .text = out, .position = position });
54 }
55 
56 pub fn documentTokenLength(text: []const u8) usize {
57     var count: usize = 0;
58     var offset: usize = 0;
59     var position: usize = 0;
60     var field: usize = 0;
61     while (text_mod.nextTextToken(text, &offset, &position, &field) != null) count += 1;
62     return count;
63 }
64 
65 pub fn textFromRow(bytes: []const u8) row.Error![]const u8 {
66     const view = try row.View.init(bytes);
67     const value = try view.column(0);
68     return switch (value) {
69         .text => |text| text,
70         else => error.InvalidRow,
71     };
72 }
73 
74 pub fn asciiEqlIgnoreCase(left: []const u8, right: []const u8) bool {
75     if (left.len != right.len) return false;
76     for (left, right) |l, r| if (std.ascii.toLower(l) != std.ascii.toLower(r)) return false;
77     return true;
78 }
79 
80 test "search tokenizes ascii text into lowercase terms" {
81     const tokens = try tokenize(std.testing.allocator, "Fix_auth bug, LOGIN42");
82     defer freeTokens(std.testing.allocator, tokens);
83     try std.testing.expectEqual(@as(usize, 4), tokens.len);
84     try std.testing.expectEqualStrings("fix", tokens[0].text);
85     try std.testing.expectEqualStrings("auth", tokens[1].text);
86     try std.testing.expectEqualStrings("bug", tokens[2].text);
87     try std.testing.expectEqualStrings("login42", tokens[3].text);
88 }