tiny.smg.storage.nodes

1pub const NodeRow = struct {2    rowid: i64,3    node: model.Node,4};

1const std = @import("std");2const sql = @import("sql");3const sys = @import("sys");4const smg = @import("../root.zig");5const storage = smg.storage;6const database_read = storage.database.read;7const store = storage.store;8const sync = storage.sync;9const rows = storage.rows;1011const graph_mod = smg.graph;12const model = smg.model;13const view = smg.view;1415pub const NodeRow = rows.NodeRow;1617pub const ListedNodes = struct {18    nodes: []const model.Node,19    total: usize,20};2122pub fn rowsById(23    allocator: std.mem.Allocator,24    root: []const u8,25    rowids: []const i64,26    limits: smg.StorageLimits,27) ![]NodeRow {28    var opened = try store.openRead(allocator, root, limits);29    defer opened.close();30    return try rowsByIdFromReader(allocator, &opened.reader, rowids);31}3233pub fn rowsByIdFromReader(allocator: std.mem.Allocator, reader: *storage.database.Reader, rowids: []const i64) ![]NodeRow {34    const requested = try uniquePositiveRowids(allocator, rowids);35    defer allocator.free(requested);36    if (requested.len == 0) return &.{};37    var handle = try reader.catalog.openRelation(reader.allocator, rows.nodes_relation);38    defer handle.deinit();39    var selected_rows: std.ArrayList(NodeRow) = .empty;40    errdefer {41        for (selected_rows.items) |*row| model.deinitNode(&row.node, allocator);42        selected_rows.deinit(allocator);43    }44    for (requested) |rowid| {45        const bytes = (try handle.relation.get(allocator, rowid)) orelse continue;46        defer allocator.free(bytes);47        try selected_rows.append(allocator, try rows.decodeNodeRow(allocator, rowid, bytes));48    }49    return try selected_rows.toOwnedSlice(allocator);50}5152pub fn listed(53    allocator: std.mem.Allocator,54    root: []const u8,55    type_filter: ?[]const u8,56    limit: usize,57    limits: smg.StorageLimits,58) !ListedNodes {59    if (limit != 0) return try listedBounded(allocator, root, type_filter, limit, limits);60    const sorted = try sortedRows(allocator, root, limits);61    defer allocator.free(sorted);62    var listed_nodes: std.ArrayList(model.Node) = .empty;63    errdefer {64        for (listed_nodes.items) |*node| model.deinitNode(node, allocator);65        listed_nodes.deinit(allocator);66    }67    var total: usize = 0;68    for (sorted) |loaded| {69        var node = loaded.node;70        if (type_filter) |filter| {71            if (!std.mem.eql(u8, node.type, filter)) {72                model.deinitNode(&node, allocator);73                continue;74            }75        }76        total += 1;77        if (limit != 0 and listed_nodes.items.len == limit) {78            model.deinitNode(&node, allocator);79            continue;80        }81        try listed_nodes.append(allocator, node);82    }83    return .{ .nodes = try listed_nodes.toOwnedSlice(allocator), .total = total };84}8586fn listedBounded(87    allocator: std.mem.Allocator,88    root: []const u8,89    type_filter: ?[]const u8,90    limit: usize,91    limits: smg.StorageLimits,92) !ListedNodes {93    var opened = try store.openRead(allocator, root, limits);94    defer opened.close();95    var handle = opened.reader.catalog.openRelation(opened.reader.allocator, rows.nodes_relation) catch |err| switch (err) {96        error.RelationNotFound => return .{ .nodes = &.{}, .total = 0 },97        else => return err,98    };99    defer handle.deinit();100    var scan: sql.TableScan = undefined;101    try handle.relation.scan(&scan, opened.reader.allocator, null, null);102    defer scan.deinit();103    var listed_nodes: std.ArrayList(model.Node) = .empty;104    errdefer {105        for (listed_nodes.items) |*node| model.deinitNode(node, allocator);106        listed_nodes.deinit(allocator);107    }108    var total: usize = 0;109    while (try scan.next()) |entry| {110        const node_type = try rows.nodeTypeView(entry.bytes);111        if (type_filter) |filter| {112            if (!std.mem.eql(u8, node_type, filter)) continue;113        }114        total += 1;115        const name = try rows.nodeNameView(entry.bytes);116        try retainBoundedNode(allocator, &listed_nodes, limit, name, entry.rowid, entry.bytes);117    }118    std.mem.sort(model.Node, listed_nodes.items, {}, nodeLess);119    return .{ .nodes = try listed_nodes.toOwnedSlice(allocator), .total = total };120}121122fn retainBoundedNode(allocator: std.mem.Allocator, listed_nodes: *std.ArrayList(model.Node), limit: usize, name: []const u8, rowid: i64, bytes: []const u8) !void {123    if (listed_nodes.items.len < limit) {124        var node = (try rows.decodeNodeRow(allocator, rowid, bytes)).node;125        errdefer model.deinitNode(&node, allocator);126        try listed_nodes.append(allocator, node);127        return;128    }129    const worst_index = worstNodeIndex(listed_nodes.items);130    if (!std.mem.lessThan(u8, name, listed_nodes.items[worst_index].name)) return;131    const row = try rows.decodeNodeRow(allocator, rowid, bytes);132    var old = listed_nodes.items[worst_index];133    listed_nodes.items[worst_index] = row.node;134    model.deinitNode(&old, allocator);135}136137fn worstNodeIndex(nodes: []const model.Node) usize {138    var worst: usize = 0;139    for (nodes[1..], 1..) |node, index| {140        if (std.mem.lessThan(u8, nodes[worst].name, node.name)) worst = index;141    }142    return worst;143}144145pub fn freeRows(allocator: std.mem.Allocator, node_rows: []NodeRow) void {146    for (node_rows) |*row| model.deinitNode(&row.node, allocator);147    if (node_rows.len != 0) allocator.free(node_rows);148}149150pub fn freeListed(allocator: std.mem.Allocator, loaded: ListedNodes) void {151    for (loaded.nodes) |*node_const| {152        var node = node_const.*;153        model.deinitNode(&node, allocator);154    }155    if (loaded.nodes.len != 0) allocator.free(loaded.nodes);156}157158pub fn sortedRows(159    allocator: std.mem.Allocator,160    root: []const u8,161    limits: smg.StorageLimits,162) ![]rows.NodeRow {163    var opened = try store.openRead(allocator, root, limits);164    defer opened.close();165    const node_rows = try database_read.readerNodes(&opened.reader, allocator, allocator);166    sortRows(node_rows);167    return node_rows;168}169170pub fn sortRows(node_rows: []rows.NodeRow) void {171    std.mem.sort(rows.NodeRow, node_rows, {}, rowLess);172}173174fn rowLess(_: void, left: rows.NodeRow, right: rows.NodeRow) bool {175    return std.mem.lessThan(u8, left.node.name, right.node.name);176}177178fn nodeLess(_: void, left: model.Node, right: model.Node) bool {179    return std.mem.lessThan(u8, left.name, right.name);180}181182fn uniquePositiveRowids(allocator: std.mem.Allocator, rowids: []const i64) ![]i64 {183    var requested: std.ArrayList(i64) = .empty;184    errdefer requested.deinit(allocator);185    for (rowids) |rowid| if (rowid > 0) try requested.append(allocator, rowid);186    std.mem.sort(i64, requested.items, {}, rowidLess);187    var write: usize = 0;188    for (requested.items) |rowid| {189        if (write != 0 and requested.items[write - 1] == rowid) continue;190        requested.items[write] = rowid;191        write += 1;192    }193    requested.items.len = write;194    return try requested.toOwnedSlice(allocator);195}196197fn rowidLess(_: void, left: i64, right: i64) bool {198    return left < right;199}200201test "node projections load search rows listings and stable rowids" {202    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);203    defer arena.deinit();204    const allocator = arena.allocator();205    const root = try testRoot(allocator, 0);206    defer sys.fs.deleteTree(root) catch {};207    try initStore(allocator, root);208    var stored = graph_mod.init(allocator);209    const meta = try model.sourcePair(allocator, "scan");210    try graph_mod.addNode(&stored, .{ .name = "a", .type = model.NodeType.module, .metadata = meta });211    try graph_mod.addNode(&stored, .{ .name = "a.f", .type = model.NodeType.function, .file = "a.zig", .line = 3, .docstring = "search body" });212    try graph_mod.addNode(&stored, .{ .name = "m.item", .type = model.NodeType.module, .file = "m.zig" });213    try graph_mod.addNode(&stored, .{ .name = "z.item", .type = model.NodeType.function, .file = "z.zig", .line = 9 });214    try graph_mod.addEdge(&stored, .{ .source = "a", .target = "a.f", .rel = model.RelType.contains });215    try graph_mod.addEdge(&stored, .{ .source = "z.item", .target = "a.f", .rel = model.RelType.contains });216    _ = try saveGraph(allocator, stored, root);217218    const loaded = try listed(allocator, root, null, 2, smg.default_limits.storage);219    defer freeListed(allocator, loaded);220    try std.testing.expectEqual(@as(usize, 4), loaded.total);221    try std.testing.expectEqual(@as(usize, 2), loaded.nodes.len);222    try std.testing.expectEqualStrings("a", loaded.nodes[0].name);223    try std.testing.expectEqualStrings("a.f", loaded.nodes[1].name);224225    const filtered = try listed(226        allocator,227        root,228        model.NodeType.function,229        1,230        smg.default_limits.storage,231    );232    defer freeListed(allocator, filtered);233    try std.testing.expectEqual(@as(usize, 2), filtered.total);234    try std.testing.expectEqual(@as(usize, 1), filtered.nodes.len);235    try std.testing.expectEqualStrings("a.f", filtered.nodes[0].name);236237    var grown = graph_mod.init(allocator);238    try graph_mod.addNode(&grown, .{ .name = "a", .type = model.NodeType.module });239    try graph_mod.addNode(&grown, .{ .name = "a.f", .type = model.NodeType.function, .file = "a.zig", .line = 3, .docstring = "search body" });240    try graph_mod.addNode(&grown, .{ .name = "c", .type = model.NodeType.module });241    try graph_mod.addNode(&grown, .{ .name = "m.item", .type = model.NodeType.module, .file = "m.zig" });242    try graph_mod.addNode(&grown, .{ .name = "z.item", .type = model.NodeType.function, .file = "z.zig", .line = 9 });243    const saved_nodes = try saveGraph(allocator, grown, root);244    try std.testing.expectEqual(@as(usize, 5), saved_nodes.len);245    try std.testing.expectEqualStrings("c", saved_nodes[2].name);246247    const loaded_rows = try rowsById(248        allocator,249        root,250        &.{ 3, 1, 3, -1 },251        smg.default_limits.storage,252    );253    try std.testing.expectEqual(@as(usize, 2), loaded_rows.len);254    try std.testing.expectEqual(@as(i64, 1), loaded_rows[0].rowid);255    try std.testing.expectEqualStrings("a", loaded_rows[0].node.name);256    try std.testing.expectEqual(@as(i64, 3), loaded_rows[1].rowid);257    try std.testing.expectEqualStrings("m.item", loaded_rows[1].node.name);258}259260test "bounded node listing keeps lexical order independent of rowid" {261    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);262    defer arena.deinit();263    const allocator = arena.allocator();264    const root = try testRoot(allocator, 3);265    defer sys.fs.deleteTree(root) catch {};266    try initStore(allocator, root);267    {268        var opened = try store.open(allocator, root, smg.default_limits.storage);269        defer opened.close();270        try storage.database.edits.apply(&opened.database, .{271            .node_puts = &.{272                .{ .rowid = 1, .node = .{ .name = "z", .type = model.NodeType.module } },273                .{ .rowid = 2, .node = .{ .name = "a", .type = model.NodeType.function } },274                .{ .rowid = 3, .node = .{ .name = "m", .type = model.NodeType.function } },275                .{ .rowid = 4, .node = .{ .name = "b", .type = model.NodeType.type } },276            },277        });278        try storage.database.commitIfDirty(&opened.database);279    }280281    const loaded = try listed(allocator, root, null, 2, smg.default_limits.storage);282    defer freeListed(allocator, loaded);283    try std.testing.expectEqual(@as(usize, 4), loaded.total);284    try std.testing.expectEqual(@as(usize, 2), loaded.nodes.len);285    try std.testing.expectEqualStrings("a", loaded.nodes[0].name);286    try std.testing.expectEqualStrings("b", loaded.nodes[1].name);287288    const filtered = try listed(289        allocator,290        root,291        model.NodeType.function,292        2,293        smg.default_limits.storage,294    );295    defer freeListed(allocator, filtered);296    try std.testing.expectEqual(@as(usize, 2), filtered.total);297    try std.testing.expectEqual(@as(usize, 2), filtered.nodes.len);298    try std.testing.expectEqualStrings("a", filtered.nodes[0].name);299    try std.testing.expectEqualStrings("m", filtered.nodes[1].name);300}301302fn initStore(allocator: std.mem.Allocator, root: []const u8) !void {303    try sys.fs.createDirPath(root);304    var opened = try store.open(allocator, root, smg.default_limits.storage);305    opened.close();306}307308fn saveGraph(allocator: std.mem.Allocator, graph: graph_mod.Graph, root: []const u8) ![]const model.Node {309    const stored_nodes = try view.allNodes(graph, allocator, null);310    var opened = try store.open(allocator, root, smg.default_limits.storage);311    defer opened.close();312    try sync.replaceGraph(allocator, &opened.database, graph);313    return stored_nodes;314}315316fn testRoot(allocator: std.mem.Allocator, offset: i64) ![]const u8 {317    return try std.fmt.allocPrint(allocator, "/tmp/smg-storage-nodes-test-{x}", .{@as(u64, @intCast(@max(0, sys.time.realMilliTimestamp() + offset)))});318}

1pub const nodes = @import("nodes.zig");