tiny.smg.scan.fragment

1const std = @import("std");2const sys = @import("sys");34const smg = @import("../root.zig");5const core = @import("core/root.zig");6const graph_mod = smg.graph;7const model = smg.model;8const scan_files = @import("files.zig");9const text = smg.text;1011const fs_io = std.Options.debug_io;1213pub const Digest = scan_files.SourceDigest;1415const fragment_magic = "tiny.smg.fragment/v1\n";16const manifest_magic = "tiny.smg.fragment-manifest/v2\n";17const cache_directory_name = "fragments";18const manifest_file_name = "fragment-state.bin";19const manifest_temp_file_name = "fragment-state.bin.tmp";20const checksum_bytes = std.crypto.hash.sha2.Sha256.digest_length;2122pub const Evidence = struct {23    files: usize = 0,24    nodes_added: usize = 0,25    lang_counts: []const core.Count = &.{},26    type_counts: []const core.Count = &.{},27};2829pub const LoadTiming = struct {30    read_ns: u64 = 0,31    validation_ns: u64 = 0,32    fold_ns: u64 = 0,33};3435pub const ManifestEntry = struct {36    key: Digest,37    live: bool,38    seen: bool = false,39};4041pub const Manifest = struct {42    project: ?scan_files.ProjectIdentity,43    entries: std.StringHashMap(ManifestEntry),4445    pub fn empty(allocator: std.mem.Allocator) Manifest {46        return .{47            .project = null,48            .entries = std.StringHashMap(ManifestEntry).init(allocator),49        };50    }5152    pub fn get(self: *const Manifest, path: []const u8, key: Digest) bool {53        const entry = self.entries.get(path) orelse return false;54        return entry.live and std.mem.eql(u8, &entry.key, &key);55    }5657    pub fn tombstones(self: *const Manifest) usize {58        var count: usize = 0;59        var iterator = self.entries.valueIterator();60        while (iterator.next()) |entry| if (!entry.live) {61            count += 1;62        };63        return count;64    }6566    pub fn matchesProject(self: *const Manifest, digest: Digest) bool {67        const project = self.project orelse return false;68        return std.mem.eql(u8, &project.digest, &digest);69    }7071    pub fn changedPaths(72        self: *Manifest,73        allocator: std.mem.Allocator,74        root: []const u8,75        current: []const scan_files.SourceIdentity,76    ) ![]const []const u8 {77        var prior = self.entries.valueIterator();78        while (prior.next()) |entry| entry.seen = false;79        var changed_count: usize = 0;80        for (current) |source| {81            if (self.entries.getPtr(source.path)) |entry| {82                entry.seen = true;83                if (entry.live and std.mem.eql(u8, &entry.key, &source.digest)) continue;84            }85            changed_count = std.math.add(usize, changed_count, 1) catch86                return error.FragmentManifestTooLarge;87        }88        var entries = self.entries.iterator();89        while (entries.next()) |entry| {90            if (!entry.value_ptr.live or entry.value_ptr.seen) continue;91            changed_count = std.math.add(usize, changed_count, 1) catch92                return error.FragmentManifestTooLarge;93        }94        const changed = try allocator.alloc([]const u8, changed_count);95        var index: usize = 0;96        for (current) |source| {97            const entry = self.entries.get(source.path);98            if (entry != null and entry.?.live and99                std.mem.eql(u8, &entry.?.key, &source.digest)) continue;100            changed[index] = try std.fs.path.join(allocator, &.{ root, source.path });101            index += 1;102        }103        entries = self.entries.iterator();104        while (entries.next()) |entry| {105            if (!entry.value_ptr.live or entry.value_ptr.seen) continue;106            changed[index] = try std.fs.path.join(allocator, &.{ root, entry.key_ptr.* });107            index += 1;108        }109        std.debug.assert(index == changed.len);110        std.mem.sort([]const u8, changed, {}, scan_files.lessThan);111        return changed;112    }113114    pub fn deinit(self: *Manifest) void {115        self.entries.deinit();116        self.* = undefined;117    }118};119120pub fn loadInto(121    allocator: std.mem.Allocator,122    scratch: std.mem.Allocator,123    root: []const u8,124    executable: Digest,125    key: Digest,126    graph: *graph_mod.Graph,127    stats: *core.Stats,128    timing: *LoadTiming,129    limits: smg.ScanLimits,130) !bool {131    var phase_started = sys.time.nanoTimestamp();132    const path = try fragmentPath(scratch, root, executable, key);133    const encoded = text.readFile(134        scratch,135        path,136        limits.max_fragment_bytes,137    ) catch |err| {138        timing.read_ns += sinceNanoseconds(phase_started);139        return switch (err) {140            error.StreamTooLong => error.MaxFragmentBytesExceeded,141            else => false,142        };143    };144    timing.read_ns += elapsedNanoseconds(&phase_started);145    if (!validFragment(encoded, executable, key)) {146        timing.validation_ns += sinceNanoseconds(phase_started);147        return false;148    }149    timing.validation_ns += elapsedNanoseconds(&phase_started);150    var reader = payloadReader(encoded, executable, key) orelse unreachable;151    const evidence = (try readEvidence(allocator, &reader)) orelse unreachable;152    try mergeEvidence(allocator, stats, evidence);153    const node_count = reader.count() orelse unreachable;154    for (0..node_count) |_| {155        const node = try readNode(allocator, &reader) orelse unreachable;156        try mergeNode(graph, node);157    }158    const edge_count = reader.count() orelse unreachable;159    for (0..edge_count) |_| {160        const edge = try readEdge(allocator, &reader) orelse unreachable;161        if (try graph_mod.addEdgeTracked(graph, edge)) stats.edges_added += 1;162    }163    const deferred_count = reader.count() orelse unreachable;164    stats.activateDeferred(allocator);165    for (0..deferred_count) |_| {166        const edge = try readEdge(allocator, &reader) orelse unreachable;167        try mergeDeferredEdge(allocator, graph, stats, edge);168    }169    std.debug.assert(reader.remaining() == 0);170    timing.fold_ns += sinceNanoseconds(phase_started);171    return true;172}173174pub fn write(175    scratch: std.mem.Allocator,176    root: []const u8,177    executable: Digest,178    key: Digest,179    graph: graph_mod.Graph,180    deferred: []const model.Edge,181    evidence: Evidence,182    limits: smg.ScanLimits,183) !void {184    var encoded: std.Io.Writer.Allocating = .init(scratch);185    defer encoded.deinit();186    const writer = &encoded.writer;187    try writer.writeAll(fragment_magic);188    try writer.writeAll(&executable);189    try writer.writeAll(&key);190    try writeEvidence(writer, evidence);191    try writeCount(writer, graph.nodes.items.len);192    for (graph.nodes.items) |node| try writeNode(writer, node);193    try writeCount(writer, graph.edges.items.len);194    for (graph.edges.items) |edge| try writeEdge(writer, edge);195    try writeCount(writer, deferred.len);196    for (deferred) |edge| try writeEdge(writer, edge);197    if (encoded.written().len > limits.max_fragment_bytes - checksum_bytes) {198        return error.MaxFragmentBytesExceeded;199    }200    var checksum: [checksum_bytes]u8 = undefined;201    std.crypto.hash.sha2.Sha256.hash(encoded.written(), &checksum, .{});202    try writer.writeAll(&checksum);203204    const directory = try executableDirectory(scratch, root, executable);205    try sys.fs.createDirPath(directory);206    const destination = try fragmentPath(scratch, root, executable, key);207    const pid = try sys.process.currentProcessId();208    const temporary = try std.fmt.allocPrint(scratch, "{s}.tmp.{d}", .{ destination, pid });209    errdefer std.Io.Dir.deleteFileAbsolute(fs_io, temporary) catch {};210    {211        var file = try std.Io.Dir.createFileAbsolute(fs_io, temporary, .{ .truncate = true });212        defer file.close(fs_io);213        try file.writeStreamingAll(fs_io, encoded.written());214    }215    try sys.fs.rename(temporary, destination);216}217218pub fn loadManifest(219    allocator: std.mem.Allocator,220    scratch: std.mem.Allocator,221    root: []const u8,222    executable: Digest,223    limits: smg.ScanLimits,224) !Manifest {225    var manifest = Manifest.empty(allocator);226    errdefer manifest.deinit();227    const path = try cachePath(scratch, root, manifest_file_name);228    const encoded = text.readFile(229        scratch,230        path,231        limits.max_manifest_bytes,232    ) catch return manifest;233    const header_bytes = manifest_magic.len + 2 * @sizeOf(Digest) +234        2 * @sizeOf(u64) + checksum_bytes;235    if (!validChecksum(encoded) or encoded.len < header_bytes) return manifest;236    var reader = Reader.init(encoded[0 .. encoded.len - checksum_bytes]);237    if (!reader.expect(manifest_magic)) return manifest;238    const stored_executable = reader.digest() orelse return manifest;239    if (!std.mem.eql(u8, &stored_executable, &executable)) return manifest;240    const project_digest = reader.digest() orelse return manifest;241    const project_files = reader.count() orelse return manifest;242    const count = reader.count() orelse return manifest;243    if (count > reader.remaining() / (1 + @sizeOf(u32) + @sizeOf(Digest))) return manifest;244    try manifest.entries.ensureTotalCapacity(@intCast(count));245    for (0..count) |_| {246        const live = switch (reader.byte() orelse return invalidManifest(&manifest)) {247            0 => false,248            1 => true,249            else => return invalidManifest(&manifest),250        };251        const path_value = try reader.owned(allocator) orelse return invalidManifest(&manifest);252        const key = reader.digest() orelse return invalidManifest(&manifest);253        manifest.entries.putAssumeCapacity(path_value, .{ .key = key, .live = live });254    }255    if (reader.remaining() != 0) return invalidManifest(&manifest);256    manifest.project = .{ .digest = project_digest, .files = project_files };257    return manifest;258}259260fn invalidManifest(manifest: *Manifest) Manifest {261    manifest.project = null;262    manifest.entries.clearRetainingCapacity();263    return manifest.*;264}265266pub fn writeManifest(267    allocator: std.mem.Allocator,268    scratch: std.mem.Allocator,269    root: []const u8,270    executable: Digest,271    project: scan_files.ProjectIdentity,272    previous: *const Manifest,273    current: []const scan_files.SourceIdentity,274    limits: smg.ScanLimits,275) !usize {276    var live = std.StringHashMap(void).init(scratch);277    defer live.deinit();278    try live.ensureTotalCapacity(@intCast(current.len));279    for (current) |entry| live.putAssumeCapacity(entry.path, {});280    var tombstones: std.ArrayList([]const u8) = .empty;281    var prior = previous.entries.iterator();282    while (prior.next()) |entry| {283        if (live.contains(entry.key_ptr.*)) continue;284        try tombstones.append(scratch, entry.key_ptr.*);285    }286    std.mem.sort([]const u8, tombstones.items, {}, scan_files.lessThan);287288    var encoded: std.Io.Writer.Allocating = .init(scratch);289    defer encoded.deinit();290    const writer = &encoded.writer;291    try writer.writeAll(manifest_magic);292    try writer.writeAll(&executable);293    try writer.writeAll(&project.digest);294    try writeCount(writer, project.files);295    const entry_count = std.math.add(usize, current.len, tombstones.items.len) catch296        return error.FragmentManifestTooLarge;297    try writeCount(writer, entry_count);298    for (current) |entry| {299        try writer.writeByte(1);300        try writeBytes(writer, entry.path);301        try writer.writeAll(&entry.digest);302    }303    const empty_key: Digest = @splat(0);304    for (tombstones.items) |path| {305        try writer.writeByte(0);306        try writeBytes(writer, path);307        try writer.writeAll(&empty_key);308    }309    if (encoded.written().len > limits.max_manifest_bytes - checksum_bytes) {310        return error.FragmentManifestTooLarge;311    }312    var checksum: [checksum_bytes]u8 = undefined;313    std.crypto.hash.sha2.Sha256.hash(encoded.written(), &checksum, .{});314    try writer.writeAll(&checksum);315316    const directory = try cacheDirectory(scratch, root);317    try sys.fs.createDirPath(directory);318    const destination = try cachePath(scratch, root, manifest_file_name);319    const temporary_name = try std.fmt.allocPrint(320        scratch,321        "{s}.{d}",322        .{ manifest_temp_file_name, try sys.process.currentProcessId() },323    );324    const temporary = try cachePath(scratch, root, temporary_name);325    errdefer std.Io.Dir.deleteFileAbsolute(fs_io, temporary) catch {};326    {327        var file = try std.Io.Dir.createFileAbsolute(fs_io, temporary, .{ .truncate = true });328        defer file.close(fs_io);329        try file.writeStreamingAll(fs_io, encoded.written());330        try file.sync(fs_io);331    }332    try sys.fs.rename(temporary, destination);333    _ = allocator;334    return tombstones.items.len;335}336337fn writeEvidence(writer: *std.Io.Writer, evidence: Evidence) !void {338    try writeCount(writer, evidence.files);339    try writeCount(writer, evidence.nodes_added);340    try writeCounts(writer, evidence.lang_counts);341    try writeCounts(writer, evidence.type_counts);342}343344fn readEvidence(allocator: std.mem.Allocator, reader: *Reader) !?Evidence {345    const files = reader.count() orelse return null;346    const nodes_added = reader.count() orelse return null;347    const lang_counts = (try readCounts(allocator, reader)) orelse return null;348    const type_counts = (try readCounts(allocator, reader)) orelse return null;349    return .{350        .files = files,351        .nodes_added = nodes_added,352        .lang_counts = lang_counts,353        .type_counts = type_counts,354    };355}356357pub fn mergeEvidence(358    allocator: std.mem.Allocator,359    stats: *core.Stats,360    evidence: Evidence,361) !void {362    stats.files += evidence.files;363    stats.nodes_added += evidence.nodes_added;364    try mergeCounts(allocator, &stats.lang_counts, evidence.lang_counts);365    try mergeCounts(allocator, &stats.type_counts, evidence.type_counts);366}367368pub fn mergeNode(graph: *graph_mod.Graph, node: model.Node) !void {369    if (std.mem.eql(u8, node.type, model.NodeType.package) and370        node.file == null and371        graph_mod.getNode(graph, node.name) != null)372    {373        return;374    }375    try graph_mod.addNode(graph, node);376}377378pub fn mergeDeferredEdge(379    allocator: std.mem.Allocator,380    graph: *graph_mod.Graph,381    stats: *core.Stats,382    edge: model.Edge,383) !void {384    if (std.mem.eql(u8, edge.rel, model.RelType.imports) and385        graph_mod.getNode(graph, edge.target) != null)386    {387        const resolved = model.Edge{388            .source = edge.source,389            .rel = edge.rel,390            .target = edge.target,391            .metadata = try core.scanEdgeMetadata(allocator, edge.metadata),392        };393        if (try graph_mod.addEdgeTracked(graph, resolved)) stats.edges_added += 1;394        return;395    }396    try stats.deferred.append(edge);397}398399fn mergeCounts(400    allocator: std.mem.Allocator,401    target: *std.ArrayList(core.Count),402    source: []const core.Count,403) !void {404    for (source) |incoming| {405        for (target.items) |*existing| {406            if (!std.mem.eql(u8, existing.name, incoming.name)) continue;407            existing.count += incoming.count;408            break;409        } else {410            try target.append(allocator, incoming);411        }412    }413}414415fn writeCounts(writer: *std.Io.Writer, counts: []const core.Count) !void {416    try writeCount(writer, counts.len);417    for (counts) |count| {418        try writeBytes(writer, count.name);419        try writeCount(writer, count.count);420    }421}422423fn readCounts(allocator: std.mem.Allocator, reader: *Reader) !?[]const core.Count {424    const count = reader.count() orelse return null;425    if (count > reader.remaining() / (@sizeOf(u32) + @sizeOf(u64))) return null;426    const values = try allocator.alloc(core.Count, count);427    for (values) |*value| {428        value.name = try reader.owned(allocator) orelse return null;429        value.count = reader.count() orelse return null;430    }431    return values;432}433434fn writeNode(writer: *std.Io.Writer, node: model.Node) !void {435    try writeBytes(writer, node.name);436    try writeBytes(writer, node.type);437    try writeOptionalBytes(writer, node.file);438    try writeOptionalInt(writer, node.line);439    try writeOptionalInt(writer, node.end_line);440    try writeOptionalBytes(writer, node.docstring);441    try writePairs(writer, node.metadata);442}443444fn readNode(allocator: std.mem.Allocator, reader: *Reader) !?model.Node {445    const name = try reader.owned(allocator) orelse return null;446    const node_type = try reader.owned(allocator) orelse return null;447    const file = try reader.optionalOwned(allocator) orelse return null;448    const line = reader.optionalInt() orelse return null;449    const end_line = reader.optionalInt() orelse return null;450    const docstring = try reader.optionalOwned(allocator) orelse return null;451    const metadata = try readPairs(allocator, reader) orelse return null;452    return .{453        .name = name,454        .type = node_type,455        .file = file.value,456        .line = line.value,457        .end_line = end_line.value,458        .docstring = docstring.value,459        .metadata = metadata,460    };461}462463fn writeEdge(writer: *std.Io.Writer, edge: model.Edge) !void {464    try writeBytes(writer, edge.source);465    try writeBytes(writer, edge.rel);466    try writeBytes(writer, edge.target);467    try writePairs(writer, edge.metadata);468}469470fn readEdge(allocator: std.mem.Allocator, reader: *Reader) !?model.Edge {471    const source = try reader.owned(allocator) orelse return null;472    const rel = try reader.owned(allocator) orelse return null;473    const target = try reader.owned(allocator) orelse return null;474    const metadata = try readPairs(allocator, reader) orelse return null;475    return .{ .source = source, .rel = rel, .target = target, .metadata = metadata };476}477478fn writePairs(writer: *std.Io.Writer, pairs: []const model.Pair) !void {479    try writeCount(writer, pairs.len);480    for (pairs) |pair| {481        try writeBytes(writer, pair.key);482        try writeBytes(writer, pair.value);483        try writer.writeByte(@intFromBool(pair.json));484    }485}486487fn readPairs(allocator: std.mem.Allocator, reader: *Reader) !?[]const model.Pair {488    const count = reader.count() orelse return null;489    if (count > reader.remaining() / (2 * @sizeOf(u32) + 1)) return null;490    const pairs = try allocator.alloc(model.Pair, count);491    for (pairs) |*pair| {492        pair.key = try reader.owned(allocator) orelse return null;493        pair.value = try reader.owned(allocator) orelse return null;494        pair.json = switch (reader.byte() orelse return null) {495            0 => false,496            1 => true,497            else => return null,498        };499    }500    return pairs;501}502503fn writeOptionalBytes(writer: *std.Io.Writer, value: ?[]const u8) !void {504    if (value) |bytes| {505        try writer.writeByte(1);506        try writeBytes(writer, bytes);507    } else {508        try writer.writeByte(0);509    }510}511512fn writeOptionalInt(writer: *std.Io.Writer, value: ?i64) !void {513    if (value) |integer| {514        try writer.writeByte(1);515        try writeInteger(i64, writer, integer);516    } else {517        try writer.writeByte(0);518    }519}520521fn writeBytes(writer: *std.Io.Writer, bytes: []const u8) !void {522    if (bytes.len > std.math.maxInt(u32)) return error.FragmentFieldTooLarge;523    try writeInteger(u32, writer, @intCast(bytes.len));524    try writer.writeAll(bytes);525}526527fn writeCount(writer: *std.Io.Writer, count: usize) !void {528    if (count > std.math.maxInt(u64)) return error.FragmentCountTooLarge;529    try writeInteger(u64, writer, @intCast(count));530}531532fn writeInteger(comptime T: type, writer: *std.Io.Writer, value: T) !void {533    var encoded: [@sizeOf(T)]u8 = undefined;534    std.mem.writeInt(T, &encoded, value, .big);535    try writer.writeAll(&encoded);536}537538const OptionalBytes = struct { value: ?[]const u8 };539const OptionalInt = struct { value: ?i64 };540541const Reader = struct {542    bytes: []const u8,543    position: usize = 0,544545    fn init(bytes: []const u8) Reader {546        return .{ .bytes = bytes };547    }548549    fn remaining(self: Reader) usize {550        return self.bytes.len - self.position;551    }552553    fn expect(self: *Reader, expected: []const u8) bool {554        const actual = self.take(expected.len) orelse return false;555        return std.mem.eql(u8, actual, expected);556    }557558    fn digest(self: *Reader) ?Digest {559        const bytes = self.take(@sizeOf(Digest)) orelse return null;560        var value: Digest = undefined;561        @memcpy(&value, bytes);562        return value;563    }564565    fn count(self: *Reader) ?usize {566        const value = self.integer(u64) orelse return null;567        return std.math.cast(usize, value);568    }569570    fn byte(self: *Reader) ?u8 {571        return (self.take(1) orelse return null)[0];572    }573574    fn owned(self: *Reader, allocator: std.mem.Allocator) !?[]const u8 {575        const length = self.integer(u32) orelse return null;576        const bytes = self.take(length) orelse return null;577        return try allocator.dupe(u8, bytes);578    }579580    fn optionalOwned(self: *Reader, allocator: std.mem.Allocator) !?OptionalBytes {581        return switch (self.byte() orelse return null) {582            0 => .{ .value = null },583            1 => .{ .value = try self.owned(allocator) orelse return null },584            else => null,585        };586    }587588    fn optionalInt(self: *Reader) ?OptionalInt {589        return switch (self.byte() orelse return null) {590            0 => .{ .value = null },591            1 => .{ .value = self.integer(i64) orelse return null },592            else => null,593        };594    }595596    fn skipBytes(self: *Reader) bool {597        const length = self.integer(u32) orelse return false;598        _ = self.take(length) orelse return false;599        return true;600    }601602    fn skipOptionalBytes(self: *Reader) bool {603        return switch (self.byte() orelse return false) {604            0 => true,605            1 => self.skipBytes(),606            else => false,607        };608    }609610    fn skipOptionalInt(self: *Reader) bool {611        return switch (self.byte() orelse return false) {612            0 => true,613            1 => self.integer(i64) != null,614            else => false,615        };616    }617618    fn integer(self: *Reader, comptime T: type) ?T {619        const bytes = self.take(@sizeOf(T)) orelse return null;620        return std.mem.readInt(T, bytes[0..@sizeOf(T)], .big);621    }622623    fn take(self: *Reader, length: usize) ?[]const u8 {624        if (length > self.remaining()) return null;625        const start = self.position;626        self.position += length;627        return self.bytes[start..self.position];628    }629};630631fn validChecksum(encoded: []const u8) bool {632    if (encoded.len < checksum_bytes) return false;633    var computed: [checksum_bytes]u8 = undefined;634    std.crypto.hash.sha2.Sha256.hash(encoded[0 .. encoded.len - checksum_bytes], &computed, .{});635    return std.mem.eql(u8, &computed, encoded[encoded.len - checksum_bytes ..]);636}637638fn cacheDirectory(allocator: std.mem.Allocator, root: []const u8) ![]const u8 {639    return try std.fs.path.join(allocator, &.{ root, ".smg", cache_directory_name });640}641642fn validFragment(encoded: []const u8, executable: Digest, key: Digest) bool {643    if (!validChecksum(encoded)) return false;644    var reader = payloadReader(encoded, executable, key) orelse return false;645    if (!skipEvidence(&reader)) return false;646    const node_count = reader.count() orelse return false;647    if (node_count > reader.remaining() / 3) return false;648    for (0..node_count) |_| if (!skipNode(&reader)) return false;649    const edge_count = reader.count() orelse return false;650    if (edge_count > reader.remaining() / 3) return false;651    for (0..edge_count) |_| if (!skipEdge(&reader)) return false;652    const deferred_count = reader.count() orelse return false;653    if (deferred_count > reader.remaining() / 3) return false;654    for (0..deferred_count) |_| if (!skipEdge(&reader)) return false;655    return reader.remaining() == 0;656}657658fn payloadReader(encoded: []const u8, executable: Digest, key: Digest) ?Reader {659    if (encoded.len < fragment_magic.len + 2 * @sizeOf(Digest) + checksum_bytes) return null;660    var reader = Reader.init(encoded[0 .. encoded.len - checksum_bytes]);661    if (!reader.expect(fragment_magic)) return null;662    const stored_executable = reader.digest() orelse return null;663    if (!std.mem.eql(u8, &stored_executable, &executable)) return null;664    const stored_key = reader.digest() orelse return null;665    if (!std.mem.eql(u8, &stored_key, &key)) return null;666    return reader;667}668669fn skipEvidence(reader: *Reader) bool {670    _ = reader.count() orelse return false;671    _ = reader.count() orelse return false;672    return skipCounts(reader) and skipCounts(reader);673}674675fn skipCounts(reader: *Reader) bool {676    const count = reader.count() orelse return false;677    if (count > reader.remaining() / (@sizeOf(u32) + @sizeOf(u64))) return false;678    for (0..count) |_| {679        if (!reader.skipBytes()) return false;680        _ = reader.count() orelse return false;681    }682    return true;683}684685fn skipNode(reader: *Reader) bool {686    return reader.skipBytes() and687        reader.skipBytes() and688        reader.skipOptionalBytes() and689        reader.skipOptionalInt() and690        reader.skipOptionalInt() and691        reader.skipOptionalBytes() and692        skipPairs(reader);693}694695fn skipEdge(reader: *Reader) bool {696    return reader.skipBytes() and697        reader.skipBytes() and698        reader.skipBytes() and699        skipPairs(reader);700}701702fn skipPairs(reader: *Reader) bool {703    const count = reader.count() orelse return false;704    if (count > reader.remaining() / (2 * @sizeOf(u32) + 1)) return false;705    for (0..count) |_| {706        if (!reader.skipBytes() or !reader.skipBytes()) return false;707        const json = reader.byte() orelse return false;708        if (json > 1) return false;709    }710    return true;711}712713fn executableDirectory(714    allocator: std.mem.Allocator,715    root: []const u8,716    executable: Digest,717) ![]const u8 {718    const hex = std.fmt.bytesToHex(executable, .lower);719    return try std.fs.path.join(allocator, &.{ root, ".smg", cache_directory_name, &hex });720}721722fn fragmentPath(723    allocator: std.mem.Allocator,724    root: []const u8,725    executable: Digest,726    key: Digest,727) ![]const u8 {728    const hex = std.fmt.bytesToHex(key, .lower);729    const file_name = try std.fmt.allocPrint(allocator, "{s}.bin", .{&hex});730    return try std.fs.path.join(731        allocator,732        &.{ try executableDirectory(allocator, root, executable), file_name },733    );734}735736fn cachePath(allocator: std.mem.Allocator, root: []const u8, name: []const u8) ![]const u8 {737    return try std.fs.path.join(allocator, &.{ root, ".smg", name });738}739740fn elapsedNanoseconds(started: *i128) u64 {741    const finished = sys.time.nanoTimestamp();742    std.debug.assert(finished >= started.*);743    const elapsed: u64 = @intCast(finished - started.*);744    started.* = finished;745    return elapsed;746}747748fn sinceNanoseconds(started: i128) u64 {749    const finished = sys.time.nanoTimestamp();750    std.debug.assert(finished >= started);751    return @intCast(finished - started);752}753754test "manifest identifies stable changed added and removed sources" {755    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);756    defer arena.deinit();757    const allocator = arena.allocator();758    const first: Digest = @splat(1);759    const second: Digest = @splat(2);760    var manifest = Manifest.empty(allocator);761    defer manifest.deinit();762    const empty = try manifest.changedPaths(allocator, "/repo", &.{});763    try std.testing.expectEqual(@as(usize, 0), empty.len);764    manifest.project = .{ .digest = first, .files = 4 };765    try manifest.entries.put("same.py", .{ .key = first, .live = true });766    try manifest.entries.put("edit.py", .{ .key = first, .live = true });767    try manifest.entries.put("gone.py", .{ .key = first, .live = true });768    try manifest.entries.put("dead.py", .{ .key = @splat(0), .live = false });769    const current = [_]scan_files.SourceIdentity{770        .{ .path = "added.py", .digest = first },771        .{ .path = "edit.py", .digest = second },772        .{ .path = "same.py", .digest = first },773    };774775    const changed = try manifest.changedPaths(allocator, "/repo", &current);776777    try std.testing.expect(manifest.matchesProject(first));778    try std.testing.expect(!manifest.matchesProject(second));779    try std.testing.expectEqual(@as(usize, 3), changed.len);780    try std.testing.expectEqualStrings("/repo/added.py", changed[0]);781    try std.testing.expectEqualStrings("/repo/edit.py", changed[1]);782    try std.testing.expectEqualStrings("/repo/gone.py", changed[2]);783}

1pub const fragment = @import("fragment.zig");