tiny.smg.command.check.refresh

1const std = @import("std");2const sql = @import("sql");3const sys = @import("sys");45const smg = @import("../../root.zig");6const concepts = smg.concepts;7const graph_mod = smg.graph;8const scanner = smg.scan;9const cli_options = smg.cli.options;10const cli_output = smg.cli.output;11const cli_scan = smg.command.scan;12const cli_validation = smg.cli.validation;13const storage = smg.storage;14const runner = @import("run.zig");1516const ScanMode = enum { full, changed, matched };17const signature = "[OPTIONS]";1819pub fn run(allocator: std.mem.Allocator, phases: std.mem.Allocator, args: []const []const u8, base_limits: smg.Limits) !u8 {20    if (try cli_validation.rejectMissingOptionValues(allocator, args, &.{21        cli_options.max_source_file_bytes_option,22        cli_options.max_fragment_bytes_option,23        cli_options.max_git_file_list_bytes_option,24        cli_options.wal_recovery_limit_bytes_option,25    })) return 2;26    const limits = (try cli_validation.configuredLimitsOrReject(27        allocator,28        "refresh-check",29        signature,30        args,31        base_limits,32    )) orelse return 2;33    return runConfigured(allocator, phases, args, limits) catch |err| switch (err) {34        error.MaxSourceFileBytesExceeded => try cli_output.writeByteLimitExceeded(35            allocator,36            "max_source_file_bytes",37            limits.scan.max_source_file_bytes,38            cli_options.max_source_file_bytes_option,39        ),40        error.MaxFragmentBytesExceeded => try cli_output.writeByteLimitExceeded(41            allocator,42            "max_fragment_bytes",43            limits.scan.max_fragment_bytes,44            cli_options.max_fragment_bytes_option,45        ),46        error.MaxGitFileListBytesExceeded => try cli_output.writeByteLimitExceeded(47            allocator,48            "max_git_file_list_bytes",49            limits.scan.max_git_file_list_bytes,50            cli_options.max_git_file_list_bytes_option,51        ),52        error.WalRecoveryLimitBytesExceeded => try cli_output.writeByteLimitExceeded(53            allocator,54            "wal_recovery_limit_bytes",55            limits.storage.wal_recovery_limit_bytes,56            cli_options.wal_recovery_limit_bytes_option,57        ),58        else => return err,59    };60}6162fn runConfigured(allocator: std.mem.Allocator, phases: std.mem.Allocator, args: []const []const u8, limits: smg.Limits) !u8 {63    const total_started = sys.time.nanoTimestamp();64    var phase_started = total_started;65    const root = try storage.initProject(allocator, null, limits.storage);66    const rules_path = try storage.files.rulesPath(allocator, root);67    try cli_output.writeErr(try cli_scan.refreshStartLine(allocator, root, rules_path));68    const source = try loadSource(allocator, phases, root, limits.storage);69    const snapshot_ns = elapsedNanoseconds(&phase_started);70    var refreshed = try refreshGraph(allocator, phases, root, source.head, limits);71    defer refreshed.manifest.deinit();72    const source_verify_ns = try verifySource(allocator, phases, root, &refreshed, limits);73    if (cli_options.hasFlag(args, "--scan") or cli_options.hasFlag(args, "--full")) {74        _ = try cli_output.writeOut(try cli_scan.renderText(75            allocator,76            refreshed.stats,77            cli_options.hasFlag(args, "--full"),78        ));79    }80    try cli_output.writeErr("refresh-check: check rules\n");81    phase_started = sys.time.nanoTimestamp();82    const result = try runner.run(allocator, root, refreshed.graph, source.concepts, args, limits);83    const rule_ns = elapsedNanoseconds(&phase_started);84    if (cli_options.hasFlag(args, "--timing")) {85        try cli_output.writeErr(try timingLine(allocator, .{86            .total_ns = sinceNanoseconds(total_started),87            .snapshot_ns = snapshot_ns,88            .source_state_ns = refreshed.source_state_ns,89            .graph_load_ns = refreshed.graph_load_ns,90            .scan_ns = refreshed.scan_ns,91            .source_verify_ns = source_verify_ns,92            .scan_mode = refreshed.mode,93            .cache = refreshed.cache_timing,94            .publish = refreshed.publish,95            .rule_ns = rule_ns,96        }));97    }98    return result;99}100101const SourceSnapshot = struct {102    head: sql.Hash,103    concepts: []const concepts.Concept,104};105106fn loadSource(107    allocator: std.mem.Allocator,108    phases: std.mem.Allocator,109    root: []const u8,110    limits: smg.StorageLimits,111) !SourceSnapshot {112    var opened = try storage.store.openRead(phases, root, limits);113    defer opened.close();114    return .{115        .head = opened.reader.head,116        .concepts = try concepts.loadFromReader(allocator, &opened.reader),117    };118}119120const RefreshedGraph = struct {121    graph: graph_mod.Graph,122    stats: scanner.Stats,123    project: ?scanner.scan_files.ProjectState,124    executable: ?scanner.scan_files.SourceDigest,125    manifest: scanner.fragment.Manifest,126    mode: ScanMode,127    cache_timing: scanner.CachedProjectTiming,128    publish: storage.graph.PublishTiming,129    source_state_ns: u64,130    graph_load_ns: u64,131    scan_ns: u64,132};133134fn refreshGraph(135    allocator: std.mem.Allocator,136    phases: std.mem.Allocator,137    root: []const u8,138    head: sql.Hash,139    limits: smg.Limits,140) !RefreshedGraph {141    var phase_started = sys.time.nanoTimestamp();142    const executable = storage.source.executableDigest(allocator) catch null;143    const stored_project = if (executable) |identity|144        try storage.source.projectForOwner(allocator, root, head, identity)145    else146        null;147    const project = scanner.scan_files.projectState(allocator, phases, root, &.{}, limits.scan) catch null;148    var manifest = if (executable) |identity|149        try scanner.fragment.loadManifest(allocator, phases, root, identity, limits.scan)150    else151        scanner.fragment.Manifest.empty(allocator);152    errdefer manifest.deinit();153    const mode: ScanMode = if (stored_project != null and project != null and154        std.mem.eql(u8, &stored_project.?, &project.?.project.digest))155        .matched156    else if (stored_project != null and project != null and manifest.matchesProject(stored_project.?))157        .changed158    else159        .full;160    const changed_paths: []const []const u8 = if (mode == .changed)161        try manifest.changedPaths(allocator, root, project.?.sources)162    else163        &.{};164    const source_state_ns = elapsedNanoseconds(&phase_started);165    var graph = switch (mode) {166        .full => graph_mod.init(allocator),167        .changed, .matched => try storage.graph.load(allocator, phases, root, limits.storage),168    };169    const graph_load_ns = elapsedNanoseconds(&phase_started);170    var cache_timing: scanner.CachedProjectTiming = .{};171    const stats = try scanGraph(172        allocator,173        phases,174        &graph,175        root,176        project,177        executable,178        changed_paths,179        mode,180        &cache_timing,181        limits,182    );183    const scan_ns = elapsedNanoseconds(&phase_started);184    try writeProgress(allocator, mode, stats, changed_paths.len, project);185    const publish = switch (mode) {186        .full, .changed => try storage.graph.publishTimed(phases, root, graph, head, limits),187        .matched => try storage.graph.publishMatchedTimed(phases, root, graph, head, limits),188    };189    return .{190        .graph = graph,191        .stats = stats,192        .project = project,193        .executable = executable,194        .manifest = manifest,195        .mode = mode,196        .cache_timing = cache_timing,197        .publish = publish,198        .source_state_ns = source_state_ns,199        .graph_load_ns = graph_load_ns,200        .scan_ns = scan_ns,201    };202}203204fn scanGraph(205    allocator: std.mem.Allocator,206    phases: std.mem.Allocator,207    graph: *graph_mod.Graph,208    root: []const u8,209    project: ?scanner.scan_files.ProjectState,210    executable: ?scanner.scan_files.SourceDigest,211    changed_paths: []const []const u8,212    mode: ScanMode,213    cache_timing: *scanner.CachedProjectTiming,214    limits: smg.Limits,215) !scanner.Stats {216    return switch (mode) {217        .full => if (executable != null and project != null) cached: {218            const cached = try scanner.scanProjectCached(219                allocator,220                phases,221                graph,222                root,223                project.?,224                executable.?,225                limits,226            );227            cache_timing.* = cached.timing;228            break :cached cached.stats;229        } else try scanner.scanPathsWithOptions(230            allocator,231            graph,232            root,233            &.{},234            false,235            .{ .limits = limits, .phases = phases },236        ),237        .changed => if (changed_paths.len == 0)238            scanner.Stats{}239        else240            try scanner.scanPathsWithOptions(241                allocator,242                graph,243                root,244                changed_paths,245                true,246                .{ .limits = limits, .phases = phases },247            ),248        .matched => scanner.Stats{},249    };250}251252fn writeProgress(253    allocator: std.mem.Allocator,254    mode: ScanMode,255    stats: scanner.Stats,256    changed_files: usize,257    project: ?scanner.scan_files.ProjectState,258) !void {259    const line = switch (mode) {260        .full => try cli_scan.refreshAbsentLine(allocator, stats),261        .changed => try cli_scan.refreshChangedLine(allocator, changed_files),262        .matched => try cli_scan.refreshMatchedLine(allocator, project.?.project.files),263    };264    try cli_output.writeErr(line);265}266267fn verifySource(268    allocator: std.mem.Allocator,269    phases: std.mem.Allocator,270    root: []const u8,271    refreshed: *RefreshedGraph,272    limits: smg.Limits,273) !u64 {274    const started = sys.time.nanoTimestamp();275    const expected = if (refreshed.project) |project| project.project.digest else null;276    const verified = if (refreshed.executable != null and expected != null)277        scanner.scan_files.projectIdentity(allocator, phases, root, &.{}, limits.scan) catch null278    else279        null;280    if (refreshed.executable != null and expected != null and verified != null and281        std.mem.eql(u8, &expected.?, &verified.?.digest))282    {283        try updateSourceCaches(allocator, phases, root, refreshed, verified.?.digest, limits);284    } else if (refreshed.executable != null) {285        try cli_output.writeErr("refresh-check: source changed during refresh; next refresh will scan fully\n");286    }287    return sinceNanoseconds(started);288}289290fn updateSourceCaches(291    allocator: std.mem.Allocator,292    phases: std.mem.Allocator,293    root: []const u8,294    refreshed: *RefreshedGraph,295    verified: scanner.scan_files.SourceDigest,296    limits: smg.Limits,297) !void {298    if (refreshed.mode == .changed) {299        const started = sys.time.nanoTimestamp();300        _ = scanner.fragment.writeManifest(301            allocator,302            phases,303            root,304            refreshed.executable.?,305            refreshed.project.?.project,306            &refreshed.manifest,307            refreshed.project.?.sources,308            limits.scan,309        ) catch {310            try cli_output.writeErr(311                "refresh-check: source snapshot cache unavailable; next change will scan fully\n",312            );313        };314        refreshed.cache_timing.manifest_write_ns += sinceNanoseconds(started);315    }316    storage.source.writeForHead(317        allocator,318        root,319        refreshed.publish.head,320        refreshed.executable.?,321        verified,322    ) catch try cli_output.writeErr(323        "refresh-check: source identity cache unavailable; next refresh will scan fully\n",324    );325}326327const Timing = struct {328    total_ns: u64,329    snapshot_ns: u64,330    source_state_ns: u64,331    graph_load_ns: u64,332    scan_ns: u64,333    source_verify_ns: u64,334    scan_mode: ScanMode,335    cache: scanner.CachedProjectTiming,336    publish: storage.graph.PublishTiming,337    rule_ns: u64,338};339340fn timingLine(allocator: std.mem.Allocator, timing: Timing) ![]const u8 {341    return try std.fmt.allocPrint(342        allocator,343        "refresh-check timing total_ms={d} snapshot_ms={d} source_state_ms={d} graph_load_ms={d} scan_ms={d} source_verify_ms={d} scan_mode={s} " ++344            "fragment_manifest_ms={d} fragment_read_ms={d} fragment_validation_ms={d} fragment_fold_ms={d} fragment_scan_ms={d} " ++345            "suffix_index_ms={d} resolution_ms={d} call_metrics_ms={d} fragment_manifest_write_ms={d} " ++346            "graph_encode_ms={d} summary_encode_ms={d} index_encode_ms={d} index_mode={s} " ++347            "node_puts={d} node_deletes={d} edge_puts={d} edge_deletes={d} " ++348            "atomic_publish_ms={d} rule_evaluation_ms={d}\n",349        .{350            milliseconds(timing.total_ns),351            milliseconds(timing.snapshot_ns),352            milliseconds(timing.source_state_ns),353            milliseconds(timing.graph_load_ns),354            milliseconds(timing.scan_ns),355            milliseconds(timing.source_verify_ns),356            @tagName(timing.scan_mode),357            milliseconds(timing.cache.manifest_ns),358            milliseconds(timing.cache.fragment_read_ns),359            milliseconds(timing.cache.fragment_validation_ns),360            milliseconds(timing.cache.fragment_fold_ns),361            milliseconds(timing.cache.fragment_scan_ns),362            milliseconds(timing.cache.suffix_index_ns),363            milliseconds(timing.cache.resolution_ns),364            milliseconds(timing.cache.call_metrics_ns),365            milliseconds(timing.cache.manifest_write_ns),366            milliseconds(timing.publish.graph_ns),367            milliseconds(timing.publish.summary_ns),368            milliseconds(timing.publish.index_ns),369            @tagName(timing.publish.index_kind),370            timing.publish.node_puts,371            timing.publish.node_deletes,372            timing.publish.edge_puts,373            timing.publish.edge_deletes,374            milliseconds(timing.publish.atomicNanoseconds()),375            milliseconds(timing.rule_ns),376        },377    );378}379fn elapsedNanoseconds(started: *i128) u64 {380    const finished = sys.time.nanoTimestamp();381    std.debug.assert(finished >= started.*);382    const elapsed: u64 = @intCast(finished - started.*);383    started.* = finished;384    return elapsed;385}386387fn sinceNanoseconds(started: i128) u64 {388    const finished = sys.time.nanoTimestamp();389    std.debug.assert(finished >= started);390    return @intCast(finished - started);391}392393fn milliseconds(nanoseconds: u64) u64 {394    return nanoseconds / std.time.ns_per_ms;395}396397test "refresh timing names complete owner phases" {398    const rendered = try timingLine(std.testing.allocator, .{399        .total_ns = 36 * std.time.ns_per_ms,400        .snapshot_ns = 1 * std.time.ns_per_ms,401        .source_state_ns = 2 * std.time.ns_per_ms,402        .graph_load_ns = 3 * std.time.ns_per_ms,403        .scan_ns = 2 * std.time.ns_per_ms,404        .source_verify_ns = 4 * std.time.ns_per_ms,405        .scan_mode = .matched,406        .cache = .{407            .manifest_ns = 5 * std.time.ns_per_ms,408            .fragment_read_ns = 6 * std.time.ns_per_ms,409            .fragment_validation_ns = 7 * std.time.ns_per_ms,410            .fragment_fold_ns = 8 * std.time.ns_per_ms,411            .fragment_scan_ns = 9 * std.time.ns_per_ms,412            .suffix_index_ns = 10 * std.time.ns_per_ms,413            .resolution_ns = 11 * std.time.ns_per_ms,414            .call_metrics_ns = 12 * std.time.ns_per_ms,415            .manifest_write_ns = 13 * std.time.ns_per_ms,416        },417        .publish = .{418            .head = @splat(0),419            .begin_ns = 3 * std.time.ns_per_ms,420            .graph_ns = 4 * std.time.ns_per_ms,421            .summary_ns = 5 * std.time.ns_per_ms,422            .index_ns = 6 * std.time.ns_per_ms,423            .index_kind = .incremental,424            .node_puts = 9,425            .node_deletes = 10,426            .edge_puts = 11,427            .edge_deletes = 12,428            .finish_ns = 7 * std.time.ns_per_ms,429        },430        .rule_ns = 8 * std.time.ns_per_ms,431    });432    defer std.testing.allocator.free(rendered);433    try std.testing.expectEqualStrings(434        "refresh-check timing total_ms=36 snapshot_ms=1 source_state_ms=2 graph_load_ms=3 scan_ms=2 source_verify_ms=4 scan_mode=matched " ++435            "fragment_manifest_ms=5 fragment_read_ms=6 fragment_validation_ms=7 fragment_fold_ms=8 fragment_scan_ms=9 " ++436            "suffix_index_ms=10 resolution_ms=11 call_metrics_ms=12 fragment_manifest_write_ms=13 " ++437            "graph_encode_ms=4 summary_encode_ms=5 index_encode_ms=6 index_mode=incremental " ++438            "node_puts=9 node_deletes=10 edge_puts=11 edge_deletes=12 " ++439            "atomic_publish_ms=10 rule_evaluation_ms=8\n",440        rendered,441    );442}

1pub const refresh = @import("refresh.zig");