tiny.smg.watch

1pub const watch = @import("watch.zig");

1const std = @import("std");2const alloc = @import("alloc");3const pretty = @import("pretty");4const sql = @import("sql");5const sys = @import("sys");6const diff = @import("diff.zig");7const graph_mod = @import("graph.zig");8const limits_mod = @import("limits/root.zig");9const scan = @import("scan/root.zig");10const storage = @import("storage/root.zig");11const text = @import("text/root.zig");1213var interrupted: std.atomic.Value(bool) = .init(false);1415pub const FileState = struct {16    path: []const u8,17    size: u64,18    mtime: i96,19};2021pub const Snapshot = struct {22    states: []FileState,2324    pub fn deinit(self: *Snapshot, allocator: std.mem.Allocator) void {25        for (self.states) |state| allocator.free(state.path);26        allocator.free(self.states);27        self.* = undefined;28    }2930    fn replace(self: *Snapshot, allocator: std.mem.Allocator, next: Snapshot) void {31        self.deinit(allocator);32        self.* = next;33    }3435    fn retainedBytes(self: Snapshot) usize {36        std.debug.assert(self.states.len <= std.math.maxInt(usize) / @sizeOf(FileState));37        var bytes = self.states.len * @sizeOf(FileState);38        for (self.states) |state| {39            std.debug.assert(bytes <= std.math.maxInt(usize) - state.path.len);40            bytes += state.path.len;41        }42        return bytes;43    }44};4546pub const Result = struct {47    files: []const []const u8,48    stats: scan.Stats,49    changes: diff.Result,50};5152pub const Save = *const fn (53    std.mem.Allocator,54    []const u8,55    graph_mod.Graph,56    sql.Hash,57    limits_mod.Limits,58) anyerror!void;5960pub fn run(61    allocator: std.mem.Allocator,62    phases: std.mem.Allocator,63    root: []const u8,64    raw_paths: []const []const u8,65    debounce_seconds: f64,66    limits: limits_mod.Limits,67    save: Save,68) !void {69    interrupted.store(false, .release);70    sys.signal.installInterruptHandler(handleInterrupt);71    defer sys.signal.restoreInterruptDefault();7273    const paths = try watchPaths(allocator, raw_paths);74    var stdout_buffer: [4096]u8 = undefined;75    var stdout = std.Io.File.stdout().writer(std.Options.debug_io, &stdout_buffer);76    var iteration_arena = std.heap.ArenaAllocator.init(phases);77    defer iteration_arena.deinit();78    try pretty.write(&stdout.interface, .{ .text = try startText(iteration_arena.allocator(), paths) }, .{ .width = 88 });79    try stdout.interface.flush();80    resetArena(&iteration_arena);81    var previous = try snapshot(phases, paths);82    defer previous.deinit(phases);83    while (!interrupted.load(.acquire)) {84        defer resetArena(&iteration_arena);85        const scratch = iteration_arena.allocator();86        sys.time.sleepMilliseconds(100);87        if (interrupted.load(.acquire)) break;88        var current = try snapshot(phases, paths);89        var current_live = true;90        defer if (current_live) current.deinit(phases);91        const first_changed = try changedFiles(scratch, previous.states, current.states);92        if (first_changed.len == 0) {93            previous.replace(phases, current);94            current_live = false;95            continue;96        }97        current.deinit(phases);98        current_live = false;99        sleepDebounce(debounce_seconds);100        if (interrupted.load(.acquire)) break;101        var after_debounce = try snapshot(phases, paths);102        var after_debounce_live = true;103        defer if (after_debounce_live) after_debounce.deinit(phases);104        const files = try changedFiles(scratch, previous.states, after_debounce.states);105        const result = rescan(scratch, phases, root, files, limits, save) catch |err| switch (err) {106            error.StaleSourceHead => continue,107            else => return err,108        };109        try pretty.write(&stdout.interface, .{ .text = try eventText(scratch, root, result) }, .{ .width = 88 });110        try stdout.interface.flush();111        previous.replace(phases, after_debounce);112        after_debounce_live = false;113    }114    try pretty.write(&stdout.interface, .{ .text = stoppedText() }, .{ .width = 88 });115    try stdout.interface.flush();116}117118pub fn startText(allocator: std.mem.Allocator, paths: []const []const u8) ![]const u8 {119    var out: std.Io.Writer.Allocating = .init(allocator);120    errdefer out.deinit();121    try out.writer.writeAll("Watching ");122    for (paths, 0..) |path, index| {123        if (index != 0) try out.writer.writeAll(", ");124        try out.writer.writeAll(path);125    }126    try out.writer.writeAll("\nPress Ctrl+C to stop.\n\n");127    return try out.toOwnedSlice();128}129130pub fn eventText(allocator: std.mem.Allocator, root: []const u8, result: Result) ![]const u8 {131    var out: std.Io.Writer.Allocating = .init(allocator);132    errdefer out.deinit();133    try out.writer.writeAll(try fileListText(allocator, root, result.files));134    if (result.changes.empty()) {135        try out.writer.writeAll(" → no structural changes\n");136        return try out.toOwnedSlice();137    }138    try out.writer.writeAll(" → ");139    var wrote = false;140    if (result.changes.added_nodes.len != 0) {141        try out.writer.print("+{d} nodes", .{result.changes.added_nodes.len});142        wrote = true;143    }144    if (result.changes.removed_nodes.len != 0) {145        if (wrote) try out.writer.writeAll(", ");146        try out.writer.print("-{d} nodes", .{result.changes.removed_nodes.len});147        wrote = true;148    }149    if (result.changes.changed_nodes.len != 0) {150        if (wrote) try out.writer.writeAll(", ");151        try out.writer.print("~{d} changed", .{result.changes.changed_nodes.len});152        wrote = true;153    }154    if (result.changes.added_edges.len != 0) {155        if (wrote) try out.writer.writeAll(", ");156        try out.writer.print("+{d} edges", .{result.changes.added_edges.len});157        wrote = true;158    }159    if (result.changes.removed_edges.len != 0) {160        if (wrote) try out.writer.writeAll(", ");161        try out.writer.print("-{d} edges", .{result.changes.removed_edges.len});162    }163    try out.writer.writeByte('\n');164    for (result.changes.added_nodes[0..@min(result.changes.added_nodes.len, 3)]) |node| try out.writer.print("  + [{s}] {s}\n", .{ node.type, node.name });165    for (result.changes.removed_nodes[0..@min(result.changes.removed_nodes.len, 3)]) |node| try out.writer.print("  - [{s}] {s}\n", .{ node.type, node.name });166    for (result.changes.changed_nodes[0..@min(result.changes.changed_nodes.len, 3)]) |changed| {167        for (changed.changes) |change| try out.writer.print("  ~ {s} {s}: {s} → {s}\n", .{ changed.node.name, change.field, change.old orelse "None", change.new orelse "None" });168    }169    for (result.stats.orphaned_manual_edges.items) |edge| try out.writer.print("  orphaned: {s} --{s}--> {s}\n", .{ edge.source, edge.rel, edge.target });170    return try out.toOwnedSlice();171}172173pub fn stoppedText() []const u8 {174    return "\nStopped.\n";175}176177pub fn rescan(178    allocator: std.mem.Allocator,179    phases: std.mem.Allocator,180    root: []const u8,181    files: []const []const u8,182    limits: limits_mod.Limits,183    save: Save,184) !Result {185    const snapshot_value = try storage.graph.loadSnapshot(186        allocator,187        phases,188        root,189        limits.storage,190    );191    const old = snapshot_value.graph;192    var graph = try graph_mod.clone(allocator, phases, old);193    const stats = try scan.scanPathsWithOptions(allocator, &graph, root, files, true, .{ .limits = limits, .phases = phases });194    try save(phases, root, graph, snapshot_value.head, limits);195    return .{ .files = files, .stats = stats, .changes = try diff.graphs(allocator, old, graph, true) };196}197198pub fn watchPaths(allocator: std.mem.Allocator, raw_paths: []const []const u8) ![]const []const u8 {199    var out: std.ArrayList([]const u8) = .empty;200    if (raw_paths.len == 0) {201        try out.append(allocator, try sys.fs.realPathAlloc(allocator, "."));202    } else {203        for (raw_paths) |path| try out.append(allocator, try sys.fs.realPathAlloc(allocator, path));204    }205    return try out.toOwnedSlice(allocator);206}207208pub fn snapshot(allocator: std.mem.Allocator, paths: []const []const u8) !Snapshot {209    var states: std.ArrayList(FileState) = .empty;210    errdefer {211        for (states.items) |state| allocator.free(state.path);212        states.deinit(allocator);213    }214    for (paths) |path| try collect(allocator, path, &states);215    std.mem.sort(FileState, states.items, {}, stateLess);216    return .{ .states = try states.toOwnedSlice(allocator) };217}218219pub fn changedFiles(allocator: std.mem.Allocator, previous: []const FileState, current: []const FileState) ![]const []const u8 {220    return (try scanChangedFiles(allocator, previous, current)).files;221}222223const ChangedFileScan = struct {224    files: []const []const u8,225    path_comparisons: usize,226};227228fn scanChangedFiles(allocator: std.mem.Allocator, previous: []const FileState, current: []const FileState) !ChangedFileScan {229    var out: std.ArrayList([]const u8) = .empty;230    errdefer out.deinit(allocator);231    var previous_index: usize = 0;232    var current_index: usize = 0;233    var path_comparisons: usize = 0;234    const entry_max = std.math.add(usize, previous.len, current.len) catch235        return error.CapacityOverflow;236    for (0..entry_max) |_| {237        if (previous_index == previous.len or current_index == current.len) break;238        const old = previous[previous_index];239        const new = current[current_index];240        switch (pathOrder(old.path, new.path, &path_comparisons)) {241            .lt => {242                try out.append(allocator, old.path);243                previous_index = stateGroupEnd(previous, previous_index, &path_comparisons);244            },245            .gt => {246                try out.append(allocator, new.path);247                current_index = stateGroupEnd(current, current_index, &path_comparisons);248            },249            .eq => {250                previous_index = stateGroupEnd(previous, previous_index, &path_comparisons);251                const current_end = stateGroupEnd(current, current_index, &path_comparisons);252                var changed_path: ?[]const u8 = null;253                for (current[current_index..current_end]) |state| {254                    if (old.size != state.size or old.mtime != state.mtime) {255                        if (changed_path == null) changed_path = state.path;256                    }257                }258                if (changed_path) |path| try out.append(allocator, path);259                current_index = current_end;260            },261        }262    }263    const previous_remaining_max = previous.len - previous_index;264    for (0..previous_remaining_max) |_| {265        if (previous_index == previous.len) break;266        try out.append(allocator, previous[previous_index].path);267        previous_index = stateGroupEnd(previous, previous_index, &path_comparisons);268    }269    const current_remaining_max = current.len - current_index;270    for (0..current_remaining_max) |_| {271        if (current_index == current.len) break;272        try out.append(allocator, current[current_index].path);273        current_index = stateGroupEnd(current, current_index, &path_comparisons);274    }275    std.debug.assert(previous_index == previous.len);276    std.debug.assert(current_index == current.len);277    return .{278        .files = try out.toOwnedSlice(allocator),279        .path_comparisons = path_comparisons,280    };281}282283pub fn isSupported(path: []const u8) bool {284    var parts = std.mem.splitAny(u8, path, "/\\");285    while (parts.next()) |part| {286        if (part.len == 0) continue;287        if (part[0] == '.') return false;288        if (excludedPart(part)) return false;289    }290    return supportedExtension(path);291}292293fn collect(allocator: std.mem.Allocator, path: []const u8, states: *std.ArrayList(FileState)) !void {294    const stat = sys.fs.statFile(path) catch |err| switch (err) {295        error.FileNotFound => return,296        else => return err,297    };298    if (stat.kind == .directory) {299        if (!directoryAllowed(path)) return;300        const entries = sys.fs.listDirAlloc(allocator, path) catch return;301        defer sys.fs.freeEntries(allocator, entries);302        for (entries) |entry| try collect(allocator, entry.path, states);303        return;304    }305    if (stat.kind != .file or !isSupported(path)) return;306    const owned_path = try allocator.dupe(u8, path);307    errdefer allocator.free(owned_path);308    try states.append(allocator, .{309        .path = owned_path,310        .size = stat.size,311        .mtime = stat.mtime.nanoseconds,312    });313}314315fn directoryAllowed(path: []const u8) bool {316    var parts = std.mem.splitAny(u8, path, "/\\");317    while (parts.next()) |part| {318        if (part.len == 0) continue;319        if (part[0] == '.') return false;320        if (excludedPart(part)) return false;321    }322    return true;323}324325fn excludedPart(part: []const u8) bool {326    const exact = [_][]const u8{ ".git", "__pycache__", ".venv", "venv", ".env", "node_modules", ".mypy_cache", ".pytest_cache", ".ruff_cache", ".hypothesis", ".tox", "dist", "build", ".smg", "site-packages", "vendor", "third_party", "zig-cache", ".zig-cache", "zig-out" };327    for (exact) |item| if (std.mem.eql(u8, part, item)) return true;328    return std.mem.endsWith(u8, part, ".egg-info");329}330331fn supportedExtension(path: []const u8) bool {332    const extensions = [_][]const u8{ ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".zig", ".chic", ".c", ".h", ".cpp", ".hpp", ".cc", ".cxx", ".hh", ".hxx", ".cu", ".cuh", ".metal" };333    for (extensions) |extension| if (std.mem.endsWith(u8, path, extension)) return true;334    return false;335}336337fn fileListText(allocator: std.mem.Allocator, root: []const u8, files: []const []const u8) ![]const u8 {338    var out: std.Io.Writer.Allocating = .init(allocator);339    errdefer out.deinit();340    const count = @min(files.len, 3);341    for (files[0..count], 0..) |file, index| {342        if (index != 0) try out.writer.writeAll(", ");343        try out.writer.writeAll(try relativeName(allocator, root, file));344    }345    if (files.len > 3) try out.writer.print(" (+{d} more)", .{files.len - 3});346    return try out.toOwnedSlice();347}348349fn relativeName(allocator: std.mem.Allocator, root: []const u8, path: []const u8) ![]const u8 {350    var rel = path;351    if (std.fs.path.isAbsolute(path) and std.mem.startsWith(u8, path, root)) {352        rel = path[root.len..];353        while (rel.len != 0 and (rel[0] == '/' or rel[0] == '\\')) rel = rel[1..];354    }355    return try text.replaceSeparators(allocator, rel, '/');356}357358fn stateGroupEnd(states: []const FileState, start: usize, path_comparisons: *usize) usize {359    std.debug.assert(start < states.len);360    const path = states[start].path;361    var end = start + 1;362    const remaining_max = states.len - end;363    for (0..remaining_max) |_| {364        if (!pathsEqual(path, states[end].path, path_comparisons)) break;365        end += 1;366    }367    return end;368}369370fn pathOrder(a: []const u8, b: []const u8, path_comparisons: *usize) std.math.Order {371    std.debug.assert(path_comparisons.* < std.math.maxInt(usize));372    path_comparisons.* += 1;373    return std.mem.order(u8, a, b);374}375376fn pathsEqual(a: []const u8, b: []const u8, path_comparisons: *usize) bool {377    std.debug.assert(path_comparisons.* < std.math.maxInt(usize));378    path_comparisons.* += 1;379    return std.mem.eql(u8, a, b);380}381382fn sleepDebounce(debounce_seconds: f64) void {383    if (!std.math.isFinite(debounce_seconds) or debounce_seconds <= 0) return;384    var remaining_ms = secondsToMilliseconds(debounce_seconds);385    while (remaining_ms != 0 and !interrupted.load(.acquire)) {386        const step = @min(remaining_ms, 100);387        sys.time.sleepMilliseconds(step);388        remaining_ms -= step;389    }390}391392fn resetArena(arena: *std.heap.ArenaAllocator) void {393    _ = arena.reset(.free_all);394}395396fn secondsToMilliseconds(seconds: f64) u64 {397    const max_seconds = @as(f64, @floatFromInt(std.math.maxInt(u64))) / 1000.0;398    if (seconds >= max_seconds) return std.math.maxInt(u64);399    return @intFromFloat(@ceil(seconds * 1000.0));400}401402fn handleInterrupt(_: sys.signal.RawSignal) callconv(.c) void {403    interrupted.store(true, .release);404}405406fn stateLess(_: void, a: FileState, b: FileState) bool {407    return std.mem.lessThan(u8, a.path, b.path);408}409410fn stringLess(_: void, a: []const u8, b: []const u8) bool {411    return std.mem.lessThan(u8, a, b);412}413414fn changedFilesReference(allocator: std.mem.Allocator, previous: []const FileState, current: []const FileState) ![]const []const u8 {415    var out: std.ArrayList([]const u8) = .empty;416    errdefer out.deinit(allocator);417    for (current) |state| {418        const old = referenceFindState(previous, state.path);419        if (old == null or old.?.size != state.size or old.?.mtime != state.mtime) {420            try referenceAppendUnique(allocator, &out, state.path);421        }422    }423    for (previous) |state| {424        if (referenceFindState(current, state.path) == null) {425            try referenceAppendUnique(allocator, &out, state.path);426        }427    }428    std.mem.sort([]const u8, out.items, {}, stringLess);429    return try out.toOwnedSlice(allocator);430}431432fn referenceFindState(states: []const FileState, path: []const u8) ?FileState {433    for (states) |state| if (std.mem.eql(u8, state.path, path)) return state;434    return null;435}436437fn referenceAppendUnique(allocator: std.mem.Allocator, out: *std.ArrayList([]const u8), path: []const u8) !void {438    for (out.items) |item| if (std.mem.eql(u8, item, path)) return;439    try out.append(allocator, path);440}441442fn expectChangedFixture(previous: []const FileState, current: []const FileState, expected: []const []const u8) !void {443    const allocator = std.testing.allocator;444    const merged = try changedFiles(allocator, previous, current);445    defer allocator.free(merged);446    const reference = try changedFilesReference(allocator, previous, current);447    defer allocator.free(reference);448    try std.testing.expectEqual(reference.len, merged.len);449    try std.testing.expectEqual(expected.len, merged.len);450    for (merged, reference, expected) |actual, old, wanted| {451        try std.testing.expectEqualStrings(old, actual);452        try std.testing.expectEqualStrings(wanted, actual);453    }454}455456test "watch filters supported and excluded paths" {457    try std.testing.expect(isSupported("src/app/core.py"));458    try std.testing.expect(isSupported("lib/server.ts"));459    try std.testing.expect(isSupported("lib/chic/kernel.chic"));460    try std.testing.expect(!isSupported("__pycache__/core.cpython-311.pyc"));461    try std.testing.expect(!isSupported(".git/hooks/pre-commit"));462    try std.testing.expect(!isSupported("node_modules/foo/index.js"));463    try std.testing.expect(!isSupported(".venv/lib/site.py"));464    try std.testing.expect(!isSupported("README.md"));465    try std.testing.expect(!isSupported("data.csv"));466}467468test "watch detects changed and deleted files" {469    const allocator = std.testing.allocator;470    const previous = [_]FileState{471        .{ .path = "a.py", .size = 1, .mtime = 1 },472        .{ .path = "b.py", .size = 2, .mtime = 2 },473    };474    const current = [_]FileState{475        .{ .path = "a.py", .size = 3, .mtime = 1 },476        .{ .path = "c.py", .size = 1, .mtime = 1 },477    };478    const changed = try changedFiles(allocator, previous[0..], current[0..]);479    defer allocator.free(changed);480    try std.testing.expectEqual(@as(usize, 3), changed.len);481    try std.testing.expectEqualStrings("a.py", changed[0]);482    try std.testing.expectEqualStrings("b.py", changed[1]);483    try std.testing.expectEqualStrings("c.py", changed[2]);484}485486test "watch merge walk matches reference change classifications" {487    const one = [_]FileState{488        .{ .path = "a.py", .size = 1, .mtime = 1 },489    };490    const two = [_]FileState{491        .{ .path = "a.py", .size = 1, .mtime = 1 },492        .{ .path = "b.py", .size = 2, .mtime = 2 },493    };494    const modified = [_]FileState{495        .{ .path = "a.py", .size = 2, .mtime = 1 },496    };497    try expectChangedFixture(&one, &one, &.{});498    try expectChangedFixture(&one, &two, &.{"b.py"});499    try expectChangedFixture(&two, &one, &.{"b.py"});500    try expectChangedFixture(&one, &modified, &.{"a.py"});501502    const mixed_previous = [_]FileState{503        .{ .path = "a.py", .size = 1, .mtime = 1 },504        .{ .path = "b.py", .size = 2, .mtime = 2 },505        .{ .path = "c.py", .size = 3, .mtime = 3 },506        .{ .path = "d.py", .size = 4, .mtime = 4 },507    };508    const mixed_current = [_]FileState{509        .{ .path = "a.py", .size = 1, .mtime = 1 },510        .{ .path = "b.py", .size = 5, .mtime = 2 },511        .{ .path = "d.py", .size = 4, .mtime = 4 },512        .{ .path = "e.py", .size = 5, .mtime = 5 },513    };514    try expectChangedFixture(515        &mixed_previous,516        &mixed_current,517        &.{ "b.py", "c.py", "e.py" },518    );519520    const duplicate_previous = [_]FileState{521        .{ .path = "a.py", .size = 1, .mtime = 1 },522        .{ .path = "a.py", .size = 2, .mtime = 2 },523    };524    const duplicate_current = [_]FileState{525        .{ .path = "a.py", .size = 1, .mtime = 1 },526        .{ .path = "a.py", .size = 3, .mtime = 3 },527    };528    try expectChangedFixture(&duplicate_previous, &duplicate_current, &.{"a.py"});529}530531test "watch merge walk path comparisons grow linearly" {532    const testing = std.testing;533    const entry_count: usize = 4096;534    var paths: [entry_count][4]u8 = undefined;535    var previous: [entry_count]FileState = undefined;536    var current: [entry_count]FileState = undefined;537    for (0..entry_count) |index| {538        std.mem.writeInt(u32, &paths[index], @intCast(index), .big);539        previous[index] = .{ .path = &paths[index], .size = 1, .mtime = 1 };540        current[index] = .{541            .path = &paths[index],542            .size = if (index + 1 == entry_count) 2 else 1,543            .mtime = 1,544        };545    }546    const scanned = try scanChangedFiles(testing.allocator, &previous, &current);547    defer testing.allocator.free(scanned.files);548    try testing.expectEqual(@as(usize, 1), scanned.files.len);549    try testing.expectEqualSlices(u8, &paths[entry_count - 1], scanned.files[0]);550    const comparison_max = entry_count * 3;551    try testing.expect(scanned.path_comparisons <= comparison_max);552}553554test "watch snapshot replacements retain only live snapshot bytes" {555    const testing = std.testing;556    const root = try std.fmt.allocPrint(557        testing.allocator,558        "/tmp/smg-watch-retention-{x}",559        .{@intFromPtr(&interrupted)},560    );561    defer testing.allocator.free(root);562    sys.fs.deleteTree(root) catch {};563    defer sys.fs.deleteTree(root) catch {};564    try sys.fs.createDirPath(root);565    const file_path = try std.fs.path.join(testing.allocator, &.{ root, "main.zig" });566    defer testing.allocator.free(file_path);567    try text.writeFile(file_path, "pub fn main() void {}\n");568569    var meter = alloc.LimitAllocator.init(testing.allocator, 1024 * 1024);570    defer meter.deinit();571    const paths = [_][]const u8{root};572    var stable = try snapshot(meter.allocator(), &paths);573    defer stable.deinit(meter.allocator());574    try testing.expectEqual(@as(usize, 1), stable.states.len);575    const retained_bytes = stable.retainedBytes();576    try testing.expectEqual(retained_bytes, meter.liveBytes());577578    const replacement_count: usize = 32;579    for (0..replacement_count) |_| {580        const next = try snapshot(meter.allocator(), &paths);581        stable.replace(meter.allocator(), next);582        try testing.expectEqual(retained_bytes, stable.retainedBytes());583        try testing.expectEqual(retained_bytes, meter.liveBytes());584    }585}586587test "watch stop text matches Python interrupt footer" {588    try std.testing.expectEqualStrings("\nStopped.\n", stoppedText());589}590591test "watch debounce sleep observes interrupts" {592    interrupted.store(false, .release);593    handleInterrupt(.INT);594    sleepDebounce(10);595    try std.testing.expect(interrupted.load(.acquire));596    interrupted.store(false, .release);597}