tiny.cook.Store

1pub const Store = struct {2    gpa: Allocator,3    io: std.Io,4    path: []const u8,5    limits: Limits,67    pub fn init(gpa: Allocator, io: std.Io, path: []const u8, limits: Limits) !Store {8        if (!std.fs.path.isAbsolute(path)) return error.CachePathNotAbsolute;9        if (limits.max_entry_bytes == 0 or limits.max_cache_bytes < limits.max_entry_bytes or10            limits.max_entries == 0 or limits.max_files == 0 or11            limits.max_directories == 0 or limits.max_depth == 0 or12            limits.max_path_bytes < 64) return error.InvalidLimits;13        return .{ .gpa = gpa, .io = io, .path = path, .limits = limits };14    }1516    /// The callback writes ordinary files below an empty directory. The17    /// returned entry holds its manifest lease until deinit.18    pub fn getOrDerive(19        store: Store,20        key: Key,21        context: anytype,22        comptime derive: fn (@TypeOf(context), std.Io, std.Io.Dir) anyerror!void,23    ) !Entry {24        try std.Io.Dir.cwd().createDirPath(store.io, store.path);25        const hex = key.hex();26        const prefix = try std.fs.path.join(store.gpa, &.{ store.path, "v2", hex[0..2] });27        defer store.gpa.free(prefix);28        try std.Io.Dir.cwd().createDirPath(store.io, prefix);29        var key_lock = try store.keyLock(prefix, key);30        defer key_lock.close(store.io);31        try key_lock.lock(store.io, .shared);32        const path = try std.fmt.allocPrint(store.gpa, "{s}/{s}", .{ prefix, hex[2..] });33        errdefer store.gpa.free(path);34        if (try store.exists(path)) return store.load(&key_lock, key, path, true, null);35        key_lock.unlock(store.io);36        try key_lock.lock(store.io, .exclusive);37        if (try store.exists(path)) return store.load(&key_lock, key, path, true, null);38        const summary = try store.publish(prefix, path, key, context, derive);39        return store.load(&key_lock, key, path, false, summary);40    }4142    pub fn open(store: Store, key: Key) !Entry {43        const hex = key.hex();44        const prefix = try std.fs.path.join(store.gpa, &.{ store.path, "v2", hex[0..2] });45        defer store.gpa.free(prefix);46        if (!try store.exists(prefix)) return error.CacheMiss;47        const path = try std.fmt.allocPrint(store.gpa, "{s}/v2/{s}/{s}", .{ store.path, hex[0..2], hex[2..] });48        errdefer store.gpa.free(path);49        if (!try store.exists(path)) return error.CacheMiss;50        var key_lock = try store.keyLock(prefix, key);51        defer key_lock.close(store.io);52        try key_lock.lock(store.io, .shared);53        if (!try store.exists(path)) return error.CacheMiss;54        return store.load(&key_lock, key, path, true, null);55    }5657    fn keyLock(store: Store, prefix: []const u8, key: Key) !std.Io.File {58        const hex = key.hex();59        const lock_path = try std.fmt.allocPrint(store.gpa, "{s}/.key-{s}", .{ prefix, &hex });60        defer store.gpa.free(lock_path);61        return std.Io.Dir.createFileAbsolute(store.io, lock_path, .{62            .truncate = false,63            .read = true,64            .permissions = .fromMode(0o600),65        });66    }6768    fn load(store: Store, key_lock: *std.Io.File, key: Key, path: []u8, hit: bool, published: ?Summary) !Entry {69        var dir = try std.Io.Dir.openDirAbsolute(store.io, path, .{ .iterate = true, .follow_symlinks = false });70        errdefer dir.close(store.io);71        var manifest = dir.openFile(store.io, ".cook", .{ .mode = .read_write }) catch return error.CorruptEntry;72        errdefer manifest.close(store.io);73        try manifest.lock(store.io, .shared);74        key_lock.unlock(store.io);75        const summary = published orelse try store.verify(dir, key);76        try store.touch(manifest);77        return .{ .io = store.io, .gpa = store.gpa, .path = path, .lock = manifest, .dir = dir, .summary = summary, .hit = hit };78    }7980    fn exists(store: Store, path: []const u8) !bool {81        _ = std.Io.Dir.cwd().statFile(store.io, path, .{ .follow_symlinks = false }) catch |err| switch (err) {82            error.FileNotFound => return false,83            else => return err,84        };85        return true;86    }8788    fn publish(89        store: Store,90        prefix: []const u8,91        path: []const u8,92        key: Key,93        context: anytype,94        comptime derive: fn (@TypeOf(context), std.Io, std.Io.Dir) anyerror!void,95    ) !Summary {96        const hex = key.hex();97        const stage_path = try std.fmt.allocPrint(store.gpa, "{s}/.stage-{s}", .{ prefix, &hex });98        defer store.gpa.free(stage_path);99        try std.Io.Dir.cwd().deleteTree(store.io, stage_path);100        try std.Io.Dir.cwd().createDir(store.io, stage_path, .default_dir);101        errdefer std.Io.Dir.cwd().deleteTree(store.io, stage_path) catch {};102        var stage = try std.Io.Dir.openDirAbsolute(store.io, stage_path, .{ .iterate = true, .follow_symlinks = false });103        defer stage.close(store.io);104        try derive(context, store.io, stage);105        const tree = try store.snapshot(stage, true);106        const encoded = encodeManifest(key, tree);107        var manifest = try stage.createFile(store.io, ".cook", .{ .exclusive = true, .permissions = .fromMode(0o600) });108        defer manifest.close(store.io);109        try manifest.writePositionalAll(store.io, &encoded, 0);110        try manifest.sync(store.io);111        try syncDirectory(store.io, stage);112        const lock_path = try std.fs.path.join(store.gpa, &.{ store.path, ".lock" });113        defer store.gpa.free(lock_path);114        var root_lock = try std.Io.Dir.createFileAbsolute(store.io, lock_path, .{115            .truncate = false,116            .read = true,117            .permissions = .fromMode(0o600),118        });119        defer root_lock.close(store.io);120        try root_lock.lock(store.io, .exclusive);121        try store.prune(tree.summary);122        var parent = try std.Io.Dir.openDirAbsolute(store.io, prefix, .{});123        defer parent.close(store.io);124        try std.Io.Dir.cwd().rename(stage_path, std.Io.Dir.cwd(), path, store.io);125        try syncDirectory(store.io, parent);126        return tree.summary;127    }128129    fn verify(store: Store, dir: std.Io.Dir, key: Key) !Summary {130        const encoded = dir.readFileAlloc(store.io, ".cook", store.gpa, .limited(85)) catch return error.CorruptEntry;131        defer store.gpa.free(encoded);132        if (encoded.len != 84 or !std.mem.eql(u8, encoded[0..8], "TCOOK001") or133            !std.mem.eql(u8, encoded[8..40], &key.digest)) return error.CorruptEntry;134        const tree = try store.snapshot(dir, false);135        const expected = encodeManifest(key, tree);136        if (!std.mem.eql(u8, encoded, &expected)) return error.CorruptEntry;137        return tree.summary;138    }139140    fn touch(store: Store, manifest: std.Io.File) !void {141        try manifest.writePositionalAll(store.io, "T", 0);142        try manifest.sync(store.io);143    }144145    fn prune(store: Store, pending: Summary) !void {146        const version = try std.fs.path.join(store.gpa, &.{ store.path, "v2" });147        defer store.gpa.free(version);148        var root = try std.Io.Dir.openDirAbsolute(store.io, version, .{ .iterate = true, .follow_symlinks = false });149        defer root.close(store.io);150        var stages: std.ArrayList(Candidate) = .empty;151        defer {152            for (stages.items) |item| store.gpa.free(item.path);153            stages.deinit(store.gpa);154        }155        var candidates: std.ArrayList(Candidate) = .empty;156        defer {157            for (candidates.items) |item| store.gpa.free(item.path);158            candidates.deinit(store.gpa);159        }160        var total_bytes: u64 = 0;161        var prefixes = root.iterate();162        while (try prefixes.next(store.io)) |prefix| {163            if (prefix.kind != .directory or prefix.name.len != 2) continue;164            const prefix_path = try std.fs.path.join(store.gpa, &.{ version, prefix.name });165            defer store.gpa.free(prefix_path);166            var shard = try root.openDir(store.io, prefix.name, .{ .iterate = true, .follow_symlinks = false });167            defer shard.close(store.io);168            var children = shard.iterate();169            while (try children.next(store.io)) |child| {170                if (child.kind != .directory) continue;171                const path = try std.fs.path.join(store.gpa, &.{ prefix_path, child.name });172                errdefer store.gpa.free(path);173                if (std.mem.startsWith(u8, child.name, ".stage-")) {174                    if (child.name.len != 71) {175                        store.gpa.free(path);176                        continue;177                    }178                    if (stages.items.len == 65536) return error.CacheInventoryExceeded;179                    try stages.append(store.gpa, .{180                        .path = path,181                        .key = Key.parseHex(child.name[7..]) catch return error.CacheInventoryExceeded,182                        .bytes = 0,183                        .mtime = 0,184                    });185                    continue;186                }187                if (child.name.len != 62) {188                    store.gpa.free(path);189                    continue;190                }191                if (candidates.items.len == 65536) return error.CacheInventoryExceeded;192                var hex: [64]u8 = undefined;193                @memcpy(hex[0..2], prefix.name);194                @memcpy(hex[2..], child.name);195                const key = Key.parseHex(&hex) catch return error.CacheInventoryExceeded;196                var entry_dir = try shard.openDir(store.io, child.name, .{ .follow_symlinks = false });197                defer entry_dir.close(store.io);198                const stat = try entry_dir.statFile(store.io, ".cook", .{ .follow_symlinks = false });199                const encoded = try entry_dir.readFileAlloc(store.io, ".cook", store.gpa, .limited(85));200                defer store.gpa.free(encoded);201                const bytes = if (encoded.len == 84 and std.mem.eql(u8, encoded[0..8], "TCOOK001"))202                    std.mem.readInt(u64, encoded[44..52], .little)203                else204                    store.limits.max_entry_bytes;205                total_bytes = std.math.add(u64, total_bytes, bytes) catch return error.CacheInventoryExceeded;206                try candidates.append(store.gpa, .{207                    .path = path,208                    .key = key,209                    .bytes = bytes,210                    .mtime = @intCast(stat.mtime.nanoseconds),211                });212            }213        }214        for (stages.items) |stage| {215            const parent_path = std.fs.path.dirname(stage.path).?;216            var key_lock = try store.keyLock(parent_path, stage.key);217            defer key_lock.close(store.io);218            if (!try key_lock.tryLock(store.io, .exclusive)) continue;219            try std.Io.Dir.cwd().deleteTree(store.io, stage.path);220            var parent = try std.Io.Dir.openDirAbsolute(store.io, parent_path, .{});221            defer parent.close(store.io);222            try syncDirectory(store.io, parent);223        }224        std.mem.sortUnstable(Candidate, candidates.items, {}, candidateLessThan);225        var count: usize = candidates.items.len;226        total_bytes = std.math.add(u64, total_bytes, pending.bytes) catch return error.CacheInventoryExceeded;227        count += 1;228        for (candidates.items) |candidate| {229            if (count <= store.limits.max_entries and total_bytes <= store.limits.max_cache_bytes) break;230            const parent_path = std.fs.path.dirname(candidate.path).?;231            var key_lock = try store.keyLock(parent_path, candidate.key);232            defer key_lock.close(store.io);233            if (!try key_lock.tryLock(store.io, .exclusive)) continue;234            const manifest_path = try std.fs.path.join(store.gpa, &.{ candidate.path, ".cook" });235            defer store.gpa.free(manifest_path);236            var manifest = try std.Io.Dir.openFileAbsolute(store.io, manifest_path, .{ .mode = .read_write });237            defer manifest.close(store.io);238            if (!try manifest.tryLock(store.io, .exclusive)) continue;239            try std.Io.Dir.cwd().deleteTree(store.io, candidate.path);240            var parent = try std.Io.Dir.openDirAbsolute(store.io, parent_path, .{});241            defer parent.close(store.io);242            try syncDirectory(store.io, parent);243            total_bytes -= candidate.bytes;244            count -= 1;245        }246        if (count > store.limits.max_entries or total_bytes > store.limits.max_cache_bytes) return error.CacheFull;247    }248249    fn snapshot(store: Store, dir: std.Io.Dir, sync_files: bool) !Snapshot {250        var records: std.ArrayList(Record) = .empty;251        defer {252            for (records.items) |record| store.gpa.free(record.path);253            records.deinit(store.gpa);254        }255        var summary: Summary = .{ .files = 0, .bytes = 0 };256        var directories: u32 = 0;257        try store.walk(dir, "", 0, sync_files, &records, &summary, &directories);258        std.mem.sortUnstable(Record, records.items, {}, recordLessThan);259        var hasher = Sha256.init(.{});260        for (records.items) |record| {261            hashFramed(&hasher, record.path);262            hasher.update(&.{record.kind});263            var size: [8]u8 = undefined;264            std.mem.writeInt(u64, &size, record.size, .little);265            hasher.update(&size);266            hasher.update(&record.digest);267        }268        var digest: [32]u8 = undefined;269        hasher.final(&digest);270        return .{ .summary = summary, .digest = digest };271    }272273    fn walk(274        store: Store,275        dir: std.Io.Dir,276        relative: []const u8,277        depth: u8,278        sync_files: bool,279        records: *std.ArrayList(Record),280        summary: *Summary,281        directories: *u32,282    ) !void {283        var iterator = dir.iterate();284        while (try iterator.next(store.io)) |item| {285            if (depth == 0 and std.mem.eql(u8, item.name, ".cook")) {286                if (sync_files) return error.ReservedPath;287                continue;288            }289            const child_path = if (relative.len == 0)290                try store.gpa.dupe(u8, item.name)291            else292                try std.fs.path.join(store.gpa, &.{ relative, item.name });293            errdefer store.gpa.free(child_path);294            if (child_path.len > store.limits.max_path_bytes) return error.PathLimitExceeded;295            switch (item.kind) {296                .directory => {297                    if (depth == store.limits.max_depth) return error.DepthLimitExceeded;298                    if (directories.* == store.limits.max_directories) return error.DirectoryLimitExceeded;299                    directories.* += 1;300                    var child = try dir.openDir(store.io, item.name, .{ .iterate = true, .follow_symlinks = false });301                    defer child.close(store.io);302                    try store.walk(child, child_path, depth + 1, sync_files, records, summary, directories);303                    if (sync_files) try syncDirectory(store.io, child);304                    try records.append(store.gpa, .{ .path = child_path, .kind = 0, .size = 0, .digest = @splat(0) });305                },306                .file => {307                    if (summary.files == store.limits.max_files) return error.FileLimitExceeded;308                    var file = try dir.openFile(store.io, item.name, .{});309                    defer file.close(store.io);310                    const stat = try file.stat(store.io);311                    if (stat.size > store.limits.max_entry_bytes -| summary.bytes) return error.EntrySizeExceeded;312                    const digest = try digestFile(store.io, file);313                    if (sync_files) try file.sync(store.io);314                    try records.append(store.gpa, .{ .path = child_path, .kind = 1, .size = stat.size, .digest = digest });315                    summary.files += 1;316                    summary.bytes += stat.size;317                },318                else => return error.UnsupportedFileType,319            }320        }321    }322};

1pub const Store = cache.Store;