tiny.smg.concepts

1const std = @import("std");2const pretty = @import("pretty");3const sql = @import("sql");4const graph_mod = @import("graph.zig");5const limits_mod = @import("limits/root.zig");6const search = @import("search.zig");7const view = @import("view.zig");8const model = @import("model.zig");9const storage = @import("storage/root.zig");10const pretty_json = pretty.json;11const testing_limits = @import("root.zig").default_limits;1213pub const Concept = struct {14    name: []const u8,15    prefixes: []const []const u8 = &.{},16    sync_points: []const []const u8 = &.{},17};1819pub const Snapshot = struct {20    concepts: []const Concept,21    head: sql.Hash,22};2324pub const Summary = struct {25    name: []const u8,26    prefixes: []const []const u8,27    anchors: []const []const u8,28    members: usize,29    internal_edges: usize,30    cross_in: usize,31    cross_out: usize,32    sync_fan_out: usize,33    sync_density: f64,34    sync_asymmetry: f64,35};3637pub const RelCount = struct {38    rel: []const u8,39    count: usize,40};4142pub const Dependency = struct {43    source: []const u8,44    target: []const u8,45    edge_count: usize,46    rels: []const RelCount,47    unsanctioned_count: usize,48    unsanctioned_rels: []const RelCount,49    witnesses: []const model.Edge,50    allowed_sync: bool,51};5253pub const Violation = struct {54    source: []const u8,55    target: []const u8,56    message: []const u8,57    rels: []const RelCount,58    source_candidates: []const []const u8,59    target_candidates: []const []const u8,60    witnesses: []const model.Edge,61};6263pub const Analysis = struct {64    declared: []const Summary,65    dependencies: []const Dependency,66    violations: []const Violation,67};6869const Materialized = struct {70    concept: Concept,71    anchors: []const []const u8,72    members: []const []const u8,73};7475const Owner = struct {76    member: []const u8,77    concept: []const u8,78};7980const Counter = struct {81    name: []const u8,82    count: usize,83};8485const PairEdges = struct {86    source: []const u8,87    target: []const u8,88    edges: std.ArrayList(model.Edge) = .empty,89    unsanctioned: std.ArrayList(model.Edge) = .empty,90};9192const MaterializeResult = struct {93    materialized: []const Materialized,94    owners: []const Owner,95};9697pub fn analyze(allocator: std.mem.Allocator, graph: graph_mod.Graph, concept_list: []const Concept, overlap_message: ?*[]const u8) !Analysis {98    const materialized_result = try materialize(allocator, graph, concept_list, overlap_message);99    const materialized = materialized_result.materialized;100    const owners = materialized_result.owners;101    var internal_edges: std.ArrayList(Counter) = .empty;102    var cross_in: std.ArrayList(Counter) = .empty;103    var cross_out: std.ArrayList(Counter) = .empty;104    var touched_edges: std.ArrayList(Counter) = .empty;105    var pairs: std.ArrayList(PairEdges) = .empty;106    for (try view.allEdges(graph, allocator)) |edge| {107        if (std.mem.eql(u8, edge.rel, model.RelType.contains)) continue;108        const source_owner = ownerFor(owners, edge.source);109        const target_owner = ownerFor(owners, edge.target);110        if (source_owner) |owner| try incrementCounter(allocator, &touched_edges, owner);111        if (target_owner) |owner| {112            if (source_owner == null or !std.mem.eql(u8, source_owner.?, owner)) try incrementCounter(allocator, &touched_edges, owner);113        }114        if (source_owner == null or target_owner == null) continue;115        if (std.mem.eql(u8, source_owner.?, target_owner.?)) {116            try incrementCounter(allocator, &internal_edges, source_owner.?);117            continue;118        }119        try incrementCounter(allocator, &cross_out, source_owner.?);120        try incrementCounter(allocator, &cross_in, target_owner.?);121        const pair_index = try pairIndex(allocator, &pairs, source_owner.?, target_owner.?);122        try pairs.items[pair_index].edges.append(allocator, edge);123        const source_concept = conceptByName(materialized, source_owner.?).?;124        const target_concept = conceptByName(materialized, target_owner.?).?;125        if (!allowedSync(edge, source_concept.concept, target_concept.concept)) try pairs.items[pair_index].unsanctioned.append(allocator, edge);126    }127    std.mem.sort(PairEdges, pairs.items, {}, pairLess);128    var declared: std.ArrayList(Summary) = .empty;129    for (materialized) |item| {130        const in_count = counterValue(cross_in.items, item.concept.name);131        const out_count = counterValue(cross_out.items, item.concept.name);132        const touched = counterValue(touched_edges.items, item.concept.name);133        try declared.append(allocator, .{134            .name = item.concept.name,135            .prefixes = item.concept.prefixes,136            .anchors = item.anchors,137            .members = item.members.len,138            .internal_edges = counterValue(internal_edges.items, item.concept.name),139            .cross_in = in_count,140            .cross_out = out_count,141            .sync_fan_out = try syncFanOut(allocator, pairs.items, item.concept.name),142            .sync_density = if (touched == 0) 0.0 else roundFloat(@as(f64, @floatFromInt(in_count + out_count)) / @as(f64, @floatFromInt(touched)), 6),143            .sync_asymmetry = roundFloat(try syncAsymmetry(allocator, pairs.items, item.concept.name), 6),144        });145    }146    var dependencies: std.ArrayList(Dependency) = .empty;147    var violations: std.ArrayList(Violation) = .empty;148    for (pairs.items) |pair| {149        const edges = try edgeSlice(allocator, pair.edges.items);150        const unsanctioned = try edgeSlice(allocator, pair.unsanctioned.items);151        const dependency_witnesses = if (unsanctioned.len != 0) try witnessEdges(allocator, unsanctioned) else try witnessEdges(allocator, edges);152        try dependencies.append(allocator, .{153            .source = pair.source,154            .target = pair.target,155            .edge_count = edges.len,156            .rels = try relCounts(allocator, edges),157            .unsanctioned_count = unsanctioned.len,158            .unsanctioned_rels = try relCounts(allocator, unsanctioned),159            .witnesses = dependency_witnesses,160            .allowed_sync = unsanctioned.len == 0,161        });162        if (unsanctioned.len != 0) {163            try violations.append(allocator, .{164                .source = pair.source,165                .target = pair.target,166                .message = try std.fmt.allocPrint(allocator, "{d} unsanctioned cross-concept edge(s)", .{unsanctioned.len}),167                .rels = try relCounts(allocator, unsanctioned),168                .source_candidates = try candidateSources(allocator, unsanctioned),169                .target_candidates = try candidateTargets(allocator, unsanctioned),170                .witnesses = try witnessEdges(allocator, unsanctioned),171            });172        }173    }174    return .{175        .declared = try declared.toOwnedSlice(allocator),176        .dependencies = try dependencies.toOwnedSlice(allocator),177        .violations = try violations.toOwnedSlice(allocator),178    };179}180181pub fn writeAnalysisJson(allocator: std.mem.Allocator, writer: *pretty_json.Writer, result: Analysis) !void {182    try writer.beginObject();183    try writer.objectField("declared");184    try writer.beginArray();185    for (result.declared) |summary| {186        try writer.beginObject();187        try writer.objectField("name");188        try writer.write(summary.name);189        try writer.objectField("prefixes");190        try writeStringArray(writer, summary.prefixes);191        try writer.objectField("anchors");192        try writeStringArray(writer, summary.anchors);193        try writer.objectField("members");194        try writer.write(summary.members);195        try writer.objectField("internal_edges");196        try writer.write(summary.internal_edges);197        try writer.objectField("cross_in");198        try writer.write(summary.cross_in);199        try writer.objectField("cross_out");200        try writer.write(summary.cross_out);201        try writer.objectField("sync_fan_out");202        try writer.write(summary.sync_fan_out);203        try writer.objectField("sync_density");204        try writeFloat(allocator, writer, summary.sync_density);205        try writer.objectField("sync_asymmetry");206        try writeFloat(allocator, writer, summary.sync_asymmetry);207        try writer.endObject();208    }209    try writer.endArray();210    try writer.objectField("dependencies");211    try writer.beginArray();212    for (result.dependencies) |dependency| {213        try writer.beginObject();214        try writer.objectField("source");215        try writer.write(dependency.source);216        try writer.objectField("target");217        try writer.write(dependency.target);218        try writer.objectField("edge_count");219        try writer.write(dependency.edge_count);220        try writer.objectField("rels");221        try writeRelCounts(writer, dependency.rels);222        try writer.objectField("unsanctioned_count");223        try writer.write(dependency.unsanctioned_count);224        try writer.objectField("unsanctioned_rels");225        try writeRelCounts(writer, dependency.unsanctioned_rels);226        try writer.objectField("witnesses");227        try writeWitnesses(writer, dependency.witnesses);228        try writer.objectField("allowed_sync");229        try writer.write(dependency.allowed_sync);230        try writer.endObject();231    }232    try writer.endArray();233    try writer.objectField("violations");234    try writer.beginArray();235    for (result.violations) |violation| {236        try writer.beginObject();237        try writer.objectField("source");238        try writer.write(violation.source);239        try writer.objectField("target");240        try writer.write(violation.target);241        try writer.objectField("message");242        try writer.write(violation.message);243        try writer.objectField("rels");244        try writeRelCounts(writer, violation.rels);245        try writer.objectField("sync_candidates");246        try writer.beginObject();247        try writer.objectField("source");248        try writeStringArray(writer, violation.source_candidates);249        try writer.objectField("target");250        try writeStringArray(writer, violation.target_candidates);251        try writer.endObject();252        try writer.objectField("sync_commands");253        try writer.beginObject();254        try writer.objectField("source");255        try writer.beginArray();256        for (violation.source_candidates) |candidate| {257            try writer.write(try std.fmt.allocPrint(allocator, "smg concept sync-point {s} {s}", .{ violation.source, candidate }));258        }259        try writer.endArray();260        try writer.objectField("target");261        try writer.beginArray();262        for (violation.target_candidates) |candidate| {263            try writer.write(try std.fmt.allocPrint(allocator, "smg concept sync-point {s} {s}", .{ violation.target, candidate }));264        }265        try writer.endArray();266        try writer.endObject();267        try writer.objectField("witnesses");268        try writeWitnesses(writer, violation.witnesses);269        try writer.endObject();270    }271    try writer.endArray();272    try writer.endObject();273}274275fn materialize(allocator: std.mem.Allocator, graph: graph_mod.Graph, concept_list: []const Concept, overlap_message: ?*[]const u8) !MaterializeResult {276    const sorted = try allocator.dupe(Concept, concept_list);277    std.mem.sort(Concept, sorted, {}, conceptLess);278    var package_or_module: std.ArrayList([]const u8) = .empty;279    var all_names: std.ArrayList([]const u8) = .empty;280    for (try view.allNodes(graph, allocator, null)) |node| {281        try all_names.append(allocator, node.name);282        if (std.mem.eql(u8, node.type, model.NodeType.package) or std.mem.eql(u8, node.type, model.NodeType.module)) try package_or_module.append(allocator, node.name);283    }284    var out: std.ArrayList(Materialized) = .empty;285    var owners: std.ArrayList(Owner) = .empty;286    for (sorted) |concept| {287        const anchors = try resolveAnchors(allocator, package_or_module.items, concept.prefixes);288        var members: std.ArrayList([]const u8) = .empty;289        for (all_names.items) |name| {290            for (anchors) |anchor| {291                if (matchesPrefix(name, anchor)) {292                    try appendUniqueList(allocator, &members, name);293                    break;294                }295            }296        }297        std.mem.sort([]const u8, members.items, {}, stringLess);298        for (members.items) |member| {299            if (ownerFor(owners.items, member)) |owner| {300                if (!std.mem.eql(u8, owner, concept.name)) {301                    if (overlap_message) |message| message.* = try std.fmt.allocPrint(allocator, "concept overlap on '{s}': '{s}' and '{s}' both claim it", .{ member, owner, concept.name });302                    return error.ConceptOverlap;303                }304            }305            try owners.append(allocator, .{ .member = member, .concept = concept.name });306        }307        try out.append(allocator, .{ .concept = concept, .anchors = anchors, .members = try members.toOwnedSlice(allocator) });308    }309    return .{ .materialized = try out.toOwnedSlice(allocator), .owners = try owners.toOwnedSlice(allocator) };310}311312fn resolveAnchors(allocator: std.mem.Allocator, package_or_module: []const []const u8, prefixes: []const []const u8) ![]const []const u8 {313    var matches: std.ArrayList([]const u8) = .empty;314    for (prefixes) |prefix| {315        for (package_or_module) |name| {316            if (matchesPrefix(name, prefix)) try appendUniqueList(allocator, &matches, name);317        }318    }319    std.mem.sort([]const u8, matches.items, {}, anchorLess);320    var anchors: std.ArrayList([]const u8) = .empty;321    for (matches.items) |name| {322        var covered = false;323        for (anchors.items) |anchor| {324            if (matchesPrefix(name, anchor)) {325                covered = true;326                break;327            }328        }329        if (!covered) try anchors.append(allocator, name);330    }331    return try anchors.toOwnedSlice(allocator);332}333334fn conceptByName(materialized: []const Materialized, name: []const u8) ?Materialized {335    for (materialized) |item| if (std.mem.eql(u8, item.concept.name, name)) return item;336    return null;337}338339fn ownerFor(owners: []const Owner, member: []const u8) ?[]const u8 {340    for (owners) |owner| if (std.mem.eql(u8, owner.member, member)) return owner.concept;341    return null;342}343344fn incrementCounter(allocator: std.mem.Allocator, counters: *std.ArrayList(Counter), name: []const u8) !void {345    for (counters.items) |*counter| {346        if (std.mem.eql(u8, counter.name, name)) {347            counter.count += 1;348            return;349        }350    }351    try counters.append(allocator, .{ .name = name, .count = 1 });352}353354fn counterValue(counters: []const Counter, name: []const u8) usize {355    for (counters) |counter| if (std.mem.eql(u8, counter.name, name)) return counter.count;356    return 0;357}358359fn pairIndex(allocator: std.mem.Allocator, pairs: *std.ArrayList(PairEdges), source: []const u8, target: []const u8) !usize {360    for (pairs.items, 0..) |pair, index| {361        if (std.mem.eql(u8, pair.source, source) and std.mem.eql(u8, pair.target, target)) return index;362    }363    try pairs.append(allocator, .{ .source = source, .target = target });364    return pairs.items.len - 1;365}366367fn allowedSync(edge: model.Edge, source: Concept, target: Concept) bool {368    return matchesAnyPrefix(edge.source, source.sync_points) or matchesAnyPrefix(edge.target, target.sync_points);369}370371fn matchesAnyPrefix(name: []const u8, prefixes: []const []const u8) bool {372    for (prefixes) |prefix| if (matchesPrefix(name, prefix)) return true;373    return false;374}375376fn syncFanOut(allocator: std.mem.Allocator, pairs: []const PairEdges, name: []const u8) !usize {377    var targets: std.ArrayList([]const u8) = .empty;378    for (pairs) |pair| {379        if (std.mem.eql(u8, pair.source, name)) try appendUniqueList(allocator, &targets, pair.target);380    }381    return targets.items.len;382}383384fn syncAsymmetry(allocator: std.mem.Allocator, pairs: []const PairEdges, name: []const u8) !f64 {385    var related: std.ArrayList([]const u8) = .empty;386    for (pairs) |pair| {387        if (std.mem.eql(u8, pair.source, name)) try appendUniqueList(allocator, &related, pair.target);388        if (std.mem.eql(u8, pair.target, name)) try appendUniqueList(allocator, &related, pair.source);389    }390    if (related.items.len == 0) return 0.0;391    var bidirectional: usize = 0;392    var unidirectional: usize = 0;393    for (related.items) |other| {394        const outgoing = hasPair(pairs, name, other);395        const incoming = hasPair(pairs, other, name);396        if (outgoing and incoming) {397            bidirectional += 1;398        } else {399            unidirectional += 1;400        }401    }402    return @as(f64, @floatFromInt(unidirectional)) / @as(f64, @floatFromInt(@max(@as(usize, 1), bidirectional)));403}404405fn hasPair(pairs: []const PairEdges, source: []const u8, target: []const u8) bool {406    for (pairs) |pair| if (std.mem.eql(u8, pair.source, source) and std.mem.eql(u8, pair.target, target)) return true;407    return false;408}409410fn edgeSlice(allocator: std.mem.Allocator, edges: []const model.Edge) ![]const model.Edge {411    const out = try allocator.dupe(model.Edge, edges);412    std.mem.sort(model.Edge, out, {}, edgeLess);413    return out;414}415416fn witnessEdges(allocator: std.mem.Allocator, edges: []const model.Edge) ![]const model.Edge {417    const count = @min(edges.len, 5);418    return try allocator.dupe(model.Edge, edges[0..count]);419}420421fn relCounts(allocator: std.mem.Allocator, edges: []const model.Edge) ![]const RelCount {422    var out: std.ArrayList(RelCount) = .empty;423    for (edges) |edge| {424        var found = false;425        for (out.items) |*item| {426            if (std.mem.eql(u8, item.rel, edge.rel)) {427                item.count += 1;428                found = true;429                break;430            }431        }432        if (!found) try out.append(allocator, .{ .rel = edge.rel, .count = 1 });433    }434    std.mem.sort(RelCount, out.items, {}, relCountLess);435    return try out.toOwnedSlice(allocator);436}437438fn candidateSources(allocator: std.mem.Allocator, edges: []const model.Edge) ![]const []const u8 {439    var out: std.ArrayList([]const u8) = .empty;440    for (edges) |edge| try appendUniqueList(allocator, &out, edge.source);441    return try limitStrings(allocator, out.items, 5);442}443444fn candidateTargets(allocator: std.mem.Allocator, edges: []const model.Edge) ![]const []const u8 {445    var out: std.ArrayList([]const u8) = .empty;446    for (edges) |edge| try appendUniqueList(allocator, &out, edge.target);447    return try limitStrings(allocator, out.items, 5);448}449450fn limitStrings(allocator: std.mem.Allocator, values: []const []const u8, limit: usize) ![]const []const u8 {451    return try allocator.dupe([]const u8, values[0..@min(values.len, limit)]);452}453454fn writeStringArray(writer: *pretty_json.Writer, values: []const []const u8) !void {455    try writer.beginArray();456    for (values) |value| try writer.write(value);457    try writer.endArray();458}459460fn writeRelCounts(writer: *pretty_json.Writer, counts: []const RelCount) !void {461    try writer.beginObject();462    for (counts) |count| {463        try writer.objectField(count.rel);464        try writer.write(count.count);465    }466    try writer.endObject();467}468469fn writeWitnesses(writer: *pretty_json.Writer, edges: []const model.Edge) !void {470    try writer.beginArray();471    for (edges) |edge| {472        try writer.beginObject();473        try writer.objectField("kind");474        try writer.write("edge");475        try writer.objectField("edges");476        try writer.beginArray();477        try writeEdgeBrief(writer, edge);478        try writer.endArray();479        try writer.endObject();480    }481    try writer.endArray();482}483484fn writeEdgeBrief(writer: *pretty_json.Writer, edge: model.Edge) !void {485    try writer.beginObject();486    try writer.objectField("source");487    try writer.write(edge.source);488    try writer.objectField("rel");489    try writer.write(edge.rel);490    try writer.objectField("target");491    try writer.write(edge.target);492    try writer.endObject();493}494495fn writeFloat(allocator: std.mem.Allocator, writer: *pretty_json.Writer, value: f64) !void {496    const raw = try std.fmt.allocPrint(allocator, "{d:.6}", .{value});497    var end = raw.len;498    while (end > 0 and raw[end - 1] == '0') end -= 1;499    if (end > 0 and raw[end - 1] == '.') end += 1;500    try writer.raw(raw[0..end]);501}502503fn roundFloat(value: f64, decimals: usize) f64 {504    var scale: f64 = 1.0;505    for (0..decimals) |_| scale *= 10.0;506    return @round(value * scale) / scale;507}508509fn matchesPrefix(name: []const u8, prefix: []const u8) bool {510    return std.mem.eql(u8, name, prefix) or (name.len > prefix.len and std.mem.startsWith(u8, name, prefix) and name[prefix.len] == '.');511}512513fn countDots(name: []const u8) usize {514    var count: usize = 0;515    for (name) |byte| {516        if (byte == '.') count += 1;517    }518    return count;519}520521fn conceptLess(_: void, a: Concept, b: Concept) bool {522    return std.mem.lessThan(u8, a.name, b.name);523}524525fn anchorLess(_: void, a: []const u8, b: []const u8) bool {526    const dots_a = countDots(a);527    const dots_b = countDots(b);528    if (dots_a != dots_b) return dots_a < dots_b;529    return std.mem.lessThan(u8, a, b);530}531532fn pairLess(_: void, a: PairEdges, b: PairEdges) bool {533    const source = std.mem.order(u8, a.source, b.source);534    if (source != .eq) return source == .lt;535    return std.mem.lessThan(u8, a.target, b.target);536}537538fn edgeLess(_: void, a: model.Edge, b: model.Edge) bool {539    const source = std.mem.order(u8, a.source, b.source);540    if (source != .eq) return source == .lt;541    const rel = std.mem.order(u8, a.rel, b.rel);542    if (rel != .eq) return rel == .lt;543    return std.mem.lessThan(u8, a.target, b.target);544}545546fn relCountLess(_: void, a: RelCount, b: RelCount) bool {547    return std.mem.lessThan(u8, a.rel, b.rel);548}549550fn stringLess(_: void, a: []const u8, b: []const u8) bool {551    return std.mem.lessThan(u8, a, b);552}553554pub fn load(555    allocator: std.mem.Allocator,556    root: []const u8,557    limits: limits_mod.Storage,558) ![]const Concept {559    return (try loadSnapshot(allocator, root, limits)).concepts;560}561562pub fn loadSnapshot(563    allocator: std.mem.Allocator,564    root: []const u8,565    limits: limits_mod.Storage,566) !Snapshot {567    var opened = try storage.store.openRead(allocator, root, limits);568    defer opened.close();569    return .{570        .concepts = try loadFromReader(allocator, &opened.reader),571        .head = opened.reader.head,572    };573}574575pub fn loadFromReader(allocator: std.mem.Allocator, reader: *storage.database.Reader) ![]const Concept {576    const documents = try storage.database.read.readerConceptDocuments(reader, allocator, allocator);577    defer storage.rows.freeConceptDocumentRowSlice(allocator, allocator, documents);578    var out: std.ArrayList(Concept) = .empty;579    for (documents) |document| {580        var parsed = std.json.parseFromSlice(std.json.Value, allocator, document.document, .{}) catch |err| switch (err) {581            error.OutOfMemory => return error.OutOfMemory,582            else => continue,583        };584        defer parsed.deinit();585        const object = switch (parsed.value) {586            .object => |object| object,587            else => continue,588        };589        const name = stringField(object, "name") orelse continue;590        try out.append(allocator, .{591            .name = try allocator.dupe(u8, name),592            .prefixes = try stringArray(allocator, object.get("prefixes")),593            .sync_points = try stringArray(allocator, object.get("sync_points")),594        });595    }596    return try out.toOwnedSlice(allocator);597}598599pub fn publish(600    allocator: std.mem.Allocator,601    root: []const u8,602    concepts: []const Concept,603    expected_head: sql.Hash,604    limits: limits_mod.Limits,605) !void {606    const sorted = try allocator.dupe(Concept, concepts);607    defer allocator.free(sorted);608    std.mem.sort(Concept, sorted, {}, conceptLess);609    const puts = try allocator.alloc(storage.rows.ConceptDocumentPut, sorted.len);610    defer allocator.free(puts);611    var rendered: std.ArrayList([]const u8) = .empty;612    defer {613        for (rendered.items) |document| allocator.free(document);614        rendered.deinit(allocator);615    }616    try rendered.ensureTotalCapacityPrecise(allocator, sorted.len);617    for (sorted, 0..) |concept, index| {618        const document = try render(allocator, concept);619        rendered.appendAssumeCapacity(document);620        puts[index] = .{621            .rowid = @intCast(index + 1),622            .name = concept.name,623            .document = document,624        };625    }626    var publisher = try storage.store.beginPublish(627        allocator,628        root,629        expected_head,630        limits.storage,631    );632    defer publisher.close();633    const existing = try storage.database.read.databaseConceptDocuments(&publisher.store.database, allocator, allocator);634    defer storage.rows.freeConceptDocumentRowSlice(allocator, allocator, existing);635    const deletes = try allocator.alloc(i64, existing.len);636    defer allocator.free(deletes);637    for (existing, 0..) |document, index| deletes[index] = document.rowid;638    try storage.database.edits.apply(&publisher.store.database, .{639        .concept_document_puts = puts,640        .concept_document_deletes = deletes,641    });642    try storage.database.commitIfDirty(&publisher.store.database);643    const head = (try publisher.store.database.connection.checkout()).head;644    try storage.summary.writeForHead(allocator, root, &publisher.store.database, head);645    _ = try search.syncStoredFromDatabase(646        allocator,647        root,648        &publisher.store.database,649        expected_head,650        head,651        &.{},652        &.{},653        limits.search,654    );655    try publisher.finish();656}657658pub fn add(allocator: std.mem.Allocator, concepts: []const Concept, name: []const u8, prefix: []const u8) ![]const Concept {659    var out: std.ArrayList(Concept) = .empty;660    var found = false;661    for (concepts) |concept| {662        if (std.mem.eql(u8, concept.name, name)) {663            found = true;664            try out.append(allocator, .{665                .name = concept.name,666                .prefixes = try appendUnique(allocator, concept.prefixes, prefix),667                .sync_points = concept.sync_points,668            });669        } else {670            try out.append(allocator, concept);671        }672    }673    if (!found) {674        const prefixes = try allocator.alloc([]const u8, 1);675        prefixes[0] = try allocator.dupe(u8, prefix);676        try out.append(allocator, .{ .name = try allocator.dupe(u8, name), .prefixes = prefixes });677    }678    return try out.toOwnedSlice(allocator);679}680681pub fn create(allocator: std.mem.Allocator, concepts: []const Concept, name: []const u8, prefixes: []const []const u8, sync_points: []const []const u8) ![]const Concept {682    for (concepts) |concept| if (std.mem.eql(u8, concept.name, name)) return error.DuplicateConcept;683    var out: std.ArrayList(Concept) = .empty;684    for (concepts) |concept| try out.append(allocator, concept);685    const owned_prefixes = try allocator.alloc([]const u8, prefixes.len);686    for (prefixes, 0..) |prefix, index| owned_prefixes[index] = try allocator.dupe(u8, prefix);687    const owned_points = try allocator.alloc([]const u8, sync_points.len);688    for (sync_points, 0..) |point, index| owned_points[index] = try allocator.dupe(u8, point);689    try out.append(allocator, .{ .name = try allocator.dupe(u8, name), .prefixes = owned_prefixes, .sync_points = owned_points });690    return try out.toOwnedSlice(allocator);691}692693pub fn hasSyncPoint(concepts: []const Concept, name: []const u8, point: []const u8) bool {694    for (concepts) |concept| {695        if (!std.mem.eql(u8, concept.name, name)) continue;696        for (concept.sync_points) |existing| if (std.mem.eql(u8, existing, point)) return true;697        return false;698    }699    return false;700}701702pub fn addSyncPoint(allocator: std.mem.Allocator, concepts: []const Concept, name: []const u8, point: []const u8) ![]const Concept {703    var out: std.ArrayList(Concept) = .empty;704    var found = false;705    for (concepts) |concept| {706        if (std.mem.eql(u8, concept.name, name)) {707            found = true;708            try out.append(allocator, .{709                .name = concept.name,710                .prefixes = concept.prefixes,711                .sync_points = try appendUnique(allocator, concept.sync_points, point),712            });713        } else {714            try out.append(allocator, concept);715        }716    }717    if (!found) return error.NotFound;718    return try out.toOwnedSlice(allocator);719}720721pub fn remove(allocator: std.mem.Allocator, concepts: []const Concept, name: []const u8) ![]const Concept {722    var out: std.ArrayList(Concept) = .empty;723    var found = false;724    for (concepts) |concept| {725        if (std.mem.eql(u8, concept.name, name)) {726            found = true;727        } else {728            try out.append(allocator, concept);729        }730    }731    if (!found) return error.NotFound;732    return try out.toOwnedSlice(allocator);733}734735pub fn render(allocator: std.mem.Allocator, concept: Concept) ![]const u8 {736    var out: std.Io.Writer.Allocating = .init(allocator);737    errdefer out.deinit();738    var writer = pretty_json.Writer.init(&out.writer, .minified);739    try writer.beginObject();740    try writer.objectField("kind");741    try writer.write("concept");742    try writer.objectField("name");743    try writer.write(concept.name);744    try writer.objectField("prefixes");745    try writeStringArray(&writer, concept.prefixes);746    if (concept.sync_points.len != 0) {747        try writer.objectField("sync_points");748        try writeStringArray(&writer, concept.sync_points);749    }750    try writer.endObject();751    return try out.toOwnedSlice();752}753754fn stringField(object: std.json.ObjectMap, key: []const u8) ?[]const u8 {755    const value = object.get(key) orelse return null;756    return switch (value) {757        .string => |string| string,758        else => null,759    };760}761762fn stringArray(allocator: std.mem.Allocator, maybe: ?std.json.Value) ![]const []const u8 {763    const value = maybe orelse return &.{};764    const array = switch (value) {765        .array => |array| array,766        else => return &.{},767    };768    var out: std.ArrayList([]const u8) = .empty;769    for (array.items) |item| {770        if (item == .string) try out.append(allocator, try allocator.dupe(u8, item.string));771    }772    return try out.toOwnedSlice(allocator);773}774775test "concept declarations persist through the smg SQL store" {776    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);777    defer arena.deinit();778    const allocator = arena.allocator();779    var tmp = std.testing.tmpDir(.{});780    defer tmp.cleanup();781    const root = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);782    _ = try storage.initProject(allocator, root, testing_limits.storage);783784    const values = [_]Concept{785        .{ .name = "runtime", .prefixes = &.{ "app.runtime", "lib.runtime" }, .sync_points = &.{"app.runtime.api"} },786        .{ .name = "cli", .prefixes = &.{"app.cli"} },787    };788    const snapshot = try loadSnapshot(allocator, root, testing_limits.storage);789    try publish(allocator, root, &values, snapshot.head, testing_limits);790    const loaded = try load(allocator, root, testing_limits.storage);791    try std.testing.expectEqual(@as(usize, 2), loaded.len);792    try std.testing.expectEqualStrings("cli", loaded[0].name);793    try std.testing.expectEqualStrings("runtime", loaded[1].name);794    try std.testing.expectEqual(@as(usize, 2), loaded[1].prefixes.len);795    try std.testing.expectEqualStrings("app.runtime.api", loaded[1].sync_points[0]);796}797798test "stale concept publication preserves the winner and supports reapply" {799    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);800    defer arena.deinit();801    const allocator = arena.allocator();802    var tmp = std.testing.tmpDir(.{});803    defer tmp.cleanup();804    const root = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);805    _ = try storage.initProject(allocator, root, testing_limits.storage);806807    const source = try loadSnapshot(allocator, root, testing_limits.storage);808    const winner = try create(allocator, source.concepts, "winner", &.{"app.winner"}, &.{});809    const loser = try create(allocator, source.concepts, "loser", &.{"app.loser"}, &.{});810    try publish(allocator, root, winner, source.head, testing_limits);811    const winner_snapshot = try loadSnapshot(allocator, root, testing_limits.storage);812813    try std.testing.expectError(error.StaleSourceHead, publish(allocator, root, loser, source.head, testing_limits));814    try std.testing.expect(!try storage.sync.refreshIncomplete(allocator, root));815    const after_conflict = try loadSnapshot(allocator, root, testing_limits.storage);816    try std.testing.expect(sql.version.same(winner_snapshot.head, after_conflict.head));817    try std.testing.expectEqual(@as(usize, 1), after_conflict.concepts.len);818    try std.testing.expectEqualStrings("winner", after_conflict.concepts[0].name);819    _ = try storage.summary.load(allocator, root, testing_limits.storage);820    _ = try search.searchStored(allocator, root, "winner", null, 10, testing_limits.storage);821822    const reapplied = try create(allocator, after_conflict.concepts, "loser", &.{"app.loser"}, &.{});823    try publish(allocator, root, reapplied, after_conflict.head, testing_limits);824    const final = try load(allocator, root, testing_limits.storage);825    try std.testing.expectEqual(@as(usize, 2), final.len);826    try std.testing.expectEqualStrings("loser", final[0].name);827    try std.testing.expectEqualStrings("winner", final[1].name);828}829830fn appendUniqueList(allocator: std.mem.Allocator, list: *std.ArrayList([]const u8), value: []const u8) !void {831    for (list.items) |item| if (std.mem.eql(u8, item, value)) return;832    try list.append(allocator, value);833}834835fn appendUnique(allocator: std.mem.Allocator, values: []const []const u8, value: []const u8) ![]const []const u8 {836    for (values) |item| if (std.mem.eql(u8, item, value)) return values;837    var out: std.ArrayList([]const u8) = .empty;838    for (values) |item| try out.append(allocator, item);839    try out.append(allocator, try allocator.dupe(u8, value));840    return try out.toOwnedSlice(allocator);841}842843test "analyze concepts matches python json contract" {844    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);845    defer arena.deinit();846    const allocator = arena.allocator();847    var graph = graph_mod.init(allocator);848    try graph_mod.addNode(&graph, .{ .name = "app", .type = model.NodeType.package });849    try graph_mod.addNode(&graph, .{ .name = "app.core", .type = model.NodeType.module });850    try graph_mod.addNode(&graph, .{ .name = "app.core.Engine", .type = model.NodeType.class });851    try graph_mod.addNode(&graph, .{ .name = "app.core.Engine.run", .type = model.NodeType.method });852    try graph_mod.addNode(&graph, .{ .name = "app.utils", .type = model.NodeType.module });853    try graph_mod.addNode(&graph, .{ .name = "app.utils.helper", .type = model.NodeType.function });854    try graph_mod.addEdge(&graph, .{ .source = "app", .rel = model.RelType.contains, .target = "app.core" });855    try graph_mod.addEdge(&graph, .{ .source = "app", .rel = model.RelType.contains, .target = "app.utils" });856    try graph_mod.addEdge(&graph, .{ .source = "app.core", .rel = model.RelType.contains, .target = "app.core.Engine" });857    try graph_mod.addEdge(&graph, .{ .source = "app.core.Engine", .rel = model.RelType.contains, .target = "app.core.Engine.run" });858    try graph_mod.addEdge(&graph, .{ .source = "app.utils", .rel = model.RelType.contains, .target = "app.utils.helper" });859    try graph_mod.addEdge(&graph, .{ .source = "app.core", .rel = model.RelType.imports, .target = "app.utils" });860    try graph_mod.addEdge(&graph, .{ .source = "app.core.Engine.run", .rel = model.RelType.calls, .target = "app.utils.helper" });861    const prefix_core = [_][]const u8{"app.core"};862    const prefix_utils = [_][]const u8{"app.utils"};863    const concept_list = [_]Concept{864        .{ .name = "core", .prefixes = &prefix_core },865        .{ .name = "utils", .prefixes = &prefix_utils },866    };867    const result = try analyze(allocator, graph, &concept_list, null);868    var out: std.Io.Writer.Allocating = .init(allocator);869    var writer = pretty_json.Writer.init(&out.writer, .minified);870    try writeAnalysisJson(allocator, &writer, result);871    try out.writer.writeByte('\n');872    try std.testing.expectEqualStrings(873        \\{"declared":[{"name":"core","prefixes":["app.core"],"anchors":["app.core"],"members":3,"internal_edges":0,"cross_in":0,"cross_out":2,"sync_fan_out":1,"sync_density":1.0,"sync_asymmetry":1.0},{"name":"utils","prefixes":["app.utils"],"anchors":["app.utils"],"members":2,"internal_edges":0,"cross_in":2,"cross_out":0,"sync_fan_out":0,"sync_density":1.0,"sync_asymmetry":1.0}],"dependencies":[{"source":"core","target":"utils","edge_count":2,"rels":{"calls":1,"imports":1},"unsanctioned_count":2,"unsanctioned_rels":{"calls":1,"imports":1},"witnesses":[{"kind":"edge","edges":[{"source":"app.core","rel":"imports","target":"app.utils"}]},{"kind":"edge","edges":[{"source":"app.core.Engine.run","rel":"calls","target":"app.utils.helper"}]}],"allowed_sync":false}],"violations":[{"source":"core","target":"utils","message":"2 unsanctioned cross-concept edge(s)","rels":{"calls":1,"imports":1},"sync_candidates":{"source":["app.core","app.core.Engine.run"],"target":["app.utils","app.utils.helper"]},"sync_commands":{"source":["smg concept sync-point core app.core","smg concept sync-point core app.core.Engine.run"],"target":["smg concept sync-point utils app.utils","smg concept sync-point utils app.utils.helper"]},"witnesses":[{"kind":"edge","edges":[{"source":"app.core","rel":"imports","target":"app.utils"}]},{"kind":"edge","edges":[{"source":"app.core.Engine.run","rel":"calls","target":"app.utils.helper"}]}]}]}874        \\875    , out.written());876}877878test "concept json omits empty sync points like python schema" {879    const rendered = try render(std.testing.allocator, .{ .name = "utils", .prefixes = &.{"app.utils"} });880    defer std.testing.allocator.free(rendered);881    try std.testing.expectEqualStrings("{\"kind\":\"concept\",\"name\":\"utils\",\"prefixes\":[\"app.utils\"]}", rendered);882}883884test "concept publication sorts by name like python storage" {885    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);886    defer arena.deinit();887    const allocator = arena.allocator();888    var tmp = std.testing.tmpDir(.{});889    defer tmp.cleanup();890    const root = try tmp.dir.realPathFileAlloc(std.Options.debug_io, ".", allocator);891    _ = try storage.initProject(allocator, root, testing_limits.storage);892    const snapshot = try loadSnapshot(allocator, root, testing_limits.storage);893    const list = [_]Concept{894        .{ .name = "ui", .prefixes = &.{"app.ui"} },895        .{ .name = "data", .prefixes = &.{"app.db"} },896    };897    try publish(allocator, root, &list, snapshot.head, testing_limits);898    const loaded = try load(allocator, root, testing_limits.storage);899    try std.testing.expectEqual(@as(usize, 2), loaded.len);900    try std.testing.expectEqualStrings("data", loaded[0].name);901    try std.testing.expectEqualStrings("ui", loaded[1].name);902    try std.testing.expectEqual(@as(usize, 0), (try storage.summary.load(allocator, root, testing_limits.storage)).nodes);903    try std.testing.expectEqual(@as(i64, 0), (try search.searchStored(allocator, root, "data", null, 10, testing_limits.storage)).total);904}

1pub const concepts = @import("concepts.zig");