tiny.smg.scan.chiclet

1//! Structural Chiclet source scanner.23const std = @import("std");45const smg = @import("../root.zig");6const api = smg.api;7const graph_mod = smg.graph;8const lines_mod = smg.lines;9const model = smg.model;10const scan_root = smg.scan;1112const core = scan_root.core;13const source_metadata = scan_root.source_metadata;1415const SourceLines = lines_mod.SourceLines;16const Stats = core.Stats;1718const Span = struct {19    start: usize,20    end: usize,21};2223const Shape = struct {24    name_index: usize = 1,25    params_index: ?usize = null,26    body_index: ?usize = null,27    node_type: []const u8,28    test_definition: bool = false,29    method_definition: bool = false,30};3132const Declaration = struct {33    name: []const u8,34    kind: []const u8,35    node_type: []const u8,36    form: Span,37    params: ?Span,38    body_owner: Span,39    body_index: ?usize,40    test_definition: bool,41    method_definition: bool,42};4344const ScanContext = struct {45    allocator: std.mem.Allocator,46    graph: *graph_mod.Graph,47    module: []const u8,48    path: []const u8,49    source: []const u8,50    source_lines: SourceLines,51    closers: []u8,52    exports: *const std.StringHashMap(void),53    stats: *Stats,54};5556const ItemIterator = struct {57    source: []const u8,58    closers: []u8,59    index: usize,60    limit: usize,6162    fn init(source: []const u8, aggregate: Span, closers: []u8) ?ItemIterator {63        if (aggregate.end <= aggregate.start + 1) return null;64        const open = source[aggregate.start];65        if (open != '(' and open != '[' and open != '{') return null;66        return .{67            .source = source,68            .closers = closers,69            .index = aggregate.start + 1,70            .limit = aggregate.end - 1,71        };72    }7374    fn next(self: *ItemIterator) ?Span {75        skipTrivia(self.source, &self.index);76        if (self.index >= self.limit) return null;77        const start = self.index;78        const end = expressionEnd(self.source, start, self.closers) orelse {79            self.index = self.limit;80            return null;81        };82        if (end > self.limit) {83            self.index = self.limit;84            return null;85        }86        self.index = end;87        return .{ .start = start, .end = end };88    }89};9091pub fn scan(92    allocator: std.mem.Allocator,93    scratch: std.mem.Allocator,94    graph: *graph_mod.Graph,95    module: []const u8,96    path: []const u8,97    source: []const u8,98    source_lines: SourceLines,99    stats: *Stats,100    limits: smg.ScanLimits,101) !void {102    const closers = try scratch.alloc(u8, limits.chiclet_nesting_depth);103    if (try moduleDocumentationAlloc(allocator, source, closers)) |docstring| {104        const module_node = graph_mod.getNode(graph, module) orelse unreachable;105        try graph_mod.addNode(graph, .{106            .name = module_node.name,107            .type = module_node.type,108            .docstring = docstring,109        });110    }111    var exports = std.StringHashMap(void).init(scratch);112    try collectExports(source, closers, &exports);113    const context: ScanContext = .{114        .allocator = allocator,115        .graph = graph,116        .module = module,117        .path = path,118        .source = source,119        .source_lines = source_lines,120        .closers = closers,121        .exports = &exports,122        .stats = stats,123    };124    var index: usize = 0;125    while (nextTopLevel(source, &index, closers)) |form| {126        const head = listHead(source, form, closers) orelse continue;127        if (std.mem.eql(u8, head, "import")) {128            try scanImport(context, form);129        } else if (try declaration(source, form, head, closers)) |record| {130            try addDeclaration(context, record);131        }132    }133}134135fn moduleDocumentationAlloc(136    allocator: std.mem.Allocator,137    source: []const u8,138    closers: []u8,139) !?[]const u8 {140    var index: usize = 0;141    while (nextTopLevel(source, &index, closers)) |form| {142        const head = listHead(source, form, closers) orelse continue;143        if (!std.mem.eql(u8, head, "module-doc")) continue;144        const documentation = listItem(source, form, 1, closers) orelse return null;145        if (listItem(source, form, 2, closers) != null) return null;146        if (try stringAlloc(allocator, source, documentation)) |description| {147            return description;148        }149        return try mapDocumentationAlloc(allocator, source, documentation, closers);150    }151    return null;152}153154fn collectExports(source: []const u8, closers: []u8, exports: *std.StringHashMap(void)) !void {155    var index: usize = 0;156    while (nextTopLevel(source, &index, closers)) |form| {157        const head = listHead(source, form, closers) orelse continue;158        if (!std.mem.eql(u8, head, "export")) continue;159        var items = ItemIterator.init(source, form, closers) orelse continue;160        _ = items.next();161        while (items.next()) |item| try collectExportItem(source, item, closers, exports);162    }163}164165fn collectExportItem(166    source: []const u8,167    item: Span,168    closers: []u8,169    exports: *std.StringHashMap(void),170) !void {171    if (atom(source, item)) |name| {172        try exports.put(name, {});173        return;174    }175    var nested = ItemIterator.init(source, item, closers) orelse return;176    const head = atom(source, nested.next() orelse return) orelse return;177    if (!std.mem.eql(u8, head, "for-syntax")) return;178    while (nested.next()) |child| {179        if (atom(source, child)) |name| try exports.put(name, {});180    }181}182183fn scanImport(context: ScanContext, form: Span) !void {184    var items = ItemIterator.init(context.source, form, context.closers) orelse return;185    _ = items.next();186    while (items.next()) |item| {187        if (try stringAlloc(context.allocator, context.source, item)) |target| {188            try core.addImportEdge(189                context.allocator,190                context.graph,191                context.module,192                try graphImportTargetAlloc(context.allocator, target),193                &.{},194                context.stats,195            );196            continue;197        }198        var nested = ItemIterator.init(context.source, item, context.closers) orelse continue;199        _ = nested.next();200        while (nested.next()) |child| {201            const target = try stringAlloc(context.allocator, context.source, child) orelse202                continue;203            try core.addImportEdge(204                context.allocator,205                context.graph,206                context.module,207                try graphImportTargetAlloc(context.allocator, target),208                &.{},209                context.stats,210            );211        }212    }213}214215fn graphImportTargetAlloc(216    allocator: std.mem.Allocator,217    target: []const u8,218) std.mem.Allocator.Error![]const u8 {219    if (std.mem.indexOfScalar(u8, target, '/') == null) return target;220    const normalized = try allocator.dupe(u8, target);221    for (normalized) |*byte| {222        if (byte.* == '/') byte.* = '.';223    }224    return normalized;225}226227fn declaration(source: []const u8, form: Span, head: []const u8, closers: []u8) !?Declaration {228    if (valueDeclaration(source, form, head, closers)) |record| return record;229    const shape = declarationShape(head) orelse return null;230    const name_span = listItem(source, form, shape.name_index, closers) orelse return null;231    const name = declarationName(source, name_span, closers) orelse return null;232    return .{233        .name = name,234        .kind = head,235        .node_type = shape.node_type,236        .form = form,237        .params = if (shape.params_index) |index| listItem(source, form, index, closers) else null,238        .body_owner = form,239        .body_index = shape.body_index,240        .test_definition = shape.test_definition,241        .method_definition = shape.method_definition,242    };243}244245fn valueDeclaration(source: []const u8, form: Span, head: []const u8, closers: []u8) ?Declaration {246    const value_form = std.mem.eql(u8, head, "define") or247        std.mem.eql(u8, head, "define-for-syntax") or248        std.mem.eql(u8, head, "define-syntax");249    if (!value_form) return null;250    const name = atom(source, listItem(source, form, 1, closers) orelse return null) orelse return null;251    const value = listItem(source, form, 2, closers) orelse return null;252    const lambda = std.mem.eql(u8, listHead(source, value, closers) orelse "", "lambda");253    return .{254        .name = name,255        .kind = head,256        .node_type = if (lambda) model.NodeType.function else model.NodeType.constant,257        .form = form,258        .params = if (lambda) listItem(source, value, 1, closers) else null,259        .body_owner = if (lambda) value else form,260        .body_index = if (lambda) 2 else null,261        .test_definition = false,262        .method_definition = false,263    };264}265266fn declarationShape(head: []const u8) ?Shape {267    if (std.mem.eql(u8, head, "defsemantic")) return .{268        .name_index = 2,269        .params_index = 3,270        .body_index = 4,271        .node_type = model.NodeType.function,272    };273    if (std.mem.eql(u8, head, "defmethod")) return .{274        .params_index = 3,275        .body_index = 4,276        .node_type = model.NodeType.method,277        .method_definition = true,278    };279    if (std.mem.eql(u8, head, "deftest")) return .{280        .body_index = 2,281        .node_type = model.NodeType.function,282        .test_definition = true,283    };284    if (std.mem.eql(u8, head, "defprotocol")) return .{285        .body_index = 2,286        .node_type = model.NodeType.interface,287    };288    if (std.mem.eql(u8, head, "defstruct") or289        std.mem.eql(u8, head, "defrecord") or290        std.mem.eql(u8, head, "define-condition")) return .{291        .params_index = 2,292        .body_index = 3,293        .node_type = model.NodeType.class,294    };295    if (!definitionHead(head)) return null;296    return .{297        .params_index = 2,298        .body_index = 3,299        .node_type = model.NodeType.function,300        .test_definition = std.mem.eql(u8, head, "deftest-support"),301    };302}303304fn definitionHead(head: []const u8) bool {305    if (!std.mem.startsWith(u8, head, "def") or head.len <= "def".len) return false;306    const non_definitions = [_][]const u8{ "default", "defaults", "definitions" };307    for (non_definitions) |name| if (std.mem.eql(u8, head, name)) return false;308    return true;309}310311fn addDeclaration(context: ScanContext, record: Declaration) !void {312    const line = lines_mod.lineForByte(context.source_lines, record.form.start);313    const end_line = lines_mod.lineForByte(context.source_lines, record.form.end - 1);314    const qualified = if (record.method_definition)315        try std.fmt.allocPrint(316            context.allocator,317            "{s}.{s}#{d}",318            .{ context.module, record.name, line },319        )320    else321        try std.fmt.allocPrint(322            context.allocator,323            "{s}.{s}",324            .{ context.module, record.name },325        );326    const existed = graph_mod.getNode(context.graph, qualified) != null;327    const docstring = if (record.body_index) |body_index|328        try documentationAlloc(329            context.allocator,330            context.source,331            record.body_owner,332            body_index,333            context.closers,334        )335    else336        null;337    const metadata = try declarationMetadata(context, record);338    try graph_mod.addNode(context.graph, .{339        .name = qualified,340        .type = record.node_type,341        .file = context.path,342        .line = line,343        .end_line = end_line,344        .docstring = docstring,345        .metadata = metadata,346    });347    const edge_metadata = try model.sourcePair(context.allocator, "scan");348    try core.addScanEdge(context.graph, context.stats, .{349        .source = context.module,350        .rel = model.RelType.contains,351        .target = qualified,352        .metadata = edge_metadata,353    });354    if (!existed) {355        context.stats.nodes_added += 1;356        try core.countType(context.stats, context.allocator, record.node_type);357    }358}359360fn declarationMetadata(context: ScanContext, record: Declaration) ![]const model.Pair {361    const base = try source_metadata.spanMetadata(362        context.allocator,363        try model.sourcePair(context.allocator, "scan"),364        context.source,365        .{366            .start_byte = @intCast(record.form.start),367            .end_byte = @intCast(record.form.end),368        },369    );370    var updates: std.ArrayList(model.Pair) = .empty;371    const public = !record.method_definition and context.exports.contains(record.name);372    try updates.append(context.allocator, try pair(373        context.allocator,374        api.visibility_key,375        if (public) "public" else "private",376    ));377    try updates.append(context.allocator, try pair(378        context.allocator,379        "chiclet_kind",380        record.kind,381    ));382    if (try signatureAlloc(context.allocator, context.source, record)) |signature| {383        try updates.append(context.allocator, try pair(384            context.allocator,385            api.signature_key,386            signature,387        ));388    }389    if (record.test_definition) {390        var test_pair = try pair(context.allocator, api.test_key, "true");391        test_pair.json = true;392        try updates.append(context.allocator, test_pair);393    }394    return try model.mergePairs(context.allocator, base, updates.items);395}396397fn signatureAlloc(398    allocator: std.mem.Allocator,399    source: []const u8,400    record: Declaration,401) !?[]const u8 {402    const params = record.params orelse return try allocator.dupe(u8, record.name);403    const raw = std.mem.trim(u8, source[params.start..params.end], " \t\r\n");404    return try std.fmt.allocPrint(allocator, "{s} {s}", .{ record.name, raw });405}406407fn documentationAlloc(408    allocator: std.mem.Allocator,409    source: []const u8,410    owner: Span,411    body_index: usize,412    closers: []u8,413) !?[]const u8 {414    var items = ItemIterator.init(source, owner, closers) orelse return null;415    for (0..body_index) |_| _ = items.next() orelse return null;416    const first = items.next() orelse return null;417    _ = items.next() orelse return null;418    if (try stringAlloc(allocator, source, first)) |description| return description;419    return try mapDocumentationAlloc(allocator, source, first, closers);420}421422fn mapDocumentationAlloc(423    allocator: std.mem.Allocator,424    source: []const u8,425    map: Span,426    closers: []u8,427) !?[]const u8 {428    if (source[map.start] != '{') return null;429    var items = ItemIterator.init(source, map, closers) orelse return null;430    while (items.next()) |key_span| {431        const value_span = items.next() orelse return null;432        const key = atom(source, key_span) orelse continue;433        if (!documentationKey(key)) continue;434        return try stringAlloc(allocator, source, value_span);435    }436    return null;437}438439fn documentationKey(key: []const u8) bool {440    return std.mem.eql(u8, key, ":summary") or441        std.mem.eql(u8, key, ":description") or442        std.mem.eql(u8, key, ":doc");443}444445fn stringAlloc(446    allocator: std.mem.Allocator,447    source: []const u8,448    span: Span,449) !?[]const u8 {450    if (span.end <= span.start + 1 or source[span.start] != '"' or451        source[span.end - 1] != '"') return null;452    const raw = source[span.start + 1 .. span.end - 1];453    if (std.mem.indexOfScalar(u8, raw, '\\') == null) {454        return try allocator.dupe(u8, raw);455    }456    var out: std.ArrayList(u8) = .empty;457    var index: usize = 0;458    while (index < raw.len) : (index += 1) {459        if (raw[index] != '\\' or index + 1 == raw.len) {460            try out.append(allocator, raw[index]);461            continue;462        }463        index += 1;464        try out.append(allocator, switch (raw[index]) {465            'n' => '\n',466            'r' => '\r',467            't' => '\t',468            else => raw[index],469        });470    }471    return try out.toOwnedSlice(allocator);472}473474fn pair(475    allocator: std.mem.Allocator,476    key: []const u8,477    value: []const u8,478) !model.Pair {479    return .{480        .key = try allocator.dupe(u8, key),481        .value = try allocator.dupe(u8, value),482    };483}484485fn declarationName(source: []const u8, span: Span, closers: []u8) ?[]const u8 {486    if (atom(source, span)) |name| return name;487    var items = ItemIterator.init(source, span, closers) orelse return null;488    return atom(source, items.next() orelse return null);489}490491fn listHead(source: []const u8, span: Span, closers: []u8) ?[]const u8 {492    var items = ItemIterator.init(source, span, closers) orelse return null;493    return atom(source, items.next() orelse return null);494}495496fn listItem(source: []const u8, span: Span, wanted: usize, closers: []u8) ?Span {497    var items = ItemIterator.init(source, span, closers) orelse return null;498    for (0..wanted) |_| _ = items.next() orelse return null;499    return items.next();500}501502fn atom(source: []const u8, span: Span) ?[]const u8 {503    if (span.start >= span.end) return null;504    const first = source[span.start];505    if (first == '(' or first == '[' or first == '{' or first == '"' or506        first == '\'' or first == '`' or first == ',') return null;507    return source[span.start..span.end];508}509510fn nextTopLevel(source: []const u8, index: *usize, closers: []u8) ?Span {511    skipTrivia(source, index);512    if (index.* >= source.len) return null;513    const start = index.*;514    const end = expressionEnd(source, start, closers) orelse {515        index.* = source.len;516        return null;517    };518    index.* = end;519    return .{ .start = start, .end = end };520}521522fn expressionEnd(source: []const u8, raw_start: usize, closers: []u8) ?usize {523    var start = raw_start;524    skipTrivia(source, &start);525    if (start >= source.len) return null;526    const first = source[start];527    if (first == '\'' or first == '`' or first == ',') {528        var operand = start + 1;529        if (first == ',' and operand < source.len and source[operand] == '@') operand += 1;530        return expressionEnd(source, operand, closers);531    }532    if (first == '"') return stringEnd(source, start);533    if (first == '(' or first == '[' or first == '{') return aggregateEnd(source, start, closers);534    if (first == ')' or first == ']' or first == '}') return null;535    var end = start;536    while (end < source.len and !delimiter(source[end])) end += 1;537    return if (end == start) null else end;538}539540fn aggregateEnd(source: []const u8, start: usize, closers: []u8) ?usize {541    var depth: usize = 1;542    closers[0] = closing(source[start]) orelse return null;543    var index = start + 1;544    while (index < source.len) : (index += 1) {545        const byte = source[index];546        if (byte == ';') {547            while (index < source.len and source[index] != '\n') index += 1;548            if (index >= source.len) return null;549            continue;550        }551        if (byte == '"') {552            index = (stringEnd(source, index) orelse return null) - 1;553            continue;554        }555        if (closing(byte)) |closer| {556            if (depth == closers.len) return null;557            closers[depth] = closer;558            depth += 1;559            continue;560        }561        if (byte != ')' and byte != ']' and byte != '}') continue;562        if (closers[depth - 1] != byte) return null;563        depth -= 1;564        if (depth == 0) return index + 1;565    }566    return null;567}568569fn stringEnd(source: []const u8, start: usize) ?usize {570    var index = start + 1;571    while (index < source.len) : (index += 1) {572        if (source[index] == '\\') {573            index += 1;574            if (index >= source.len) return null;575            continue;576        }577        if (source[index] == '"') return index + 1;578    }579    return null;580}581582fn closing(byte: u8) ?u8 {583    return switch (byte) {584        '(' => ')',585        '[' => ']',586        '{' => '}',587        else => null,588    };589}590591fn delimiter(byte: u8) bool {592    return byte == ' ' or byte == '\t' or byte == '\r' or byte == '\n' or593        byte == '(' or byte == ')' or byte == '[' or byte == ']' or594        byte == '{' or byte == '}' or byte == '"' or byte == ';' or595        byte == '\'' or byte == '`' or byte == ',';596}597598fn skipTrivia(source: []const u8, index: *usize) void {599    while (index.* < source.len) {600        const byte = source[index.*];601        if (byte == ' ' or byte == '\t' or byte == '\r' or byte == '\n') {602            index.* += 1;603            continue;604        }605        if (byte != ';') return;606        while (index.* < source.len and source[index.*] != '\n') index.* += 1;607    }608}

1pub const chiclet = @import("chiclet.zig");