tiny.syn.scan.line

1//! Each call runs one line through the routine for its language and records the marked ranges in2//! storage the caller provides. A line may begin inside a comment or string that an earlier line3//! opened, and the rest of the line has to be marked as usual once that comment or string closes. A4//! line may also be too long for the storage, and what it leaves open for the next line must still5//! come out right.6//!7//! When the line starts inside an open comment or string, the text up to the closing delimiter8//! becomes one marked byte range of a line and its role (*span*), and the routine for the language9//! marks the rest of the line. That first span has the kind of the open comment or string. When the10//! line is too long (*discarded line*), the storage drops every span, but the scan still runs over11//! the whole line so what it leaves open stays exact. Languages share scanners by family: Lisp and12//! Clojure use one with this repository's own Lisp-shaped programming language (*Chiclet*), HTML,13//! XML, Vue and Svelte use another, and every language without a scanner of its own uses a generic14//! one for C-like syntax.15//!16//! - *continuation state*: what one line leaves open for the next line17//! - *text limit*: the longest line, in bytes, whose marks the storage keeps18const std = @import("std");19const syn = @import("../root.zig");2021const diff = @import("diff.zig");22const generic = @import("generic.zig");23const json = @import("json.zig");24const lisp = @import("lisp.zig");25const markdown = @import("markdown.zig");26const markup = @import("markup.zig");27const python = @import("python.zig");28const shell = @import("shell.zig");29const structured = @import("structured.zig");3031const Language = syn.Language;32const SpanStorage = syn.SpanStorage;33const Kind = syn.Kind;34const Mode = syn.Mode;35const State = syn.State;3637/// Marks up one line from a fresh continuation state. A caller with one line on its own, such as a38/// snippet or the README example, marks it up without keeping anything between calls. The function39/// behaves as `highlightLine` with a new `State` that is dropped after the call, so a comment or40/// string the line leaves open is forgotten. The call replaces the spans the storage held for the41/// previous line. The storage must be activated before the call, and the call allocates nothing.42/// The call returns `.complete` when the spans were kept and `.discarded` when the line was longer43/// than the text limit.44pub fn highlight(45    language: Language,46    text: []const u8,47    spans: *SpanStorage,48) syn.SpanMaterialization {49    var state: State = .{};50    return highlightLine(language, text, &state, spans);51}5253/// Marks up one line of a document and updates `state` with what the line leaves open. A caller54/// walking a document line by line passes the same `State` to every call, so comments and strings55/// that run over more than one line are marked correctly. The call replaces the storage's spans56/// with this line's spans, each an offset into `text`. The call keeps no spans and returns57/// `.discarded` when the line is longer than the text limit. That line is still scanned in full, so58/// `state` comes out the same as it would with room to spare. A comment or string that `state`59/// carries from a different language is dropped, and the line is scanned fresh. A comment or string60/// that `state` carries from this language runs up to its closing delimiter as one comment or61/// string span, and the rest of the line is scanned as usual. A line with no closing delimiter62/// becomes one span, and the comment or string stays open. The storage must be activated before the63/// call, and the call allocates nothing. The call keeps no reference to `text`.64pub fn highlightLine(65    language: Language,66    text: []const u8,67    state: *State,68    spans: *SpanStorage,69) syn.SpanMaterialization {70    const materialization = prepare(spans, text.len);71    state.rebind(language);72    if (!highlightContinuing(language, text, state, spans)) {73        state.retain(language, highlightFlat(language, text, spans));74    }75    return materialization;76}7778fn prepare(spans: *SpanStorage, text_bytes: usize) syn.SpanMaterialization {79    spans.prepare(text_bytes) catch |err| switch (err) {80        error.MaterializationCapacityExceeded => return .discarded,81    };82    return .complete;83}8485fn highlightFlat(language: Language, text: []const u8, spans: *SpanStorage) ?Mode {86    return switch (language) {87        .chiclet, .lisp, .clojure => {88            lisp.highlight(text, spans);89            return null;90        },91        .json => {92            json.highlight(text, spans);93            return null;94        },95        .shell => {96            shell.highlight(text, spans);97            return null;98        },99        .diff => {100            diff.highlight(text, spans);101            return null;102        },103        .markdown => {104            markdown.highlight(text, spans);105            return null;106        },107        .python => python.highlight(text, spans),108        .yaml => {109            structured.highlightYaml(text, spans);110            return null;111        },112        .toml => structured.highlightToml(text, spans),113        .html, .xml, .vue, .svelte => markup.highlightMarkup(language, text, spans),114        .css => markup.highlightCss(text, spans),115        else => generic.highlight(language, text, spans),116    };117}118119fn highlightContinuing(120    language: Language,121    text: []const u8,122    state: *State,123    spans: *SpanStorage,124) bool {125    const mode = state.mode() orelse return false;126    const format: ContinuationFormat = switch (mode) {127        .block_comment => .{ .close = "*/", .kind = .comment },128        .haskell_comment => .{ .close = "-}", .kind = .comment },129        .lua_comment => .{ .close = "]]", .kind = .comment },130        .markup_comment => .{ .close = "-->", .kind = .comment },131        .python_triple_single,132        .toml_triple_single,133        => .{ .close = "'''", .kind = .string },134        .python_triple_double,135        .toml_triple_double,136        => .{ .close = "\"\"\"", .kind = .string },137    };138    highlightDelimitedContinuation(139        language,140        text,141        state,142        spans,143        mode,144        format,145    );146    return true;147}148149const ContinuationFormat = struct {150    close: []const u8,151    kind: Kind,152};153154fn highlightDelimitedContinuation(155    language: Language,156    text: []const u8,157    state: *State,158    spans: *SpanStorage,159    mode: Mode,160    format: ContinuationFormat,161) void {162    const end = continuationEnd(mode, text, format.close) orelse {163        spans.append(.{164            .start = 0,165            .end = text.len,166            .kind = format.kind,167        });168        return;169    };170    spans.append(.{171        .start = 0,172        .end = end,173        .kind = format.kind,174    });175    state.reset();176    if (end == text.len) return;177    appendHighlightedTail(language, text, end, state, spans);178}179180fn continuationEnd(181    mode: Mode,182    text: []const u8,183    close: []const u8,184) ?usize {185    return switch (mode) {186        .toml_triple_single => structured.tomlTripleEnd(text, '\''),187        .toml_triple_double => structured.tomlTripleEnd(text, '"'),188        else => if (std.mem.indexOf(u8, text, close)) |index|189            index + close.len190        else191            null,192    };193}194195fn appendHighlightedTail(196    language: Language,197    text: []const u8,198    start: usize,199    state: *State,200    spans: *SpanStorage,201) void {202    const tail_text = text[start..];203    const previous_base = spans.setBase(start);204    defer _ = spans.setBase(previous_base);205    state.retain(language, highlightFlat(language, tail_text, spans));206}

1pub const line = @import("line.zig");