lib/syn/src/scan/line.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 //! Each call runs one line through the routine for its language and records the marked ranges in
2 //! storage the caller provides. A line may begin inside a comment or string that an earlier line
3 //! opened, and the rest of the line has to be marked as usual once that comment or string closes. A
4 //! line may also be too long for the storage, and what it leaves open for the next line must still
5 //! come out right.
6 //!
7 //! When the line starts inside an open comment or string, the text up to the closing delimiter
8 //! becomes one marked byte range of a line and its role (*span*), and the routine for the language
9 //! marks the rest of the line. That first span has the kind of the open comment or string. When the
10 //! line is too long (*discarded line*), the storage drops every span, but the scan still runs over
11 //! the whole line so what it leaves open stays exact. Languages share scanners by family: Lisp and
12 //! Clojure use one with this repository's own Lisp-shaped programming language (*Chiclet*), HTML,
13 //! XML, Vue and Svelte use another, and every language without a scanner of its own uses a generic
14 //! one for C-like syntax.
15 //!
16 //! - *continuation state*: what one line leaves open for the next line
17 //! - *text limit*: the longest line, in bytes, whose marks the storage keeps
18 const std = @import("std");
19 const syn = @import("../root.zig");
20
21 const diff = @import("diff.zig");
22 const generic = @import("generic.zig");
23 const json = @import("json.zig");
24 const lisp = @import("lisp.zig");
25 const markdown = @import("markdown.zig");
26 const markup = @import("markup.zig");
27 const python = @import("python.zig");
28 const shell = @import("shell.zig");
29 const structured = @import("structured.zig");
30
31 const Language = syn.Language;
32 const SpanStorage = syn.SpanStorage;
33 const Kind = syn.Kind;
34 const Mode = syn.Mode;
35 const State = syn.State;
36
37 /// Marks up one line from a fresh continuation state. A caller with one line on its own, such as a
38 /// snippet or the README example, marks it up without keeping anything between calls. The function
39 /// behaves as `highlightLine` with a new `State` that is dropped after the call, so a comment or
40 /// string the line leaves open is forgotten. The call replaces the spans the storage held for the
41 /// previous line. The storage must be activated before the call, and the call allocates nothing.
42 /// The call returns `.complete` when the spans were kept and `.discarded` when the line was longer
43 /// than the text limit.
44 pub fn highlight(
45 language: Language,
46 text: []const u8,
47 spans: *SpanStorage,
48 ) syn.SpanMaterialization {
49 var state: State = .{};
50 return highlightLine(language, text, &state, spans);
51 }
52
53 /// Marks up one line of a document and updates `state` with what the line leaves open. A caller
54 /// walking a document line by line passes the same `State` to every call, so comments and strings
55 /// that run over more than one line are marked correctly. The call replaces the storage's spans
56 /// with this line's spans, each an offset into `text`. The call keeps no spans and returns
57 /// `.discarded` when the line is longer than the text limit. That line is still scanned in full, so
58 /// `state` comes out the same as it would with room to spare. A comment or string that `state`
59 /// carries from a different language is dropped, and the line is scanned fresh. A comment or string
60 /// that `state` carries from this language runs up to its closing delimiter as one comment or
61 /// string span, and the rest of the line is scanned as usual. A line with no closing delimiter
62 /// becomes one span, and the comment or string stays open. The storage must be activated before the
63 /// call, and the call allocates nothing. The call keeps no reference to `text`.
64 pub fn highlightLine(
65 language: Language,
66 text: []const u8,
67 state: *State,
68 spans: *SpanStorage,
69 ) syn.SpanMaterialization {
70 const materialization = prepare(spans, text.len);
71 state.rebind(language);
72 if (!highlightContinuing(language, text, state, spans)) {
73 state.retain(language, highlightFlat(language, text, spans));
74 }
75 return materialization;
76 }
77
78 fn prepare(spans: *SpanStorage, text_bytes: usize) syn.SpanMaterialization {
79 spans.prepare(text_bytes) catch |err| switch (err) {
80 error.MaterializationCapacityExceeded => return .discarded,
81 };
82 return .complete;
83 }
84
85 fn highlightFlat(language: Language, text: []const u8, spans: *SpanStorage) ?Mode {
86 return switch (language) {
87 .chiclet, .lisp, .clojure => {
88 lisp.highlight(text, spans);
89 return null;
90 },
91 .json => {
92 json.highlight(text, spans);
93 return null;
94 },
95 .shell => {
96 shell.highlight(text, spans);
97 return null;
98 },
99 .diff => {
100 diff.highlight(text, spans);
101 return null;
102 },
103 .markdown => {
104 markdown.highlight(text, spans);
105 return null;
106 },
107 .python => python.highlight(text, spans),
108 .yaml => {
109 structured.highlightYaml(text, spans);
110 return null;
111 },
112 .toml => structured.highlightToml(text, spans),
113 .html, .xml, .vue, .svelte => markup.highlightMarkup(language, text, spans),
114 .css => markup.highlightCss(text, spans),
115 else => generic.highlight(language, text, spans),
116 };
117 }
118
119 fn highlightContinuing(
120 language: Language,
121 text: []const u8,
122 state: *State,
123 spans: *SpanStorage,
124 ) bool {
125 const mode = state.mode() orelse return false;
126 const format: ContinuationFormat = switch (mode) {
127 .block_comment => .{ .close = "*/", .kind = .comment },
128 .haskell_comment => .{ .close = "-}", .kind = .comment },
129 .lua_comment => .{ .close = "]]", .kind = .comment },
130 .markup_comment => .{ .close = "-->", .kind = .comment },
131 .python_triple_single,
132 .toml_triple_single,
133 => .{ .close = "'''", .kind = .string },
134 .python_triple_double,
135 .toml_triple_double,
136 => .{ .close = "\"\"\"", .kind = .string },
137 };
138 highlightDelimitedContinuation(
139 language,
140 text,
141 state,
142 spans,
143 mode,
144 format,
145 );
146 return true;
147 }
148
149 const ContinuationFormat = struct {
150 close: []const u8,
151 kind: Kind,
152 };
153
154 fn highlightDelimitedContinuation(
155 language: Language,
156 text: []const u8,
157 state: *State,
158 spans: *SpanStorage,
159 mode: Mode,
160 format: ContinuationFormat,
161 ) void {
162 const end = continuationEnd(mode, text, format.close) orelse {
163 spans.append(.{
164 .start = 0,
165 .end = text.len,
166 .kind = format.kind,
167 });
168 return;
169 };
170 spans.append(.{
171 .start = 0,
172 .end = end,
173 .kind = format.kind,
174 });
175 state.reset();
176 if (end == text.len) return;
177 appendHighlightedTail(language, text, end, state, spans);
178 }
179
180 fn continuationEnd(
181 mode: Mode,
182 text: []const u8,
183 close: []const u8,
184 ) ?usize {
185 return switch (mode) {
186 .toml_triple_single => structured.tomlTripleEnd(text, '\''),
187 .toml_triple_double => structured.tomlTripleEnd(text, '"'),
188 else => if (std.mem.indexOf(u8, text, close)) |index|
189 index + close.len
190 else
191 null,
192 };
193 }
194
195 fn appendHighlightedTail(
196 language: Language,
197 text: []const u8,
198 start: usize,
199 state: *State,
200 spans: *SpanStorage,
201 ) void {
202 const tail_text = text[start..];
203 const previous_base = spans.setBase(start);
204 defer _ = spans.setBase(previous_base);
205 state.retain(language, highlightFlat(language, tail_text, spans));
206 }