lib/syn/src/scan/test.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const syn = @import("../root.zig");
3 const subject = @import("root.zig");
4
5 const highlight = subject.highlight;
6 const highlightLine = subject.highlightLine;
7
8 const Language = syn.Language;
9 const Mode = syn.Mode;
10 const SpanStorage = syn.SpanStorage;
11 const State = syn.State;
12
13 const Admission = struct {
14 language: Language,
15 mode: Mode,
16 };
17
18 const admissions = [_]Admission{
19 .{ .language = .zig, .mode = .block_comment },
20 .{ .language = .c, .mode = .block_comment },
21 .{ .language = .cpp, .mode = .block_comment },
22 .{ .language = .csharp, .mode = .block_comment },
23 .{ .language = .java, .mode = .block_comment },
24 .{ .language = .javascript, .mode = .block_comment },
25 .{ .language = .typescript, .mode = .block_comment },
26 .{ .language = .go, .mode = .block_comment },
27 .{ .language = .rust, .mode = .block_comment },
28 .{ .language = .php, .mode = .block_comment },
29 .{ .language = .swift, .mode = .block_comment },
30 .{ .language = .kotlin, .mode = .block_comment },
31 .{ .language = .scala, .mode = .block_comment },
32 .{ .language = .dart, .mode = .block_comment },
33 .{ .language = .css, .mode = .block_comment },
34 .{ .language = .haskell, .mode = .haskell_comment },
35 .{ .language = .lua, .mode = .lua_comment },
36 .{ .language = .html, .mode = .markup_comment },
37 .{ .language = .xml, .mode = .markup_comment },
38 .{ .language = .vue, .mode = .markup_comment },
39 .{ .language = .svelte, .mode = .markup_comment },
40 .{ .language = .python, .mode = .python_triple_single },
41 .{ .language = .python, .mode = .python_triple_double },
42 .{ .language = .toml, .mode = .toml_triple_single },
43 .{ .language = .toml, .mode = .toml_triple_double },
44 };
45
46 fn activatedStorage(allocator: std.mem.Allocator, max_text_bytes: usize) !SpanStorage {
47 var storage = try SpanStorage.init(allocator, .{ .max_text_bytes = max_text_bytes });
48 storage.activate();
49 return storage;
50 }
51
52 test "continuation admission matrix is exact" {
53 for (std.enums.values(Language)) |language| {
54 for (std.enums.values(Mode)) |mode| {
55 var expected = false;
56 for (admissions) |admission| {
57 if (admission.language == language and
58 admission.mode == mode)
59 {
60 expected = true;
61 break;
62 }
63 }
64 try std.testing.expectEqual(
65 expected,
66 mode.admittedBy(language),
67 );
68 }
69 }
70 }
71
72 test "syn highlights existing agent language categories" {
73 var spans = try activatedStorage(std.testing.allocator, 128);
74 defer spans.deinit(std.testing.allocator);
75
76 _ = highlight(.zig, "const value = \"hi\" // note", &spans);
77 try std.testing.expect(spans.contains("const value = \"hi\" // note", .keyword, "const"));
78 try std.testing.expect(spans.contains("const value = \"hi\" // note", .string, "\"hi\""));
79 try std.testing.expect(spans.contains("const value = \"hi\" // note", .comment, "// note"));
80
81 _ = highlight(.chiclet, "(note \"hi\") ; comment", &spans);
82 try std.testing.expect(spans.contains("(note \"hi\") ; comment", .delimiter, "("));
83 try std.testing.expect(spans.contains("(note \"hi\") ; comment", .head, "note"));
84
85 const json_text = "{\"name\":\"tiny\",\"ok\":true,\"count\":12}";
86 _ = highlight(.json, json_text, &spans);
87 try std.testing.expect(spans.contains(json_text, .key, "\"name\""));
88 try std.testing.expect(spans.contains(json_text, .literal, "true"));
89 try std.testing.expect(spans.contains(json_text, .number, "12"));
90 }
91
92 test "syn highlights markup css and generic languages" {
93 var spans = try activatedStorage(std.testing.allocator, 128);
94 defer spans.deinit(std.testing.allocator);
95
96 const html = "<div class=\"note\">hi</div>";
97 _ = highlight(.html, html, &spans);
98 try std.testing.expect(spans.contains(html, .tag, "div"));
99 try std.testing.expect(spans.contains(html, .attribute, "class"));
100 try std.testing.expect(spans.contains(html, .string, "\"note\""));
101
102 const css = "body { color: #fff; margin: 1rem; }";
103 _ = highlight(.css, css, &spans);
104 try std.testing.expect(spans.contains(css, .selector, "body"));
105 try std.testing.expect(spans.contains(css, .property, "color"));
106 try std.testing.expect(spans.contains(css, .number, "1rem"));
107
108 const rust = "pub fn answer() -> i32 { return 42; }";
109 _ = highlight(.rust, rust, &spans);
110 try std.testing.expect(spans.contains(rust, .keyword, "pub"));
111 try std.testing.expect(spans.contains(rust, .function, "answer"));
112 try std.testing.expect(spans.contains(rust, .type, "i32"));
113
114 const lifetime = "fn borrow<'a>(value: &'a str) -> &'a str { value }";
115 _ = highlight(.rust, lifetime, &spans);
116 try std.testing.expect(spans.contains(lifetime, .variable, "'a"));
117 try std.testing.expect(!spans.contains(
118 lifetime,
119 .string,
120 "'a>(value: &'a str) -> &'a str { value }",
121 ));
122 }
123
124 test "markdown heading keeps its inline delimiters inside the heading span" {
125 var spans = try activatedStorage(std.testing.allocator, 128);
126 defer spans.deinit(std.testing.allocator);
127
128 _ = highlight(.markdown, "# a `b`", &spans);
129 try std.testing.expectEqualSlices(syn.Span, &.{
130 .{ .start = 0, .end = 1, .kind = .marker },
131 .{ .start = 2, .end = 7, .kind = .heading },
132 }, spans.items());
133 }
134
135 test "markdown list emphasis follows the bullet marker" {
136 var spans = try activatedStorage(std.testing.allocator, 128);
137 defer spans.deinit(std.testing.allocator);
138
139 _ = highlight(.markdown, "* *a*", &spans);
140 try std.testing.expectEqualSlices(syn.Span, &.{
141 .{ .start = 0, .end = 1, .kind = .marker },
142 .{ .start = 2, .end = 5, .kind = .emphasis },
143 }, spans.items());
144 }
145
146 test "syn line state carries generic block comments" {
147 var spans = try activatedStorage(std.testing.allocator, 128);
148 defer spans.deinit(std.testing.allocator);
149 var state: State = .{};
150
151 const first = "const value = 1; /* note";
152 _ = highlightLine(.zig, first, &state, &spans);
153 try std.testing.expectEqual(Mode.block_comment, state.mode().?);
154 try std.testing.expectEqual(syn.Language.zig, state.owner().?);
155 try std.testing.expect(spans.contains(first, .keyword, "const"));
156 try std.testing.expect(spans.contains(first, .comment, "/* note"));
157
158 const second = "still note */ const next = 2;";
159 _ = highlightLine(.zig, second, &state, &spans);
160 try std.testing.expect(state.mode() == null);
161 try std.testing.expect(spans.contains(second, .comment, "still note */"));
162 try std.testing.expect(spans.contains(second, .keyword, "const"));
163 }
164
165 test "syn line state carries a new block opened after a continuation closes" {
166 var spans = try activatedStorage(std.testing.allocator, 128);
167 defer spans.deinit(std.testing.allocator);
168 var state: State = .{};
169
170 _ = highlightLine(.zig, "/* open", &state, &spans);
171
172 const text = "done */ const next = 2; /* again";
173 _ = highlightLine(.zig, text, &state, &spans);
174 try std.testing.expectEqual(Mode.block_comment, state.mode().?);
175 try std.testing.expect(spans.contains(text, .comment, "done */"));
176 try std.testing.expect(spans.contains(text, .keyword, "const"));
177 try std.testing.expect(spans.contains(text, .comment, "/* again"));
178 }
179
180 test "syn line state carries python triple strings" {
181 var spans = try activatedStorage(std.testing.allocator, 128);
182 defer spans.deinit(std.testing.allocator);
183 var state: State = .{};
184
185 const first = "value = f\"\"\"hello";
186 _ = highlightLine(.python, first, &state, &spans);
187 try std.testing.expectEqual(Mode.python_triple_double, state.mode().?);
188 try std.testing.expect(spans.contains(first, .string, "f\"\"\"hello"));
189
190 const middle = "middle";
191 _ = highlightLine(.python, middle, &state, &spans);
192 try std.testing.expectEqual(Mode.python_triple_double, state.mode().?);
193 try std.testing.expect(spans.contains(middle, .string, "middle"));
194
195 const last = "done\"\"\"; return True";
196 _ = highlightLine(.python, last, &state, &spans);
197 try std.testing.expect(state.mode() == null);
198 try std.testing.expect(spans.contains(last, .string, "done\"\"\""));
199 try std.testing.expect(spans.contains(last, .keyword, "return"));
200 try std.testing.expect(spans.contains(last, .literal, "True"));
201 }
202
203 test "syn line state carries markup comments and toml strings" {
204 var spans = try activatedStorage(std.testing.allocator, 128);
205 defer spans.deinit(std.testing.allocator);
206 var state: State = .{};
207
208 const html_first = "<!-- open";
209 _ = highlightLine(.html, html_first, &state, &spans);
210 try std.testing.expectEqual(Mode.markup_comment, state.mode().?);
211 try std.testing.expect(spans.contains(html_first, .comment, "<!-- open"));
212
213 const html_second = "done --> <span>";
214 _ = highlightLine(.html, html_second, &state, &spans);
215 try std.testing.expect(state.mode() == null);
216 try std.testing.expect(spans.contains(html_second, .comment, "done -->"));
217 try std.testing.expect(spans.contains(html_second, .tag, "span"));
218
219 state.reset();
220 const toml_first = "text = '''one";
221 _ = highlightLine(.toml, toml_first, &state, &spans);
222 try std.testing.expectEqual(Mode.toml_triple_single, state.mode().?);
223 try std.testing.expect(spans.contains(toml_first, .string, "'''one"));
224
225 const toml_second = "two'''";
226 _ = highlightLine(.toml, toml_second, &state, &spans);
227 try std.testing.expect(state.mode() == null);
228 try std.testing.expect(spans.contains(toml_second, .string, "two'''"));
229 }
230
231 test "toml escaped triple delimiter retains the scanner transition" {
232 var spans = try activatedStorage(std.testing.allocator, 128);
233 defer spans.deinit(std.testing.allocator);
234 var state: State = .{};
235
236 const first = "text = \"\"\"escaped \\\"\"\"";
237 _ = highlightLine(.toml, first, &state, &spans);
238 try std.testing.expectEqual(Mode.toml_triple_double, state.mode().?);
239 try std.testing.expect(spans.contains(
240 first,
241 .string,
242 "\"\"\"escaped \\\"\"\"",
243 ));
244
245 const escaped = "still \\\"\"\"";
246 _ = highlightLine(.toml, escaped, &state, &spans);
247 try std.testing.expectEqual(Mode.toml_triple_double, state.mode().?);
248 try std.testing.expect(spans.contains(escaped, .string, escaped));
249
250 const second = "still open\"\"\"";
251 _ = highlightLine(.toml, second, &state, &spans);
252 try std.testing.expect(state.mode() == null);
253 try std.testing.expect(spans.contains(second, .string, second));
254 }
255
256 test "foreign continuation equals a fresh target-language scan" {
257 var carried_spans = try activatedStorage(std.testing.allocator, 128);
258 defer carried_spans.deinit(std.testing.allocator);
259 var fresh_spans = try activatedStorage(std.testing.allocator, 128);
260 defer fresh_spans.deinit(std.testing.allocator);
261 var carried: State = .{};
262 var fresh: State = .{};
263
264 _ = highlightLine(.zig, "/* open", &carried, &carried_spans);
265 try std.testing.expectEqual(Mode.block_comment, carried.mode().?);
266
267 const python = "return True";
268 _ = highlightLine(.python, python, &carried, &carried_spans);
269 _ = highlightLine(.python, python, &fresh, &fresh_spans);
270
271 try std.testing.expectEqual(fresh, carried);
272 try std.testing.expectEqualSlices(
273 syn.Span,
274 fresh_spans.items(),
275 carried_spans.items(),
276 );
277 try std.testing.expect(carried_spans.contains(
278 python,
279 .keyword,
280 "return",
281 ));
282 }
283
284 test "rust lifetime and block opener produce one coherent transition" {
285 var spans = try activatedStorage(std.testing.allocator, 128);
286 defer spans.deinit(std.testing.allocator);
287 var state: State = .{};
288
289 const first = "fn f<'a>(x: i32) { /* open";
290 _ = highlightLine(.rust, first, &state, &spans);
291 try std.testing.expectEqual(Mode.block_comment, state.mode().?);
292 try std.testing.expect(spans.contains(first, .variable, "'a"));
293 try std.testing.expect(spans.contains(first, .comment, "/* open"));
294
295 const second = "still open */ let y = 1;";
296 _ = highlightLine(.rust, second, &state, &spans);
297 try std.testing.expect(state.mode() == null);
298 try std.testing.expect(spans.contains(
299 second,
300 .comment,
301 "still open */",
302 ));
303 try std.testing.expect(spans.contains(second, .keyword, "let"));
304 }
305
306 test "syntax highlighting discards max plus one text bytes after exact state scanning" {
307 comptime {
308 @stardustClaim(
309 @import("alloc_phase").capacity.witness(@import("../span/root.zig").Storage, "syn_span_boundaries"),
310 null,
311 null,
312 null,
313 null,
314 null,
315 null,
316 );
317 }
318
319 var bounded = try activatedStorage(std.testing.allocator, 9);
320 defer bounded.deinit(std.testing.allocator);
321 var oracle = try activatedStorage(std.testing.allocator, 16);
322 defer oracle.deinit(std.testing.allocator);
323 var bounded_state: State = .{};
324 var oracle_state: State = .{};
325
326 const exact_limit = "const x=1";
327 try std.testing.expectEqual(@as(usize, 9), exact_limit.len);
328 try std.testing.expectEqual(
329 syn.SpanMaterialization.complete,
330 highlightLine(.zig, exact_limit, &bounded_state, &bounded),
331 );
332 _ = highlightLine(.zig, exact_limit, &oracle_state, &oracle);
333 try std.testing.expectEqualSlices(
334 syn.Span,
335 oracle.items(),
336 bounded.items(),
337 );
338
339 try std.testing.expectEqual(
340 syn.SpanMaterialization.complete,
341 highlightLine(.zig, "/*", &bounded_state, &bounded),
342 );
343 _ = highlightLine(.zig, "/*", &oracle_state, &oracle);
344
345 const discarded = "aaaaa*/ /*";
346 try std.testing.expectEqual(
347 syn.SpanMaterialization.discarded,
348 highlightLine(.zig, discarded, &bounded_state, &bounded),
349 );
350 _ = highlightLine(.zig, discarded, &oracle_state, &oracle);
351 try std.testing.expectEqual(oracle_state, bounded_state);
352 try std.testing.expectEqual(@as(usize, 0), bounded.items().len);
353 try std.testing.expectEqual(
354 @as(u64, 1),
355 bounded.status().discarded_line_count,
356 );
357
358 const admitted = "tail";
359 try std.testing.expectEqual(
360 syn.SpanMaterialization.complete,
361 highlightLine(.zig, admitted, &bounded_state, &bounded),
362 );
363 _ = highlightLine(.zig, admitted, &oracle_state, &oracle);
364 try std.testing.expectEqual(oracle_state, bounded_state);
365 try std.testing.expectEqualSlices(
366 syn.Span,
367 oracle.items(),
368 bounded.items(),
369 );
370 }
371
372 test "syntax span storage reuses one region across languages and lines" {
373 comptime {
374 @stardustClaim(
375 @import("alloc_phase").capacity.witness(@import("../span/root.zig").Storage, "syn_span_reuse"),
376 null,
377 null,
378 null,
379 null,
380 null,
381 null,
382 );
383 }
384
385 var spans = try activatedStorage(std.testing.allocator, 40);
386 defer spans.deinit(std.testing.allocator);
387 const pointer = spans.bytes.ptr;
388 var state: State = .{};
389
390 _ = highlight(.json, "{\"alpha\":1,\"beta\":2}", &spans);
391 const first_high_water = spans.status().high_water_spans;
392 _ = highlightLine(.zig, "const value = 1; /*", &state, &spans);
393 _ = highlightLine(.zig, "done */", &state, &spans);
394
395 try std.testing.expect(spans.bytes.ptr == pointer);
396 try std.testing.expect(spans.status().high_water_spans >= first_high_water);
397 try std.testing.expect(spans.status().span_count <= spans.status().max_text_bytes);
398 }
399
400 test "activated syntax highlighting performs no backing allocation" {
401 comptime {
402 @stardustClaim(
403 @import("alloc_phase").capacity.witness(@import("../span/root.zig").Storage, "syn_span_sealed_overload"),
404 null,
405 null,
406 null,
407 null,
408 null,
409 null,
410 );
411 }
412 comptime {
413 @stardustClaim(
414 @import("alloc_phase").capacity.witness(@import("../span/root.zig").Storage, "syn_span_sealed_transitive_risk"),
415 null,
416 null,
417 null,
418 null,
419 null,
420 null,
421 );
422 }
423 comptime {
424 @stardustClaim(
425 @import("alloc_phase").capacity.witness(@import("../span/root.zig").Storage, "syn_span_sealed_foreign_risk"),
426 null,
427 null,
428 null,
429 null,
430 null,
431 null,
432 );
433 }
434
435 const alloc_phase = @import("alloc_phase");
436 var phase_allocator = try alloc_phase.SealedPhaseAllocator.init(std.testing.allocator);
437 var spans = SpanStorage.init(
438 phase_allocator.initializationAllocator(),
439 .{ .max_text_bytes = 40 },
440 ) catch |err| {
441 phase_allocator.abortInitialization();
442 phase_allocator.deinit();
443 return err;
444 };
445 defer {
446 if (phase_allocator.phase() == .initialization) {
447 phase_allocator.abortInitialization();
448 }
449 if (phase_allocator.phase() == .steady) phase_allocator.beginTeardown();
450 if (spans.phase != .teardown) {
451 spans.deinit(phase_allocator.teardownAllocator());
452 }
453 phase_allocator.deinit();
454 }
455
456 phase_allocator.seal();
457 spans.activate();
458 const pointer = spans.bytes.ptr;
459 const span_capacity = spans.status().span_capacity;
460 var state: State = .{};
461 _ = highlight(.json, "{\"alpha\":1,\"beta\":2}", &spans);
462 _ = highlightLine(.zig, "const value = 1; /*", &state, &spans);
463 _ = highlightLine(.zig, "done */", &state, &spans);
464 try std.testing.expectEqual(
465 syn.SpanMaterialization.discarded,
466 highlightLine(
467 .zig,
468 "/* aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
469 &state,
470 &spans,
471 ),
472 );
473 try std.testing.expectEqual(@as(usize, 0), spans.items().len);
474 try std.testing.expect(spans.bytes.ptr == pointer);
475 try std.testing.expectEqual(
476 span_capacity,
477 spans.status().span_capacity,
478 );
479 try std.testing.expectEqual(
480 syn.SpanMaterialization.complete,
481 highlightLine(.zig, "tail", &state, &spans),
482 );
483 try std.testing.expectEqual(Mode.block_comment, state.mode().?);
484 try std.testing.expectEqual(
485 @as(usize, 0),
486 phase_allocator.violations().total(),
487 );
488 }
489
490 test {
491 std.testing.refAllDecls(subject);
492 }