lib/zen/src/syntax.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const html = @import("html.zig");
3
4 const Allocator = std.mem.Allocator;
5
6 const Language = enum {
7 plain,
8 zig,
9 c,
10 };
11
12 pub const Highlighter = struct {
13 language: Language,
14 enabled: bool,
15 block_comment: bool = false,
16
17 pub fn init(language: []const u8, enabled: bool) Highlighter {
18 return .{
19 .language = detectLanguage(language),
20 .enabled = enabled,
21 };
22 }
23
24 pub fn appendLine(self: *Highlighter, out: *std.ArrayList(u8), allocator: Allocator, line: []const u8) Allocator.Error!void {
25 if (!self.enabled) {
26 try html.appendEscaped(out, allocator, line);
27 return;
28 }
29 switch (self.language) {
30 .plain => try html.appendEscaped(out, allocator, line),
31 .zig => try self.appendZig(out, allocator, line),
32 .c => try self.appendC(out, allocator, line),
33 }
34 }
35
36 fn appendZig(self: *Highlighter, out: *std.ArrayList(u8), allocator: Allocator, line: []const u8) Allocator.Error!void {
37 _ = self;
38 var index: usize = 0;
39 while (index < line.len) {
40 if (index + 1 < line.len and line[index] == '/' and line[index + 1] == '/') {
41 try appendToken(out, allocator, "zen-code-comment", line[index..]);
42 return;
43 }
44 if (index + 1 < line.len and line[index] == '\\' and line[index + 1] == '\\') {
45 try appendToken(out, allocator, "zen-code-string", line[index..]);
46 return;
47 }
48 if (line[index] == '"' or line[index] == '\'') {
49 index = try appendQuoted(out, allocator, line, index, line[index]);
50 continue;
51 }
52 if (line[index] == '@' and index + 1 < line.len and isIdentifierStart(line[index + 1])) {
53 const end = scanIdentifier(line, index + 1);
54 try appendToken(out, allocator, "zen-code-builtin", line[index..end]);
55 index = end;
56 continue;
57 }
58 if (isDigit(line[index])) {
59 const end = scanNumber(line, index);
60 try appendToken(out, allocator, "zen-code-number", line[index..end]);
61 index = end;
62 continue;
63 }
64 if (isIdentifierStart(line[index])) {
65 const end = scanIdentifier(line, index);
66 const token = line[index..end];
67 if (isZigKeyword(token)) {
68 try appendToken(out, allocator, "zen-code-keyword", token);
69 } else if (isZigType(token)) {
70 try appendToken(out, allocator, "zen-code-type", token);
71 } else if (isZigConstant(token)) {
72 try appendToken(out, allocator, "zen-code-constant", token);
73 } else {
74 try html.appendEscaped(out, allocator, token);
75 }
76 index = end;
77 continue;
78 }
79 try html.appendEscapedByte(out, allocator, line[index]);
80 index += 1;
81 }
82 }
83
84 fn appendC(self: *Highlighter, out: *std.ArrayList(u8), allocator: Allocator, line: []const u8) Allocator.Error!void {
85 var index: usize = 0;
86 const directive = firstNonSpace(line);
87 while (index < line.len) {
88 if (self.block_comment) {
89 if (std.mem.indexOfPos(u8, line, index, "*/")) |close| {
90 try appendToken(out, allocator, "zen-code-comment", line[index .. close + 2]);
91 self.block_comment = false;
92 index = close + 2;
93 continue;
94 }
95 try appendToken(out, allocator, "zen-code-comment", line[index..]);
96 return;
97 }
98 if (index == directive and line[index] == '#') {
99 const end = scanPreprocessor(line, index);
100 try appendToken(out, allocator, "zen-code-builtin", line[index..end]);
101 index = end;
102 continue;
103 }
104 if (index + 1 < line.len and line[index] == '/' and line[index + 1] == '/') {
105 try appendToken(out, allocator, "zen-code-comment", line[index..]);
106 return;
107 }
108 if (index + 1 < line.len and line[index] == '/' and line[index + 1] == '*') {
109 if (std.mem.indexOfPos(u8, line, index + 2, "*/")) |close| {
110 try appendToken(out, allocator, "zen-code-comment", line[index .. close + 2]);
111 index = close + 2;
112 continue;
113 }
114 try appendToken(out, allocator, "zen-code-comment", line[index..]);
115 self.block_comment = true;
116 return;
117 }
118 if (line[index] == '"' or line[index] == '\'') {
119 index = try appendQuoted(out, allocator, line, index, line[index]);
120 continue;
121 }
122 if (isDigit(line[index])) {
123 const end = scanNumber(line, index);
124 try appendToken(out, allocator, "zen-code-number", line[index..end]);
125 index = end;
126 continue;
127 }
128 if (isIdentifierStart(line[index])) {
129 const end = scanIdentifier(line, index);
130 const token = line[index..end];
131 if (isCKeyword(token)) {
132 try appendToken(out, allocator, "zen-code-keyword", token);
133 } else if (isCType(token)) {
134 try appendToken(out, allocator, "zen-code-type", token);
135 } else if (isCConstant(token)) {
136 try appendToken(out, allocator, "zen-code-constant", token);
137 } else {
138 try html.appendEscaped(out, allocator, token);
139 }
140 index = end;
141 continue;
142 }
143 try html.appendEscapedByte(out, allocator, line[index]);
144 index += 1;
145 }
146 }
147 };
148
149 fn detectLanguage(language: []const u8) Language {
150 if (asciiEqual(language, "zig")) return .zig;
151 if (asciiEqual(language, "c") or
152 asciiEqual(language, "h") or
153 asciiEqual(language, "objc") or
154 asciiEqual(language, "objective-c")) return .c;
155 return .plain;
156 }
157
158 fn appendQuoted(
159 out: *std.ArrayList(u8),
160 allocator: Allocator,
161 line: []const u8,
162 start: usize,
163 quote: u8,
164 ) Allocator.Error!usize {
165 var end = start + 1;
166 while (end < line.len) {
167 if (line[end] == '\\' and end + 1 < line.len) {
168 end += 2;
169 continue;
170 }
171 end += 1;
172 if (line[end - 1] == quote) break;
173 }
174 try appendToken(out, allocator, "zen-code-string", line[start..end]);
175 return end;
176 }
177
178 fn appendToken(out: *std.ArrayList(u8), allocator: Allocator, class: []const u8, token: []const u8) Allocator.Error!void {
179 try out.appendSlice(allocator, "<span class=\"");
180 try out.appendSlice(allocator, class);
181 try out.appendSlice(allocator, "\">");
182 try html.appendEscaped(out, allocator, token);
183 try out.appendSlice(allocator, "</span>");
184 }
185
186 fn firstNonSpace(line: []const u8) usize {
187 var index: usize = 0;
188 while (index < line.len and (line[index] == ' ' or line[index] == '\t')) : (index += 1) {}
189 return index;
190 }
191
192 fn scanPreprocessor(line: []const u8, start: usize) usize {
193 var index = start + 1;
194 while (index < line.len and isIdentifierContinue(line[index])) : (index += 1) {}
195 return index;
196 }
197
198 fn scanIdentifier(line: []const u8, start: usize) usize {
199 var index = start + 1;
200 while (index < line.len and isIdentifierContinue(line[index])) : (index += 1) {}
201 return index;
202 }
203
204 fn scanNumber(line: []const u8, start: usize) usize {
205 var index = start;
206 if (index + 1 < line.len and line[index] == '0' and (line[index + 1] == 'x' or line[index + 1] == 'X')) {
207 index += 2;
208 while (index < line.len and (isHex(line[index]) or line[index] == '_')) : (index += 1) {}
209 return index;
210 }
211 while (index < line.len and (isDigit(line[index]) or line[index] == '_')) : (index += 1) {}
212 if (index + 1 < line.len and line[index] == '.' and isDigit(line[index + 1])) {
213 index += 1;
214 while (index < line.len and (isDigit(line[index]) or line[index] == '_')) : (index += 1) {}
215 }
216 if (index + 1 < line.len and (line[index] == 'e' or line[index] == 'E')) {
217 var exponent = index + 1;
218 if (exponent < line.len and (line[exponent] == '+' or line[exponent] == '-')) exponent += 1;
219 if (exponent < line.len and isDigit(line[exponent])) {
220 index = exponent + 1;
221 while (index < line.len and (isDigit(line[index]) or line[index] == '_')) : (index += 1) {}
222 }
223 }
224 return index;
225 }
226
227 fn isIdentifierStart(byte: u8) bool {
228 return isAlpha(byte) or byte == '_';
229 }
230
231 fn isIdentifierContinue(byte: u8) bool {
232 return isIdentifierStart(byte) or isDigit(byte);
233 }
234
235 fn isAlpha(byte: u8) bool {
236 return (byte >= 'a' and byte <= 'z') or (byte >= 'A' and byte <= 'Z');
237 }
238
239 fn isDigit(byte: u8) bool {
240 return byte >= '0' and byte <= '9';
241 }
242
243 fn isHex(byte: u8) bool {
244 return isDigit(byte) or
245 (byte >= 'a' and byte <= 'f') or
246 (byte >= 'A' and byte <= 'F');
247 }
248
249 fn asciiEqual(lhs: []const u8, rhs: []const u8) bool {
250 if (lhs.len != rhs.len) return false;
251 for (lhs, rhs) |left, right| {
252 if (std.ascii.toLower(left) != std.ascii.toLower(right)) return false;
253 }
254 return true;
255 }
256
257 fn inSet(token: []const u8, comptime values: []const []const u8) bool {
258 inline for (values) |value| {
259 if (std.mem.eql(u8, token, value)) return true;
260 }
261 return false;
262 }
263
264 fn isZigKeyword(token: []const u8) bool {
265 return inSet(token, &.{
266 "addrspace", "align", "allowzero", "and", "anyframe", "anytype",
267 "asm", "async", "await", "break", "callconv", "catch",
268 "comptime", "const", "continue", "defer", "else", "enum",
269 "errdefer", "error", "export", "extern", "fn", "for",
270 "if", "inline", "noalias", "nosuspend", "opaque", "or",
271 "orelse", "packed", "pub", "resume", "return", "struct",
272 "suspend", "switch", "test", "threadlocal", "try", "union",
273 "unreachable", "usingnamespace", "var", "volatile", "while",
274 });
275 }
276
277 fn isZigType(token: []const u8) bool {
278 return inSet(token, &.{
279 "bool", "c_char", "c_int", "c_long", "c_longdouble", "c_longlong",
280 "c_short", "c_uint", "c_ulong", "c_ulonglong", "c_ushort", "comptime_float",
281 "comptime_int", "f128", "f16", "f32", "f64", "f80",
282 "i128", "i16", "i32", "i64", "i8", "isize",
283 "noreturn", "type", "u128", "u16", "u32", "u64",
284 "u8", "usize", "void",
285 });
286 }
287
288 fn isZigConstant(token: []const u8) bool {
289 return inSet(token, &.{ "false", "null", "true", "undefined" });
290 }
291
292 fn isCKeyword(token: []const u8) bool {
293 return inSet(token, &.{
294 "auto", "break", "case", "const", "continue", "default", "do", "else",
295 "enum", "extern", "for", "goto", "if", "inline", "register", "restrict",
296 "return", "sizeof", "static", "struct", "switch", "typedef", "union", "volatile",
297 "while",
298 });
299 }
300
301 fn isCType(token: []const u8) bool {
302 return inSet(token, &.{
303 "bool", "char", "double", "float", "int", "int16_t", "int32_t",
304 "int64_t", "int8_t", "long", "size_t", "ssize_t", "uint16_t", "uint32_t",
305 "uint64_t", "uint8_t", "void",
306 });
307 }
308
309 fn isCConstant(token: []const u8) bool {
310 if (std.mem.eql(u8, token, "NULL")) return true;
311 var has_marker = false;
312 for (token) |byte| {
313 if (byte >= 'a' and byte <= 'z') return false;
314 if (byte == '_' or isDigit(byte)) has_marker = true;
315 }
316 return has_marker and token.len > 1;
317 }
318
319 test "syntax highlights zig and c tokens" {
320 var out: std.ArrayList(u8) = .empty;
321 defer out.deinit(std.testing.allocator);
322 var zig = Highlighter.init("zig", true);
323 try zig.appendLine(&out, std.testing.allocator, "const x: u8 = @as(u8, 1);");
324 try std.testing.expect(std.mem.indexOf(u8, out.items, "<span class=\"zen-code-keyword\">const</span>") != null);
325 try std.testing.expect(std.mem.indexOf(u8, out.items, "<span class=\"zen-code-builtin\">@as</span>") != null);
326 out.clearRetainingCapacity();
327 var c = Highlighter.init("c", true);
328 try c.appendLine(&out, std.testing.allocator, "ACCY_OP_SQRT /* ok */");
329 try std.testing.expect(std.mem.indexOf(u8, out.items, "<span class=\"zen-code-constant\">ACCY_OP_SQRT</span>") != null);
330 try std.testing.expect(std.mem.indexOf(u8, out.items, "<span class=\"zen-code-comment\">/* ok */</span>") != null);
331 }