lib/chant/src/token.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2
3 pub const Kind = enum {
4 identifier,
5 integer,
6 floating,
7 character,
8 string,
9
10 kw_auto,
11 kw_break,
12 kw_case,
13 kw_char,
14 kw_const,
15 kw_continue,
16 kw_default,
17 kw_do,
18 kw_double,
19 kw_decimal32,
20 kw_decimal64,
21 kw_decimal128,
22 kw_else,
23 kw_enum,
24 kw_extern,
25 kw_float,
26 kw_for,
27 kw_goto,
28 kw_if,
29 kw_inline,
30 kw_int,
31 kw_long,
32 kw_register,
33 kw_restrict,
34 kw_return,
35 kw_short,
36 kw_signed,
37 kw_sizeof,
38 kw_static,
39 kw_struct,
40 kw_switch,
41 kw_typedef,
42 kw_union,
43 kw_unsigned,
44 kw_bitint,
45 kw_void,
46 kw_volatile,
47 kw_while,
48 kw_bool,
49 kw_true,
50 kw_false,
51 kw_nullptr,
52 kw_alignas,
53 kw_alignof,
54 kw_constexpr,
55 kw_static_assert,
56 kw_thread_local,
57 kw_typeof,
58 kw_typeof_unqual,
59
60 lparen,
61 rparen,
62 lbrace,
63 rbrace,
64 lbracket,
65 rbracket,
66 semicolon,
67 comma,
68 colon,
69 question,
70 plus,
71 minus,
72 star,
73 slash,
74 percent,
75 amp,
76 pipe,
77 caret,
78 tilde,
79 bang,
80 lt,
81 gt,
82 le,
83 ge,
84 eq,
85 ne,
86 amp_amp,
87 pipe_pipe,
88 shl,
89 shr,
90 assign,
91 plus_assign,
92 minus_assign,
93 star_assign,
94 slash_assign,
95 percent_assign,
96 amp_assign,
97 pipe_assign,
98 caret_assign,
99 shl_assign,
100 shr_assign,
101 plus_plus,
102 minus_minus,
103 arrow,
104 dot,
105 ellipsis,
106 eof,
107 };
108
109 pub const Token = struct {
110 kind: Kind,
111 text: []const u8,
112 file: []const u8,
113 line: u32,
114 column: u32,
115 };
116
117 const Keyword = struct {
118 text: []const u8,
119 kind: Kind,
120 };
121
122 const keywords = [_]Keyword{
123 .{ .text = "auto", .kind = .kw_auto },
124 .{ .text = "break", .kind = .kw_break },
125 .{ .text = "case", .kind = .kw_case },
126 .{ .text = "char", .kind = .kw_char },
127 .{ .text = "const", .kind = .kw_const },
128 .{ .text = "continue", .kind = .kw_continue },
129 .{ .text = "default", .kind = .kw_default },
130 .{ .text = "do", .kind = .kw_do },
131 .{ .text = "double", .kind = .kw_double },
132 .{ .text = "_Decimal32", .kind = .kw_decimal32 },
133 .{ .text = "_Decimal64", .kind = .kw_decimal64 },
134 .{ .text = "_Decimal128", .kind = .kw_decimal128 },
135 .{ .text = "else", .kind = .kw_else },
136 .{ .text = "enum", .kind = .kw_enum },
137 .{ .text = "extern", .kind = .kw_extern },
138 .{ .text = "float", .kind = .kw_float },
139 .{ .text = "for", .kind = .kw_for },
140 .{ .text = "goto", .kind = .kw_goto },
141 .{ .text = "if", .kind = .kw_if },
142 .{ .text = "inline", .kind = .kw_inline },
143 .{ .text = "int", .kind = .kw_int },
144 .{ .text = "long", .kind = .kw_long },
145 .{ .text = "register", .kind = .kw_register },
146 .{ .text = "restrict", .kind = .kw_restrict },
147 .{ .text = "return", .kind = .kw_return },
148 .{ .text = "short", .kind = .kw_short },
149 .{ .text = "signed", .kind = .kw_signed },
150 .{ .text = "sizeof", .kind = .kw_sizeof },
151 .{ .text = "static", .kind = .kw_static },
152 .{ .text = "struct", .kind = .kw_struct },
153 .{ .text = "switch", .kind = .kw_switch },
154 .{ .text = "typedef", .kind = .kw_typedef },
155 .{ .text = "union", .kind = .kw_union },
156 .{ .text = "unsigned", .kind = .kw_unsigned },
157 .{ .text = "_BitInt", .kind = .kw_bitint },
158 .{ .text = "void", .kind = .kw_void },
159 .{ .text = "volatile", .kind = .kw_volatile },
160 .{ .text = "while", .kind = .kw_while },
161 .{ .text = "bool", .kind = .kw_bool },
162 .{ .text = "_Bool", .kind = .kw_bool },
163 .{ .text = "true", .kind = .kw_true },
164 .{ .text = "false", .kind = .kw_false },
165 .{ .text = "nullptr", .kind = .kw_nullptr },
166 .{ .text = "alignas", .kind = .kw_alignas },
167 .{ .text = "_Alignas", .kind = .kw_alignas },
168 .{ .text = "alignof", .kind = .kw_alignof },
169 .{ .text = "_Alignof", .kind = .kw_alignof },
170 .{ .text = "constexpr", .kind = .kw_constexpr },
171 .{ .text = "static_assert", .kind = .kw_static_assert },
172 .{ .text = "_Static_assert", .kind = .kw_static_assert },
173 .{ .text = "thread_local", .kind = .kw_thread_local },
174 .{ .text = "_Thread_local", .kind = .kw_thread_local },
175 .{ .text = "typeof", .kind = .kw_typeof },
176 .{ .text = "__typeof__", .kind = .kw_typeof },
177 .{ .text = "typeof_unqual", .kind = .kw_typeof_unqual },
178 .{ .text = "__typeof_unqual__", .kind = .kw_typeof_unqual },
179 .{ .text = "__restrict", .kind = .kw_restrict },
180 .{ .text = "__restrict__", .kind = .kw_restrict },
181 .{ .text = "__inline", .kind = .kw_inline },
182 .{ .text = "__inline__", .kind = .kw_inline },
183 };
184
185 pub fn keywordKind(text: []const u8) ?Kind {
186 for (keywords) |keyword| {
187 if (std.mem.eql(u8, keyword.text, text)) return keyword.kind;
188 }
189 return null;
190 }
191
192 pub const IntegerValue = struct {
193 value: u64,
194 is_unsigned: bool,
195 is_long: bool,
196 bit_width: ?u16 = null,
197 };
198
199 pub fn decodeInteger(text: []const u8) ?IntegerValue {
200 var digits = text;
201 var is_unsigned = false;
202 var is_long = false;
203 var is_bit_precise = false;
204 while (digits.len > 0) {
205 const last = digits[digits.len - 1];
206 if (endsWithBitPreciseSuffix(digits)) {
207 if (is_bit_precise) return null;
208 is_bit_precise = true;
209 digits = digits[0 .. digits.len - 2];
210 } else if (last == 'u' or last == 'U') {
211 if (is_unsigned) return null;
212 is_unsigned = true;
213 digits = digits[0 .. digits.len - 1];
214 } else if (last == 'l' or last == 'L') {
215 if (is_long and digits.len >= 2 and (digits[digits.len - 2] == 'l' or digits[digits.len - 2] == 'L')) return null;
216 is_long = true;
217 digits = digits[0 .. digits.len - 1];
218 } else {
219 break;
220 }
221 }
222 if (is_bit_precise and is_long) return null;
223 if (digits.len == 0) return null;
224
225 var base: u8 = 10;
226 if (digits.len >= 2 and digits[0] == '0' and (digits[1] == 'x' or digits[1] == 'X')) {
227 base = 16;
228 digits = digits[2..];
229 } else if (digits.len >= 2 and digits[0] == '0' and (digits[1] == 'b' or digits[1] == 'B')) {
230 base = 2;
231 digits = digits[2..];
232 } else if (digits.len >= 2 and digits[0] == '0') {
233 base = 8;
234 digits = digits[1..];
235 }
236 if (digits.len == 0) return null;
237
238 const value = parseSeparatedUnsigned(digits, base) orelse return null;
239 const bit_width = if (is_bit_precise) bitPreciseWidth(value, is_unsigned) else null;
240 return .{ .value = value, .is_unsigned = is_unsigned, .is_long = is_long, .bit_width = bit_width };
241 }
242
243 fn endsWithBitPreciseSuffix(text: []const u8) bool {
244 if (text.len < 2) return false;
245 const suffix = text[text.len - 2 ..];
246 return std.mem.eql(u8, suffix, "wb") or std.mem.eql(u8, suffix, "WB");
247 }
248
249 fn bitPreciseWidth(value: u64, is_unsigned: bool) u16 {
250 const bits: u16 = if (value == 0) 0 else @intCast(64 - @clz(value));
251 if (is_unsigned) return @max(@as(u16, 1), bits);
252 return @max(@as(u16, 2), bits + 1);
253 }
254
255 fn parseSeparatedUnsigned(digits: []const u8, base: u8) ?u64 {
256 var value: u64 = 0;
257 var saw_digit = false;
258 var previous_digit = false;
259 for (digits, 0..) |byte, index| {
260 if (byte == '\'') {
261 if (!previous_digit or index + 1 >= digits.len or digitValue(digits[index + 1], base) == null) return null;
262 previous_digit = false;
263 continue;
264 }
265 const digit = digitValue(byte, base) orelse return null;
266 value = std.math.mul(u64, value, base) catch return null;
267 value = std.math.add(u64, value, digit) catch return null;
268 saw_digit = true;
269 previous_digit = true;
270 }
271 return if (saw_digit and previous_digit) value else null;
272 }
273
274 fn digitValue(byte: u8, base: u8) ?u64 {
275 const digit: u8 = if (byte >= '0' and byte <= '9')
276 byte - '0'
277 else if (byte >= 'a' and byte <= 'f')
278 byte - 'a' + 10
279 else if (byte >= 'A' and byte <= 'F')
280 byte - 'A' + 10
281 else
282 return null;
283 if (digit >= base) return null;
284 return digit;
285 }
286
287 pub const FloatKind = enum {
288 float,
289 double,
290 decimal32,
291 decimal64,
292 decimal128,
293 };
294
295 pub const FloatValue = struct {
296 value: f64,
297 kind: FloatKind,
298 };
299
300 pub fn decodeFloat(allocator: std.mem.Allocator, text: []const u8) ?FloatValue {
301 var digits = text;
302 var kind: FloatKind = .double;
303 if (digits.len > 0) {
304 const last = digits[digits.len - 1];
305 if (decimalFloatSuffixKind(digits)) |decimal_kind| {
306 if (isHexFloating(digits)) return null;
307 kind = decimal_kind;
308 digits = digits[0 .. digits.len - 2];
309 } else if (last == 'f' or last == 'F') {
310 kind = .float;
311 digits = digits[0 .. digits.len - 1];
312 } else if (last == 'l' or last == 'L') {
313 digits = digits[0 .. digits.len - 1];
314 }
315 }
316 if (digits.len == 0) return null;
317 const normalized = stripFloatSeparators(allocator, digits) orelse return null;
318 defer if (normalized.ptr != digits.ptr) allocator.free(normalized);
319 const value = std.fmt.parseFloat(f64, normalized) catch return null;
320 return .{ .value = value, .kind = kind };
321 }
322
323 fn decimalFloatSuffixKind(text: []const u8) ?FloatKind {
324 if (text.len < 2) return null;
325 const suffix = text[text.len - 2 ..];
326 if (std.mem.eql(u8, suffix, "df") or std.mem.eql(u8, suffix, "DF")) return .decimal32;
327 if (std.mem.eql(u8, suffix, "dd") or std.mem.eql(u8, suffix, "DD")) return .decimal64;
328 if (std.mem.eql(u8, suffix, "dl") or std.mem.eql(u8, suffix, "DL")) return .decimal128;
329 return null;
330 }
331
332 fn isHexFloating(text: []const u8) bool {
333 return text.len >= 2 and text[0] == '0' and (text[1] == 'x' or text[1] == 'X');
334 }
335
336 fn stripFloatSeparators(allocator: std.mem.Allocator, text: []const u8) ?[]const u8 {
337 if (std.mem.indexOfScalar(u8, text, '\'') == null) return text;
338 var out = std.ArrayListUnmanaged(u8).empty;
339 defer out.deinit(allocator);
340 var previous_digit = false;
341 for (text, 0..) |byte, index| {
342 if (byte == '\'') {
343 if (!previous_digit or index + 1 >= text.len or !std.ascii.isDigit(text[index + 1])) return null;
344 previous_digit = false;
345 continue;
346 }
347 out.append(allocator, byte) catch return null;
348 previous_digit = std.ascii.isDigit(byte);
349 }
350 if (!previous_digit) return null;
351 return out.toOwnedSlice(allocator) catch null;
352 }
353
354 pub fn decodeCharacter(text: []const u8) ?u8 {
355 const literal = if (std.mem.startsWith(u8, text, "u8")) text[2..] else text;
356 if (literal.len < 3 or literal[0] != '\'' or literal[literal.len - 1] != '\'') return null;
357 const body = literal[1 .. literal.len - 1];
358 if (body.len == 1) return body[0];
359 if (body.len == 2 and body[0] == '\\') {
360 return switch (body[1]) {
361 'n' => '\n',
362 't' => '\t',
363 'r' => '\r',
364 '0' => 0,
365 '\\' => '\\',
366 '\'' => '\'',
367 '"' => '"',
368 else => null,
369 };
370 }
371 return null;
372 }
373
374 pub fn decodeString(allocator: std.mem.Allocator, text: []const u8) ?[]const u8 {
375 const literal = if (std.mem.startsWith(u8, text, "u8")) text[2..] else text;
376 if (literal.len < 2 or literal[0] != '"' or literal[literal.len - 1] != '"') return null;
377 const body = literal[1 .. literal.len - 1];
378 var out = std.ArrayListUnmanaged(u8).empty;
379 var index: usize = 0;
380 while (index < body.len) {
381 const byte = body[index];
382 if (byte != '\\') {
383 out.append(allocator, byte) catch return null;
384 index += 1;
385 continue;
386 }
387 if (index + 1 >= body.len) return null;
388 const decoded: u8 = switch (body[index + 1]) {
389 'n' => '\n',
390 't' => '\t',
391 'r' => '\r',
392 '0' => 0,
393 '\\' => '\\',
394 '\'' => '\'',
395 '"' => '"',
396 else => return null,
397 };
398 out.append(allocator, decoded) catch return null;
399 index += 2;
400 }
401 return out.toOwnedSlice(allocator) catch null;
402 }
403
404 test "keywords resolve and identifiers do not" {
405 try std.testing.expectEqual(Kind.kw_double, keywordKind("double").?);
406 try std.testing.expectEqual(Kind.kw_decimal32, keywordKind("_Decimal32").?);
407 try std.testing.expectEqual(Kind.kw_decimal64, keywordKind("_Decimal64").?);
408 try std.testing.expectEqual(Kind.kw_decimal128, keywordKind("_Decimal128").?);
409 try std.testing.expectEqual(Kind.kw_for, keywordKind("for").?);
410 try std.testing.expectEqual(Kind.kw_bool, keywordKind("bool").?);
411 try std.testing.expectEqual(Kind.kw_true, keywordKind("true").?);
412 try std.testing.expectEqual(Kind.kw_false, keywordKind("false").?);
413 try std.testing.expectEqual(Kind.kw_nullptr, keywordKind("nullptr").?);
414 try std.testing.expectEqual(Kind.kw_bitint, keywordKind("_BitInt").?);
415 try std.testing.expectEqual(Kind.kw_alignas, keywordKind("alignas").?);
416 try std.testing.expectEqual(Kind.kw_alignof, keywordKind("alignof").?);
417 try std.testing.expectEqual(Kind.kw_constexpr, keywordKind("constexpr").?);
418 try std.testing.expectEqual(Kind.kw_static_assert, keywordKind("static_assert").?);
419 try std.testing.expectEqual(Kind.kw_thread_local, keywordKind("thread_local").?);
420 try std.testing.expectEqual(Kind.kw_typeof, keywordKind("typeof").?);
421 try std.testing.expectEqual(Kind.kw_typeof_unqual, keywordKind("typeof_unqual").?);
422 try std.testing.expectEqual(Kind.kw_restrict, keywordKind("__restrict").?);
423 try std.testing.expect(keywordKind("kernel_gemm") == null);
424 }
425
426 test "integer constants decode bases and suffixes" {
427 try std.testing.expectEqual(@as(u64, 42), decodeInteger("42").?.value);
428 try std.testing.expectEqual(@as(u64, 255), decodeInteger("0xFF").?.value);
429 try std.testing.expectEqual(@as(u64, 0xFEDCBA98), decodeInteger("0xFE'DC'BA'98").?.value);
430 try std.testing.expectEqual(@as(u64, 0xaa), decodeInteger("0b1010'1010").?.value);
431 try std.testing.expectEqual(@as(u64, 299792458), decodeInteger("299'792'458").?.value);
432 try std.testing.expectEqual(@as(u64, 8), decodeInteger("010").?.value);
433 try std.testing.expectEqual(@as(u64, 0), decodeInteger("0").?.value);
434
435 const unsigned_long = decodeInteger("42UL").?;
436 try std.testing.expect(unsigned_long.is_unsigned);
437 try std.testing.expect(unsigned_long.is_long);
438
439 const signed_bitint = decodeInteger("3wb").?;
440 try std.testing.expectEqual(@as(u16, 3), signed_bitint.bit_width.?);
441 try std.testing.expect(!signed_bitint.is_unsigned);
442
443 const unsigned_bitint = decodeInteger("3uwb").?;
444 try std.testing.expectEqual(@as(u16, 2), unsigned_bitint.bit_width.?);
445 try std.testing.expect(unsigned_bitint.is_unsigned);
446
447 try std.testing.expectEqual(@as(u16, 2), decodeInteger("0WB").?.bit_width.?);
448 try std.testing.expectEqual(@as(u16, 1), decodeInteger("0uWB").?.bit_width.?);
449
450 try std.testing.expect(decodeInteger("") == null);
451 try std.testing.expect(decodeInteger("0x") == null);
452 try std.testing.expect(decodeInteger("0b") == null);
453 try std.testing.expect(decodeInteger("1''2") == null);
454 try std.testing.expect(decodeInteger("1'") == null);
455 try std.testing.expect(decodeInteger("1wbwb") == null);
456 try std.testing.expect(decodeInteger("1Lwb") == null);
457 }
458
459 test "float constants decode suffixes" {
460 try std.testing.expectEqual(@as(f64, 1.5), decodeFloat(std.testing.allocator, "1.5").?.value);
461 try std.testing.expectEqual(@as(f64, 1.414213562), decodeFloat(std.testing.allocator, "1.414'213'562").?.value);
462 try std.testing.expectEqual(@as(f64, 1000.0), decodeFloat(std.testing.allocator, "1'000.0").?.value);
463 try std.testing.expectEqual(FloatKind.float, decodeFloat(std.testing.allocator, "2.5f").?.kind);
464 try std.testing.expectEqual(FloatKind.decimal32, decodeFloat(std.testing.allocator, "2.5df").?.kind);
465 try std.testing.expectEqual(FloatKind.decimal64, decodeFloat(std.testing.allocator, "2.5DD").?.kind);
466 try std.testing.expectEqual(FloatKind.decimal128, decodeFloat(std.testing.allocator, "2.5DL").?.kind);
467 try std.testing.expectEqual(@as(f64, 1.0e3), decodeFloat(std.testing.allocator, "1e3").?.value);
468 try std.testing.expectEqual(@as(f64, 0.25), decodeFloat(std.testing.allocator, "0.25L").?.value);
469 try std.testing.expect(decodeFloat(std.testing.allocator, "1e'3") == null);
470 try std.testing.expect(decodeFloat(std.testing.allocator, "0x1p0df") == null);
471 }
472
473 test "character constants decode escapes" {
474 try std.testing.expectEqual(@as(u8, 'a'), decodeCharacter("'a'").?);
475 try std.testing.expectEqual(@as(u8, '\n'), decodeCharacter("'\\n'").?);
476 try std.testing.expectEqual(@as(u8, 0), decodeCharacter("'\\0'").?);
477 try std.testing.expect(decodeCharacter("'ab'") == null);
478 }
479
480 test "string constants decode escapes" {
481 const decoded = decodeString(std.testing.allocator, "\"a\\n\\\\\\\"\"").?;
482 defer std.testing.allocator.free(decoded);
483 try std.testing.expectEqualStrings("a\n\\\"", decoded);
484 }