tiny.chant.token
Defined in tiny.chant.
API (10)
Actions
Public operations.
Types and contracts
Public types and contracts.
Source
Source: lib/chant/src/root.zig:59
zig
pub const token = @import("token.zig");Source: lib/chant/src/token.zig
zig
const std = @import("std");pub const Kind = enum { identifier, integer, floating, character, string, kw_auto, kw_break, kw_case, kw_char, kw_const, kw_continue, kw_default, kw_do, kw_double, kw_decimal32, kw_decimal64, kw_decimal128, kw_else, kw_enum, kw_extern, kw_float, kw_for, kw_goto, kw_if, kw_inline, kw_int, kw_long, kw_register, kw_restrict, kw_return, kw_short, kw_signed, kw_sizeof, kw_static, kw_struct, kw_switch, kw_typedef, kw_union, kw_unsigned, kw_bitint, kw_void, kw_volatile, kw_while, kw_bool, kw_true, kw_false, kw_nullptr, kw_alignas, kw_alignof, kw_constexpr, kw_static_assert, kw_thread_local, kw_typeof, kw_typeof_unqual, lparen, rparen, lbrace, rbrace, lbracket, rbracket, semicolon, comma, colon, question, plus, minus, star, slash, percent, amp, pipe, caret, tilde, bang, lt, gt, le, ge, eq, ne, amp_amp, pipe_pipe, shl, shr, assign, plus_assign, minus_assign, star_assign, slash_assign, percent_assign, amp_assign, pipe_assign, caret_assign, shl_assign, shr_assign, plus_plus, minus_minus, arrow, dot, ellipsis, eof,};pub const Token = struct { kind: Kind, text: []const u8, file: []const u8, line: u32, column: u32,};const Keyword = struct { text: []const u8, kind: Kind,};const keywords = [_]Keyword{ .{ .text = "auto", .kind = .kw_auto }, .{ .text = "break", .kind = .kw_break }, .{ .text = "case", .kind = .kw_case }, .{ .text = "char", .kind = .kw_char }, .{ .text = "const", .kind = .kw_const }, .{ .text = "continue", .kind = .kw_continue }, .{ .text = "default", .kind = .kw_default }, .{ .text = "do", .kind = .kw_do }, .{ .text = "double", .kind = .kw_double }, .{ .text = "_Decimal32", .kind = .kw_decimal32 }, .{ .text = "_Decimal64", .kind = .kw_decimal64 }, .{ .text = "_Decimal128", .kind = .kw_decimal128 }, .{ .text = "else", .kind = .kw_else }, .{ .text = "enum", .kind = .kw_enum }, .{ .text = "extern", .kind = .kw_extern }, .{ .text = "float", .kind = .kw_float }, .{ .text = "for", .kind = .kw_for }, .{ .text = "goto", .kind = .kw_goto }, .{ .text = "if", .kind = .kw_if }, .{ .text = "inline", .kind = .kw_inline }, .{ .text = "int", .kind = .kw_int }, .{ .text = "long", .kind = .kw_long }, .{ .text = "register", .kind = .kw_register }, .{ .text = "restrict", .kind = .kw_restrict }, .{ .text = "return", .kind = .kw_return }, .{ .text = "short", .kind = .kw_short }, .{ .text = "signed", .kind = .kw_signed }, .{ .text = "sizeof", .kind = .kw_sizeof }, .{ .text = "static", .kind = .kw_static }, .{ .text = "struct", .kind = .kw_struct }, .{ .text = "switch", .kind = .kw_switch }, .{ .text = "typedef", .kind = .kw_typedef }, .{ .text = "union", .kind = .kw_union }, .{ .text = "unsigned", .kind = .kw_unsigned }, .{ .text = "_BitInt", .kind = .kw_bitint }, .{ .text = "void", .kind = .kw_void }, .{ .text = "volatile", .kind = .kw_volatile }, .{ .text = "while", .kind = .kw_while }, .{ .text = "bool", .kind = .kw_bool }, .{ .text = "_Bool", .kind = .kw_bool }, .{ .text = "true", .kind = .kw_true }, .{ .text = "false", .kind = .kw_false }, .{ .text = "nullptr", .kind = .kw_nullptr }, .{ .text = "alignas", .kind = .kw_alignas }, .{ .text = "_Alignas", .kind = .kw_alignas }, .{ .text = "alignof", .kind = .kw_alignof }, .{ .text = "_Alignof", .kind = .kw_alignof }, .{ .text = "constexpr", .kind = .kw_constexpr }, .{ .text = "static_assert", .kind = .kw_static_assert }, .{ .text = "_Static_assert", .kind = .kw_static_assert }, .{ .text = "thread_local", .kind = .kw_thread_local }, .{ .text = "_Thread_local", .kind = .kw_thread_local }, .{ .text = "typeof", .kind = .kw_typeof }, .{ .text = "__typeof__", .kind = .kw_typeof }, .{ .text = "typeof_unqual", .kind = .kw_typeof_unqual }, .{ .text = "__typeof_unqual__", .kind = .kw_typeof_unqual }, .{ .text = "__restrict", .kind = .kw_restrict }, .{ .text = "__restrict__", .kind = .kw_restrict }, .{ .text = "__inline", .kind = .kw_inline }, .{ .text = "__inline__", .kind = .kw_inline },};pub fn keywordKind(text: []const u8) ?Kind { for (keywords) |keyword| { if (std.mem.eql(u8, keyword.text, text)) return keyword.kind; } return null;}pub const IntegerValue = struct { value: u64, is_unsigned: bool, is_long: bool, bit_width: ?u16 = null,};pub fn decodeInteger(text: []const u8) ?IntegerValue { var digits = text; var is_unsigned = false; var is_long = false; var is_bit_precise = false; while (digits.len > 0) { const last = digits[digits.len - 1]; if (endsWithBitPreciseSuffix(digits)) { if (is_bit_precise) return null; is_bit_precise = true; digits = digits[0 .. digits.len - 2]; } else if (last == 'u' or last == 'U') { if (is_unsigned) return null; is_unsigned = true; digits = digits[0 .. digits.len - 1]; } else if (last == 'l' or last == 'L') { if (is_long and digits.len >= 2 and (digits[digits.len - 2] == 'l' or digits[digits.len - 2] == 'L')) return null; is_long = true; digits = digits[0 .. digits.len - 1]; } else { break; } } if (is_bit_precise and is_long) return null; if (digits.len == 0) return null; var base: u8 = 10; if (digits.len >= 2 and digits[0] == '0' and (digits[1] == 'x' or digits[1] == 'X')) { base = 16; digits = digits[2..]; } else if (digits.len >= 2 and digits[0] == '0' and (digits[1] == 'b' or digits[1] == 'B')) { base = 2; digits = digits[2..]; } else if (digits.len >= 2 and digits[0] == '0') { base = 8; digits = digits[1..]; } if (digits.len == 0) return null; const value = parseSeparatedUnsigned(digits, base) orelse return null; const bit_width = if (is_bit_precise) bitPreciseWidth(value, is_unsigned) else null; return .{ .value = value, .is_unsigned = is_unsigned, .is_long = is_long, .bit_width = bit_width };}fn endsWithBitPreciseSuffix(text: []const u8) bool { if (text.len < 2) return false; const suffix = text[text.len - 2 ..]; return std.mem.eql(u8, suffix, "wb") or std.mem.eql(u8, suffix, "WB");}fn bitPreciseWidth(value: u64, is_unsigned: bool) u16 { const bits: u16 = if (value == 0) 0 else @intCast(64 - @clz(value)); if (is_unsigned) return @max(@as(u16, 1), bits); return @max(@as(u16, 2), bits + 1);}fn parseSeparatedUnsigned(digits: []const u8, base: u8) ?u64 { var value: u64 = 0; var saw_digit = false; var previous_digit = false; for (digits, 0..) |byte, index| { if (byte == '\'') { if (!previous_digit or index + 1 >= digits.len or digitValue(digits[index + 1], base) == null) return null; previous_digit = false; continue; } const digit = digitValue(byte, base) orelse return null; value = std.math.mul(u64, value, base) catch return null; value = std.math.add(u64, value, digit) catch return null; saw_digit = true; previous_digit = true; } return if (saw_digit and previous_digit) value else null;}fn digitValue(byte: u8, base: u8) ?u64 { const digit: u8 = if (byte >= '0' and byte <= '9') byte - '0' else if (byte >= 'a' and byte <= 'f') byte - 'a' + 10 else if (byte >= 'A' and byte <= 'F') byte - 'A' + 10 else return null; if (digit >= base) return null; return digit;}pub const FloatKind = enum { float, double, decimal32, decimal64, decimal128,};pub const FloatValue = struct { value: f64, kind: FloatKind,};pub fn decodeFloat(allocator: std.mem.Allocator, text: []const u8) ?FloatValue { var digits = text; var kind: FloatKind = .double; if (digits.len > 0) { const last = digits[digits.len - 1]; if (decimalFloatSuffixKind(digits)) |decimal_kind| { if (isHexFloating(digits)) return null; kind = decimal_kind; digits = digits[0 .. digits.len - 2]; } else if (last == 'f' or last == 'F') { kind = .float; digits = digits[0 .. digits.len - 1]; } else if (last == 'l' or last == 'L') { digits = digits[0 .. digits.len - 1]; } } if (digits.len == 0) return null; const normalized = stripFloatSeparators(allocator, digits) orelse return null; defer if (normalized.ptr != digits.ptr) allocator.free(normalized); const value = std.fmt.parseFloat(f64, normalized) catch return null; return .{ .value = value, .kind = kind };}fn decimalFloatSuffixKind(text: []const u8) ?FloatKind { if (text.len < 2) return null; const suffix = text[text.len - 2 ..]; if (std.mem.eql(u8, suffix, "df") or std.mem.eql(u8, suffix, "DF")) return .decimal32; if (std.mem.eql(u8, suffix, "dd") or std.mem.eql(u8, suffix, "DD")) return .decimal64; if (std.mem.eql(u8, suffix, "dl") or std.mem.eql(u8, suffix, "DL")) return .decimal128; return null;}fn isHexFloating(text: []const u8) bool { return text.len >= 2 and text[0] == '0' and (text[1] == 'x' or text[1] == 'X');}fn stripFloatSeparators(allocator: std.mem.Allocator, text: []const u8) ?[]const u8 { if (std.mem.indexOfScalar(u8, text, '\'') == null) return text; var out = std.ArrayListUnmanaged(u8).empty; defer out.deinit(allocator); var previous_digit = false; for (text, 0..) |byte, index| { if (byte == '\'') { if (!previous_digit or index + 1 >= text.len or !std.ascii.isDigit(text[index + 1])) return null; previous_digit = false; continue; } out.append(allocator, byte) catch return null; previous_digit = std.ascii.isDigit(byte); } if (!previous_digit) return null; return out.toOwnedSlice(allocator) catch null;}pub fn decodeCharacter(text: []const u8) ?u8 { const literal = if (std.mem.startsWith(u8, text, "u8")) text[2..] else text; if (literal.len < 3 or literal[0] != '\'' or literal[literal.len - 1] != '\'') return null; const body = literal[1 .. literal.len - 1]; if (body.len == 1) return body[0]; if (body.len == 2 and body[0] == '\\') { return switch (body[1]) { 'n' => '\n', 't' => '\t', 'r' => '\r', '0' => 0, '\\' => '\\', '\'' => '\'', '"' => '"', else => null, }; } return null;}pub fn decodeString(allocator: std.mem.Allocator, text: []const u8) ?[]const u8 { const literal = if (std.mem.startsWith(u8, text, "u8")) text[2..] else text; if (literal.len < 2 or literal[0] != '"' or literal[literal.len - 1] != '"') return null; const body = literal[1 .. literal.len - 1]; var out = std.ArrayListUnmanaged(u8).empty; var index: usize = 0; while (index < body.len) { const byte = body[index]; if (byte != '\\') { out.append(allocator, byte) catch return null; index += 1; continue; } if (index + 1 >= body.len) return null; const decoded: u8 = switch (body[index + 1]) { 'n' => '\n', 't' => '\t', 'r' => '\r', '0' => 0, '\\' => '\\', '\'' => '\'', '"' => '"', else => return null, }; out.append(allocator, decoded) catch return null; index += 2; } return out.toOwnedSlice(allocator) catch null;}test "keywords resolve and identifiers do not" { try std.testing.expectEqual(Kind.kw_double, keywordKind("double").?); try std.testing.expectEqual(Kind.kw_decimal32, keywordKind("_Decimal32").?); try std.testing.expectEqual(Kind.kw_decimal64, keywordKind("_Decimal64").?); try std.testing.expectEqual(Kind.kw_decimal128, keywordKind("_Decimal128").?); try std.testing.expectEqual(Kind.kw_for, keywordKind("for").?); try std.testing.expectEqual(Kind.kw_bool, keywordKind("bool").?); try std.testing.expectEqual(Kind.kw_true, keywordKind("true").?); try std.testing.expectEqual(Kind.kw_false, keywordKind("false").?); try std.testing.expectEqual(Kind.kw_nullptr, keywordKind("nullptr").?); try std.testing.expectEqual(Kind.kw_bitint, keywordKind("_BitInt").?); try std.testing.expectEqual(Kind.kw_alignas, keywordKind("alignas").?); try std.testing.expectEqual(Kind.kw_alignof, keywordKind("alignof").?); try std.testing.expectEqual(Kind.kw_constexpr, keywordKind("constexpr").?); try std.testing.expectEqual(Kind.kw_static_assert, keywordKind("static_assert").?); try std.testing.expectEqual(Kind.kw_thread_local, keywordKind("thread_local").?); try std.testing.expectEqual(Kind.kw_typeof, keywordKind("typeof").?); try std.testing.expectEqual(Kind.kw_typeof_unqual, keywordKind("typeof_unqual").?); try std.testing.expectEqual(Kind.kw_restrict, keywordKind("__restrict").?); try std.testing.expect(keywordKind("kernel_gemm") == null);}test "integer constants decode bases and suffixes" { try std.testing.expectEqual(@as(u64, 42), decodeInteger("42").?.value); try std.testing.expectEqual(@as(u64, 255), decodeInteger("0xFF").?.value); try std.testing.expectEqual(@as(u64, 0xFEDCBA98), decodeInteger("0xFE'DC'BA'98").?.value); try std.testing.expectEqual(@as(u64, 0xaa), decodeInteger("0b1010'1010").?.value); try std.testing.expectEqual(@as(u64, 299792458), decodeInteger("299'792'458").?.value); try std.testing.expectEqual(@as(u64, 8), decodeInteger("010").?.value); try std.testing.expectEqual(@as(u64, 0), decodeInteger("0").?.value); const unsigned_long = decodeInteger("42UL").?; try std.testing.expect(unsigned_long.is_unsigned); try std.testing.expect(unsigned_long.is_long); const signed_bitint = decodeInteger("3wb").?; try std.testing.expectEqual(@as(u16, 3), signed_bitint.bit_width.?); try std.testing.expect(!signed_bitint.is_unsigned); const unsigned_bitint = decodeInteger("3uwb").?; try std.testing.expectEqual(@as(u16, 2), unsigned_bitint.bit_width.?); try std.testing.expect(unsigned_bitint.is_unsigned); try std.testing.expectEqual(@as(u16, 2), decodeInteger("0WB").?.bit_width.?); try std.testing.expectEqual(@as(u16, 1), decodeInteger("0uWB").?.bit_width.?); try std.testing.expect(decodeInteger("") == null); try std.testing.expect(decodeInteger("0x") == null); try std.testing.expect(decodeInteger("0b") == null); try std.testing.expect(decodeInteger("1''2") == null); try std.testing.expect(decodeInteger("1'") == null); try std.testing.expect(decodeInteger("1wbwb") == null); try std.testing.expect(decodeInteger("1Lwb") == null);}test "float constants decode suffixes" { try std.testing.expectEqual(@as(f64, 1.5), decodeFloat(std.testing.allocator, "1.5").?.value); try std.testing.expectEqual(@as(f64, 1.414213562), decodeFloat(std.testing.allocator, "1.414'213'562").?.value); try std.testing.expectEqual(@as(f64, 1000.0), decodeFloat(std.testing.allocator, "1'000.0").?.value); try std.testing.expectEqual(FloatKind.float, decodeFloat(std.testing.allocator, "2.5f").?.kind); try std.testing.expectEqual(FloatKind.decimal32, decodeFloat(std.testing.allocator, "2.5df").?.kind); try std.testing.expectEqual(FloatKind.decimal64, decodeFloat(std.testing.allocator, "2.5DD").?.kind); try std.testing.expectEqual(FloatKind.decimal128, decodeFloat(std.testing.allocator, "2.5DL").?.kind); try std.testing.expectEqual(@as(f64, 1.0e3), decodeFloat(std.testing.allocator, "1e3").?.value); try std.testing.expectEqual(@as(f64, 0.25), decodeFloat(std.testing.allocator, "0.25L").?.value); try std.testing.expect(decodeFloat(std.testing.allocator, "1e'3") == null); try std.testing.expect(decodeFloat(std.testing.allocator, "0x1p0df") == null);}test "character constants decode escapes" { try std.testing.expectEqual(@as(u8, 'a'), decodeCharacter("'a'").?); try std.testing.expectEqual(@as(u8, '\n'), decodeCharacter("'\\n'").?); try std.testing.expectEqual(@as(u8, 0), decodeCharacter("'\\0'").?); try std.testing.expect(decodeCharacter("'ab'") == null);}test "string constants decode escapes" { const decoded = decodeString(std.testing.allocator, "\"a\\n\\\\\\\"\"").?; defer std.testing.allocator.free(decoded); try std.testing.expectEqualStrings("a\n\\\"", decoded);}Audit
| Definitions | 9 |
|---|---|
| Public names | 9 |
| Members | 11 |
| Version | 26.7.0 |
| Revision | daab053ee433 |