tiny.smg.tree.runtime.lexer

1const std = @import("std");2const runtime = @import("root.zig");34const abi = runtime.abi;56pub const Position = struct {7    byte: u32,8    point: abi.Point,9};1011pub const Token = struct {12    symbol: abi.Symbol,13    start: Position,14    end: Position,15};1617pub const Lexer = struct {18    abi: abi.Lexer,19    source: []const u8,20    current: Position,21    token_start: Position,22    token_end: Position,23    marked: bool,2425    pub fn init(source: []const u8, initial: Position) Lexer {26        var self = Lexer{27            .abi = .{28                .lookahead = 0,29                .result_symbol = 0,30                .advance = advanceAbi,31                .mark_end = markEndAbi,32                .get_column = columnAbi,33                .is_at_included_range_start = rangeStartAbi,34                .eof = eofAbi,35                .log = null,36            },37            .source = source,38            .current = initial,39            .token_start = initial,40            .token_end = initial,41            .marked = false,42        };43        self.refresh();44        return self;45    }4647    pub fn start(self: *Lexer) void {48        self.token_start = self.current;49        self.token_end = self.current;50        self.marked = false;51        self.abi.result_symbol = 0;52        if (self.current.byte == 0 and self.abi.lookahead == 0xfeff) self.advance(true);53    }5455    pub fn finish(self: *Lexer) Token {56        if (!self.marked) self.markEnd();57        if (self.token_end.byte < self.token_start.byte) self.token_start = self.token_end;58        return .{59            .symbol = self.abi.result_symbol,60            .start = self.token_start,61            .end = self.token_end,62        };63    }6465    pub fn lookahead(self: *const Lexer) i32 {66        return self.abi.lookahead;67    }6869    pub fn setResult(self: *Lexer, symbol: abi.Symbol) void {70        self.abi.result_symbol = symbol;71    }7273    pub fn advance(self: *Lexer, skip: bool) void {74        if (self.current.byte < self.source.len) {75            const length = codepointLength(self.source[self.current.byte..]);76            if (self.abi.lookahead == '\n') {77                self.current.point.row += 1;78                self.current.point.column = 0;79            } else {80                self.current.point.column += @intCast(length);81            }82            self.current.byte += @intCast(length);83        }84        if (skip) self.token_start = self.current;85        self.refresh();86    }8788    pub fn markEnd(self: *Lexer) void {89        self.token_end = self.current;90        self.marked = true;91    }9293    pub fn includedRangeStart(_: *const Lexer) bool {94        return false;95    }9697    pub fn eof(self: *const Lexer) bool {98        return self.current.byte == self.source.len;99    }100101    fn refresh(self: *Lexer) void {102        if (self.eof()) {103            self.abi.lookahead = 0;104            return;105        }106        const remaining = self.source[self.current.byte..];107        const length = codepointLength(remaining);108        if (length == 1 and remaining[0] >= 0x80) return self.setDecodeError();109        self.abi.lookahead = @intCast(std.unicode.utf8Decode(remaining[0..length]) catch {110            return self.setDecodeError();111        });112    }113114    fn setDecodeError(self: *Lexer) void {115        self.abi.lookahead = -1;116    }117118    fn codepointLength(source: []const u8) usize {119        const length = std.unicode.utf8ByteSequenceLength(source[0]) catch return 1;120        if (length > source.len) return 1;121        _ = std.unicode.utf8Decode(source[0..length]) catch return 1;122        return length;123    }124125    fn parent(pointer: anytype) if (@TypeOf(pointer) == *abi.Lexer) *Lexer else *const Lexer {126        return @fieldParentPtr("abi", pointer);127    }128129    fn advanceAbi(pointer: *abi.Lexer, skip: bool) callconv(.c) void {130        parent(pointer).advance(skip);131    }132133    fn markEndAbi(pointer: *abi.Lexer) callconv(.c) void {134        parent(pointer).markEnd();135    }136137    fn columnAbi(pointer: *abi.Lexer) callconv(.c) u32 {138        return parent(pointer).current.point.column;139    }140141    fn rangeStartAbi(pointer: *const abi.Lexer) callconv(.c) bool {142        return parent(pointer).includedRangeStart();143    }144145    fn eofAbi(pointer: *const abi.Lexer) callconv(.c) bool {146        return parent(pointer).eof();147    }148};149150test "lexer exposes generated and native scanner semantics" {151    var value = Lexer.init("a\n\xe2\x82\xac", .{ .byte = 0, .point = .{ .row = 0, .column = 0 } });152    value.start();153    try std.testing.expectEqual(@as(i32, 'a'), value.lookahead());154    value.advance(true);155    try std.testing.expectEqual(@as(u32, 1), value.token_start.byte);156    value.advance(false);157    try std.testing.expectEqual(@as(u32, 1), value.current.point.row);158    try std.testing.expectEqual(@as(i32, 0x20ac), value.lookahead());159    value.setResult(7);160    value.advance(false);161    const token = value.finish();162    try std.testing.expectEqual(@as(abi.Symbol, 7), token.symbol);163    try std.testing.expectEqual(@as(u32, 1), token.start.byte);164    try std.testing.expectEqual(@as(u32, 5), token.end.byte);165}

1pub const lexer = @import("lexer.zig");