tiny.zen.math.token

1pub const token = @import("token.zig");

1const std = @import("std");23pub const Error = error{InvalidEquation};45pub const Token = union(enum) {6    char: u21,7    command: []const u8,8    open,9    close,10    caret,11    underscore,12    prime,13    end,14};1516pub const Tokenizer = struct {17    source: []const u8,18    index: usize = 0,19    start: usize = 0,20    reason: []const u8 = "",2122    pub fn next(self: *Tokenizer) Error!Token {23        while (self.index < self.source.len and isSpace(self.source[self.index])) self.index += 1;24        self.start = self.index;25        if (self.index >= self.source.len) return .end;26        const byte = self.source[self.index];27        switch (byte) {28            '{' => {29                self.index += 1;30                return .open;31            },32            '}' => {33                self.index += 1;34                return .close;35            },36            '^' => {37                self.index += 1;38                return .caret;39            },40            '_' => {41                self.index += 1;42                return .underscore;43            },44            '\'' => {45                self.index += 1;46                return .prime;47            },48            '\\' => {49                self.index += 1;50                if (self.index >= self.source.len) return self.fail("incomplete command");51                if (!std.ascii.isAlphabetic(self.source[self.index])) {52                    self.index += 1;53                    return .{ .command = self.source[self.index - 1 .. self.index] };54                }55                const start = self.index;56                while (self.index < self.source.len and std.ascii.isAlphabetic(self.source[self.index])) self.index += 1;57                return .{ .command = self.source[start..self.index] };58            },59            else => {60                if (byte < 0x80) {61                    self.index += 1;62                    return .{ .char = byte };63                }64                const len = std.unicode.utf8ByteSequenceLength(byte) catch return self.fail("invalid utf-8");65                if (self.index + len > self.source.len) return self.fail("invalid utf-8");66                const decoded = std.unicode.utf8Decode(self.source[self.index .. self.index + len]) catch return self.fail("invalid utf-8");67                self.index += len;68                return .{ .char = decoded };69            },70        }71    }7273    fn fail(self: *Tokenizer, reason: []const u8) Error {74        self.reason = reason;75        return error.InvalidEquation;76    }7778    fn isSpace(byte: u8) bool {79        return byte == ' ' or byte == '\t' or byte == '\n' or byte == '\r';80    }81};8283test "tokenizer splits commands scripts and characters" {84    var tokens = Tokenizer{ .source = "\\frac{a}{2} ^ x_\\pi \\, ω'" };85    try std.testing.expectEqualStrings("frac", (try tokens.next()).command);86    try std.testing.expect((try tokens.next()) == .open);87    try std.testing.expectEqual(@as(u21, 'a'), (try tokens.next()).char);88    try std.testing.expect((try tokens.next()) == .close);89    try std.testing.expect((try tokens.next()) == .open);90    try std.testing.expectEqual(@as(u21, '2'), (try tokens.next()).char);91    try std.testing.expect((try tokens.next()) == .close);92    try std.testing.expect((try tokens.next()) == .caret);93    try std.testing.expectEqual(@as(u21, 'x'), (try tokens.next()).char);94    try std.testing.expect((try tokens.next()) == .underscore);95    try std.testing.expectEqualStrings("pi", (try tokens.next()).command);96    try std.testing.expectEqualStrings(",", (try tokens.next()).command);97    try std.testing.expectEqual(@as(u21, 'ω'), (try tokens.next()).char);98    try std.testing.expect((try tokens.next()) == .prime);99    try std.testing.expect((try tokens.next()) == .end);100}101102test "tokenizer records token starts" {103    var tokens = Tokenizer{ .source = "  x \\pi" };104    _ = try tokens.next();105    try std.testing.expectEqual(@as(usize, 2), tokens.start);106    _ = try tokens.next();107    try std.testing.expectEqual(@as(usize, 4), tokens.start);108    _ = try tokens.next();109    try std.testing.expectEqual(@as(usize, 7), tokens.start);110}111112test "tokenizer rejects trailing backslash and bad utf8" {113    var trailing = Tokenizer{ .source = "x\\" };114    _ = try trailing.next();115    try std.testing.expectError(error.InvalidEquation, trailing.next());116    try std.testing.expectEqualStrings("incomplete command", trailing.reason);117    try std.testing.expectEqual(@as(usize, 1), trailing.start);118    var bad = Tokenizer{ .source = &.{ 0xFF, 0x20 } };119    try std.testing.expectError(error.InvalidEquation, bad.next());120    try std.testing.expectEqualStrings("invalid utf-8", bad.reason);121    try std.testing.expectEqual(@as(usize, 0), bad.start);122}