lib/zen/src/math/token.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

  1 const std = @import("std");
  2 
  3 pub const Error = error{InvalidEquation};
  4 
  5 pub const Token = union(enum) {
  6     char: u21,
  7     command: []const u8,
  8     open,
  9     close,
 10     caret,
 11     underscore,
 12     prime,
 13     end,
 14 };
 15 
 16 pub const Tokenizer = struct {
 17     source: []const u8,
 18     index: usize = 0,
 19     start: usize = 0,
 20     reason: []const u8 = "",
 21 
 22     pub fn next(self: *Tokenizer) Error!Token {
 23         while (self.index < self.source.len and isSpace(self.source[self.index])) self.index += 1;
 24         self.start = self.index;
 25         if (self.index >= self.source.len) return .end;
 26         const byte = self.source[self.index];
 27         switch (byte) {
 28             '{' => {
 29                 self.index += 1;
 30                 return .open;
 31             },
 32             '}' => {
 33                 self.index += 1;
 34                 return .close;
 35             },
 36             '^' => {
 37                 self.index += 1;
 38                 return .caret;
 39             },
 40             '_' => {
 41                 self.index += 1;
 42                 return .underscore;
 43             },
 44             '\'' => {
 45                 self.index += 1;
 46                 return .prime;
 47             },
 48             '\\' => {
 49                 self.index += 1;
 50                 if (self.index >= self.source.len) return self.fail("incomplete command");
 51                 if (!std.ascii.isAlphabetic(self.source[self.index])) {
 52                     self.index += 1;
 53                     return .{ .command = self.source[self.index - 1 .. self.index] };
 54                 }
 55                 const start = self.index;
 56                 while (self.index < self.source.len and std.ascii.isAlphabetic(self.source[self.index])) self.index += 1;
 57                 return .{ .command = self.source[start..self.index] };
 58             },
 59             else => {
 60                 if (byte < 0x80) {
 61                     self.index += 1;
 62                     return .{ .char = byte };
 63                 }
 64                 const len = std.unicode.utf8ByteSequenceLength(byte) catch return self.fail("invalid utf-8");
 65                 if (self.index + len > self.source.len) return self.fail("invalid utf-8");
 66                 const decoded = std.unicode.utf8Decode(self.source[self.index .. self.index + len]) catch return self.fail("invalid utf-8");
 67                 self.index += len;
 68                 return .{ .char = decoded };
 69             },
 70         }
 71     }
 72 
 73     fn fail(self: *Tokenizer, reason: []const u8) Error {
 74         self.reason = reason;
 75         return error.InvalidEquation;
 76     }
 77 
 78     fn isSpace(byte: u8) bool {
 79         return byte == ' ' or byte == '\t' or byte == '\n' or byte == '\r';
 80     }
 81 };
 82 
 83 test "tokenizer splits commands scripts and characters" {
 84     var tokens = Tokenizer{ .source = "\\frac{a}{2} ^ x_\\pi \\, ω'" };
 85     try std.testing.expectEqualStrings("frac", (try tokens.next()).command);
 86     try std.testing.expect((try tokens.next()) == .open);
 87     try std.testing.expectEqual(@as(u21, 'a'), (try tokens.next()).char);
 88     try std.testing.expect((try tokens.next()) == .close);
 89     try std.testing.expect((try tokens.next()) == .open);
 90     try std.testing.expectEqual(@as(u21, '2'), (try tokens.next()).char);
 91     try std.testing.expect((try tokens.next()) == .close);
 92     try std.testing.expect((try tokens.next()) == .caret);
 93     try std.testing.expectEqual(@as(u21, 'x'), (try tokens.next()).char);
 94     try std.testing.expect((try tokens.next()) == .underscore);
 95     try std.testing.expectEqualStrings("pi", (try tokens.next()).command);
 96     try std.testing.expectEqualStrings(",", (try tokens.next()).command);
 97     try std.testing.expectEqual(@as(u21, 'ω'), (try tokens.next()).char);
 98     try std.testing.expect((try tokens.next()) == .prime);
 99     try std.testing.expect((try tokens.next()) == .end);
100 }
101 
102 test "tokenizer records token starts" {
103     var tokens = Tokenizer{ .source = "  x \\pi" };
104     _ = try tokens.next();
105     try std.testing.expectEqual(@as(usize, 2), tokens.start);
106     _ = try tokens.next();
107     try std.testing.expectEqual(@as(usize, 4), tokens.start);
108     _ = try tokens.next();
109     try std.testing.expectEqual(@as(usize, 7), tokens.start);
110 }
111 
112 test "tokenizer rejects trailing backslash and bad utf8" {
113     var trailing = Tokenizer{ .source = "x\\" };
114     _ = try trailing.next();
115     try std.testing.expectError(error.InvalidEquation, trailing.next());
116     try std.testing.expectEqualStrings("incomplete command", trailing.reason);
117     try std.testing.expectEqual(@as(usize, 1), trailing.start);
118     var bad = Tokenizer{ .source = &.{ 0xFF, 0x20 } };
119     try std.testing.expectError(error.InvalidEquation, bad.next());
120     try std.testing.expectEqualStrings("invalid utf-8", bad.reason);
121     try std.testing.expectEqual(@as(usize, 0), bad.start);
122 }