lib/xkb/src/keymap/text/lexer.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

  1 const std = @import("std");
  2 
  3 pub const Error = error{InvalidSyntax};
  4 
  5 pub const Tag = enum {
  6     word,
  7     string,
  8     key_name,
  9     left_brace,
 10     right_brace,
 11     left_bracket,
 12     right_bracket,
 13     left_paren,
 14     right_paren,
 15     equal,
 16     semicolon,
 17     comma,
 18     plus,
 19     dot,
 20     bang,
 21     minus,
 22     end,
 23 };
 24 
 25 pub const Token = struct {
 26     tag: Tag,
 27     text: []const u8,
 28 };
 29 
 30 pub const Lexer = struct {
 31     input: []const u8,
 32     position: usize = 0,
 33 
 34     pub fn next(self: *Lexer) Error!Token {
 35         try self.skipIgnored();
 36         if (self.position == self.input.len) return .{ .tag = .end, .text = "" };
 37 
 38         const start = self.position;
 39         const byte = self.input[self.position];
 40         self.position += 1;
 41         return switch (byte) {
 42             '{' => self.punctuation(.left_brace, start),
 43             '}' => self.punctuation(.right_brace, start),
 44             '[' => self.punctuation(.left_bracket, start),
 45             ']' => self.punctuation(.right_bracket, start),
 46             '(' => self.punctuation(.left_paren, start),
 47             ')' => self.punctuation(.right_paren, start),
 48             '=' => self.punctuation(.equal, start),
 49             ';' => self.punctuation(.semicolon, start),
 50             ',' => self.punctuation(.comma, start),
 51             '+' => self.punctuation(.plus, start),
 52             '.' => self.punctuation(.dot, start),
 53             '!' => self.punctuation(.bang, start),
 54             '-' => self.punctuation(.minus, start),
 55             '"' => try self.quoted(start),
 56             '<' => try self.keyName(start),
 57             else => self.word(start),
 58         };
 59     }
 60 
 61     fn punctuation(self: *const Lexer, tag: Tag, start: usize) Token {
 62         return .{ .tag = tag, .text = self.input[start .. start + 1] };
 63     }
 64 
 65     fn quoted(self: *Lexer, start: usize) Error!Token {
 66         var escaped = false;
 67         while (self.position < self.input.len) : (self.position += 1) {
 68             const byte = self.input[self.position];
 69             if (escaped) {
 70                 escaped = false;
 71                 continue;
 72             }
 73             if (byte == '\\') {
 74                 escaped = true;
 75                 continue;
 76             }
 77             if (byte == '"') {
 78                 const end = self.position;
 79                 self.position += 1;
 80                 return .{ .tag = .string, .text = self.input[start + 1 .. end] };
 81             }
 82             if (byte == 0) return error.InvalidSyntax;
 83         }
 84         return error.InvalidSyntax;
 85     }
 86 
 87     fn keyName(self: *Lexer, start: usize) Error!Token {
 88         const end = std.mem.indexOfScalarPos(u8, self.input, self.position, '>') orelse
 89             return error.InvalidSyntax;
 90         if (end == self.position) return error.InvalidSyntax;
 91         self.position = end + 1;
 92         return .{ .tag = .key_name, .text = self.input[start + 1 .. end] };
 93     }
 94 
 95     fn word(self: *Lexer, start: usize) Token {
 96         while (self.position < self.input.len and !delimiter(self.input[self.position])) {
 97             self.position += 1;
 98         }
 99         return .{ .tag = .word, .text = self.input[start..self.position] };
100     }
101 
102     fn skipIgnored(self: *Lexer) Error!void {
103         while (self.position < self.input.len) {
104             if (std.ascii.isWhitespace(self.input[self.position])) {
105                 self.position += 1;
106                 continue;
107             }
108             if (self.input[self.position] != '/' or self.position + 1 == self.input.len) return;
109             switch (self.input[self.position + 1]) {
110                 '/' => {
111                     self.position += 2;
112                     while (self.position < self.input.len and self.input[self.position] != '\n') {
113                         self.position += 1;
114                     }
115                 },
116                 '*' => {
117                     const end = std.mem.indexOfPos(
118                         u8,
119                         self.input,
120                         self.position + 2,
121                         "*/",
122                     ) orelse return error.InvalidSyntax;
123                     self.position = end + 2;
124                 },
125                 else => return,
126             }
127         }
128     }
129 };
130 
131 fn delimiter(byte: u8) bool {
132     return std.ascii.isWhitespace(byte) or switch (byte) {
133         '{', '}', '[', ']', '(', ')', '=', ';', ',', '+', '.', '!', '-', '"', '<' => true,
134         else => false,
135     };
136 }
137 
138 test "lexer separates resolved XKB punctuation and comments" {
139     var lexer = Lexer{ .input = "/* a */ key <AD01> { [ 0x61, NoSymbol ] }; // b" };
140     const expected = [_]Tag{
141         .word,
142         .key_name,
143         .left_brace,
144         .left_bracket,
145         .word,
146         .comma,
147         .word,
148         .right_bracket,
149         .right_brace,
150         .semicolon,
151         .end,
152     };
153     for (expected) |tag| try std.testing.expectEqual(tag, (try lexer.next()).tag);
154 }
155 
156 test "lexer rejects unterminated strings key names and comments" {
157     var string = Lexer{ .input = "\"bad" };
158     try std.testing.expectError(error.InvalidSyntax, string.next());
159     var key_name = Lexer{ .input = "<BAD" };
160     try std.testing.expectError(error.InvalidSyntax, key_name.next());
161     var comment = Lexer{ .input = "/* bad" };
162     try std.testing.expectError(error.InvalidSyntax, comment.next());
163 }