lib/xkb/src/compose/parser.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const xkb = @import("../root.zig");
3 const compose = @import("root.zig");
4
5 const Keysym = xkb.keysym.Keysym;
6
7 pub const DiagnosticCode = enum {
8 invalid_syntax,
9 unknown_keysym,
10 sequence_too_long,
11 output_too_long,
12 invalid_utf8,
13 };
14
15 pub const Diagnostic = struct {
16 source: []const u8,
17 line: usize,
18 code: DiagnosticCode,
19 };
20
21 pub const DiagnosticFn = *const fn (context: *anyopaque, diagnostic: Diagnostic) void;
22
23 pub const Error = std.Io.File.OpenError || std.Io.File.StatError ||
24 std.Io.File.Reader.Error || compose.Exhaustion || compose.ScratchExhaustion || error{
25 StreamTooLong,
26 InvalidEncoding,
27 TooManyErrors,
28 InvalidSyntax,
29 UnknownKeysym,
30 SequenceTooLong,
31 OutputTooLong,
32 InvalidUtf8,
33 InvalidIncludeExpansion,
34 MissingHome,
35 LocaleUnavailable,
36 InvalidFileKind,
37 IncludeDepthExceeded,
38 };
39
40 pub const Options = struct {
41 paths: compose.PathOptions,
42 diagnostic_context: ?*anyopaque = null,
43 diagnostic: ?DiagnosticFn = null,
44 };
45
46 pub fn parse(
47 storage: *compose.Storage,
48 scratch_storage: *compose.ScratchStorage,
49 input: []const u8,
50 source: []const u8,
51 options: Options,
52 include_depth: usize,
53 ) Error!void {
54 if (!std.unicode.utf8ValidateSlice(input)) return error.InvalidEncoding;
55 const content = if (std.mem.startsWith(u8, input, "\xef\xbb\xbf")) input[3..] else input;
56 if (content.len >= 2 and
57 (content[0] == 0 or content[1] == 0 or !std.ascii.isAscii(content[0])))
58 {
59 return error.InvalidEncoding;
60 }
61
62 var error_count: usize = 0;
63 var lines = std.mem.splitScalar(u8, content, '\n');
64 var line_number: usize = 0;
65 while (lines.next()) |raw_line| {
66 line_number += 1;
67 if (raw_line.len > scratch_storage.lineLimit()) {
68 return error.LineByteCapacityExceeded;
69 }
70 try parseRecovering(
71 storage,
72 scratch_storage,
73 std.mem.trimEnd(u8, raw_line, "\r"),
74 source,
75 line_number,
76 options,
77 include_depth,
78 &error_count,
79 );
80 }
81 }
82
83 pub fn parseOpenedFile(
84 storage: *compose.Storage,
85 scratch_storage: *compose.ScratchStorage,
86 file: std.Io.File,
87 file_bytes: u64,
88 source: []const u8,
89 options: Options,
90 include_depth: usize,
91 ) Error!void {
92 if (file_bytes >= compose.max_file_bytes) return error.StreamTooLong;
93 var reader = file.reader(options.paths.io, scratch_storage.fileLine(include_depth));
94 var error_count: usize = 0;
95 var line_number: usize = 0;
96 while (true) {
97 const raw_line = reader.interface.takeDelimiter('\n') catch |err| switch (err) {
98 error.ReadFailed => return reader.err.?,
99 error.StreamTooLong => return error.LineByteCapacityExceeded,
100 } orelse return;
101 if (raw_line.len > scratch_storage.lineLimit()) {
102 return error.LineByteCapacityExceeded;
103 }
104 line_number += 1;
105 if (!std.unicode.utf8ValidateSlice(raw_line)) return error.InvalidEncoding;
106 const content = if (line_number == 1 and
107 std.mem.startsWith(u8, raw_line, "\xef\xbb\xbf"))
108 raw_line[3..]
109 else
110 raw_line;
111 if (line_number == 1 and content.len >= 2 and
112 (content[0] == 0 or content[1] == 0 or !std.ascii.isAscii(content[0])))
113 {
114 return error.InvalidEncoding;
115 }
116 try parseRecovering(
117 storage,
118 scratch_storage,
119 std.mem.trimEnd(u8, content, "\r"),
120 source,
121 line_number,
122 options,
123 include_depth,
124 &error_count,
125 );
126 }
127 }
128
129 fn parseFile(
130 storage: *compose.Storage,
131 scratch_storage: *compose.ScratchStorage,
132 file_path: []const u8,
133 options: Options,
134 include_depth: usize,
135 ) Error!void {
136 var file = try std.Io.Dir.cwd().openFile(options.paths.io, file_path, .{});
137 defer file.close(options.paths.io);
138 const stat = try file.stat(options.paths.io);
139 if (stat.kind != .file) return error.InvalidFileKind;
140 try parseOpenedFile(
141 storage,
142 scratch_storage,
143 file,
144 stat.size,
145 file_path,
146 options,
147 include_depth,
148 );
149 }
150
151 fn parseRecovering(
152 storage: *compose.Storage,
153 scratch_storage: *compose.ScratchStorage,
154 raw_line: []const u8,
155 source: []const u8,
156 line_number: usize,
157 options: Options,
158 include_depth: usize,
159 error_count: *usize,
160 ) Error!void {
161 parseLine(
162 storage,
163 scratch_storage,
164 raw_line,
165 source,
166 line_number,
167 options,
168 include_depth,
169 ) catch |err| switch (err) {
170 error.InvalidSyntax => {
171 error_count.* += 1;
172 report(options, .{ .source = source, .line = line_number, .code = .invalid_syntax });
173 if (error_count.* > 10) return error.TooManyErrors;
174 },
175 error.UnknownKeysym => {
176 error_count.* += 1;
177 report(options, .{ .source = source, .line = line_number, .code = .unknown_keysym });
178 if (error_count.* > 10) return error.TooManyErrors;
179 },
180 error.SequenceTooLong => {
181 report(options, .{ .source = source, .line = line_number, .code = .sequence_too_long });
182 },
183 error.OutputTooLong => {
184 report(options, .{ .source = source, .line = line_number, .code = .output_too_long });
185 },
186 error.InvalidUtf8 => {
187 error_count.* += 1;
188 report(options, .{ .source = source, .line = line_number, .code = .invalid_utf8 });
189 if (error_count.* > 10) return error.TooManyErrors;
190 },
191 else => return err,
192 };
193 }
194
195 fn parseLine(
196 storage: *compose.Storage,
197 scratch_storage: *compose.ScratchStorage,
198 raw_line: []const u8,
199 source: []const u8,
200 line_number: usize,
201 options: Options,
202 include_depth: usize,
203 ) Error!void {
204 var cursor = Cursor{ .input = raw_line };
205 cursor.skipSpace();
206 if (cursor.done() or cursor.peek() == '#') return;
207
208 const initial = cursor.position;
209 if (cursor.identifier()) |identifier| {
210 if (std.mem.eql(u8, identifier, "include")) {
211 return parseInclude(
212 storage,
213 scratch_storage,
214 &cursor,
215 options,
216 include_depth,
217 );
218 }
219 }
220 cursor.position = initial;
221
222 var sequence: [compose.max_sequence_length]Keysym = undefined;
223 var sequence_length: usize = 0;
224 while (true) {
225 cursor.skipSpace();
226 if (cursor.take(':')) break;
227 if (cursor.done() or cursor.peek() == '#') return error.InvalidSyntax;
228
229 const opening = std.mem.indexOfScalarPos(
230 u8,
231 cursor.input,
232 cursor.position,
233 '<',
234 ) orelse return error.InvalidSyntax;
235 if (!validModifiers(cursor.input[cursor.position..opening])) {
236 return error.InvalidSyntax;
237 }
238 cursor.position = opening + 1;
239 const closing = std.mem.indexOfScalarPos(
240 u8,
241 cursor.input,
242 cursor.position,
243 '>',
244 ) orelse return error.InvalidSyntax;
245 const name = cursor.input[cursor.position..closing];
246 cursor.position = closing + 1;
247 const symbol = xkb.keysym.fromName(name) orelse return error.UnknownKeysym;
248 if (symbol == .no_symbol) return error.UnknownKeysym;
249 if (sequence_length == sequence.len) return error.SequenceTooLong;
250 sequence[sequence_length] = symbol;
251 sequence_length += 1;
252 }
253 if (sequence_length == 0) return error.InvalidSyntax;
254
255 cursor.skipSpace();
256 var text_buffer: [compose.max_output_bytes]u8 = undefined;
257 const text = if (!cursor.done() and cursor.peek() == '"')
258 try parseString(&cursor, &text_buffer)
259 else
260 null;
261 cursor.skipSpace();
262 const symbol = if (!cursor.done() and cursor.peek() != '#') blk: {
263 const name = cursor.identifier() orelse return error.InvalidSyntax;
264 const value = xkb.keysym.fromName(name) orelse return error.UnknownKeysym;
265 if (value == .no_symbol) return error.UnknownKeysym;
266 break :blk value;
267 } else null;
268 cursor.skipSpace();
269 if (!cursor.done() and cursor.peek() != '#') return error.InvalidSyntax;
270 if (text == null and symbol == null) return error.InvalidSyntax;
271
272 try storage.insert(sequence[0..sequence_length], .{
273 .text = text,
274 .symbol = symbol,
275 });
276 _ = source;
277 _ = line_number;
278 }
279
280 fn parseInclude(
281 storage: *compose.Storage,
282 scratch_storage: *compose.ScratchStorage,
283 cursor: *Cursor,
284 options: Options,
285 include_depth: usize,
286 ) Error!void {
287 cursor.skipSpace();
288 if (!cursor.take('"')) return error.InvalidSyntax;
289 const start = cursor.position;
290 while (!cursor.done() and cursor.peek() != '"') cursor.position += 1;
291 if (cursor.done()) return error.InvalidSyntax;
292 const raw_path = cursor.input[start..cursor.position];
293 cursor.position += 1;
294 cursor.skipSpace();
295 if (!cursor.done() and cursor.peek() != '#') return error.InvalidSyntax;
296 if (include_depth >= compose.max_include_depth) return error.IncludeDepthExceeded;
297
298 const file_path = try compose.expandInclude(
299 scratch_storage,
300 scratch_storage.filePath(include_depth + 1),
301 options.paths,
302 raw_path,
303 );
304 try parseFile(
305 storage,
306 scratch_storage,
307 file_path,
308 options,
309 include_depth + 1,
310 );
311 }
312
313 fn parseString(cursor: *Cursor, output: *[compose.max_output_bytes]u8) ![]const u8 {
314 std.debug.assert(cursor.take('"'));
315 var length: usize = 0;
316 while (!cursor.done() and cursor.peek() != '"') {
317 if (!cursor.take('\\')) {
318 try appendOutput(output, &length, cursor.peek());
319 cursor.position += 1;
320 continue;
321 }
322 if (cursor.done()) return error.InvalidSyntax;
323 switch (cursor.peek()) {
324 '\\', '"' => {
325 try appendOutput(output, &length, cursor.peek());
326 cursor.position += 1;
327 },
328 'x', 'X' => {
329 cursor.position += 1;
330 const value = parseEscapedInteger(cursor, 16, 2);
331 if (value) |byte| if (byte != 0) try appendOutput(output, &length, byte);
332 },
333 '0'...'7' => {
334 const value = parseEscapedInteger(cursor, 8, 4);
335 if (value) |byte| if (byte != 0) try appendOutput(output, &length, byte);
336 },
337 else => {
338 try appendOutput(output, &length, cursor.peek());
339 cursor.position += 1;
340 },
341 }
342 }
343 if (!cursor.take('"')) return error.InvalidSyntax;
344 if (!std.unicode.utf8ValidateSlice(output[0..length])) return error.InvalidUtf8;
345 return output[0..length];
346 }
347
348 fn parseEscapedInteger(cursor: *Cursor, base: u8, maximum_digits: usize) ?u8 {
349 var value: u16 = 0;
350 var digits: usize = 0;
351 while (!cursor.done() and digits < maximum_digits) {
352 const digit = std.fmt.charToDigit(cursor.peek(), base) catch break;
353 if (value > (std.math.maxInt(u8) - digit) / base) {
354 cursor.position += 1;
355 return null;
356 }
357 value = value * base + digit;
358 cursor.position += 1;
359 digits += 1;
360 }
361 if (digits == 0) return null;
362 return @intCast(value);
363 }
364
365 fn appendOutput(output: *[compose.max_output_bytes]u8, length: *usize, byte: u8) !void {
366 if (length.* == output.len) return error.OutputTooLong;
367 output[length.*] = byte;
368 length.* += 1;
369 }
370
371 fn validModifiers(input: []const u8) bool {
372 var cursor = Cursor{ .input = input };
373 cursor.skipSpace();
374 if (cursor.done()) return true;
375 if (cursor.identifier()) |identifier| {
376 if (std.mem.eql(u8, identifier, "None")) {
377 cursor.skipSpace();
378 return cursor.done();
379 }
380 cursor.position = 0;
381 }
382 _ = cursor.take('!');
383 while (true) {
384 cursor.skipSpace();
385 if (cursor.done()) return true;
386 _ = cursor.take('~');
387 cursor.skipSpace();
388 const identifier = cursor.identifier() orelse return false;
389 if (!isModifierName(identifier)) return false;
390 }
391 }
392
393 fn isModifierName(input: []const u8) bool {
394 return std.mem.eql(u8, input, "Ctrl") or
395 std.mem.eql(u8, input, "Lock") or
396 std.mem.eql(u8, input, "Caps") or
397 std.mem.eql(u8, input, "Shift") or
398 std.mem.eql(u8, input, "Alt") or
399 std.mem.eql(u8, input, "Meta");
400 }
401
402 fn report(options: Options, diagnostic: Diagnostic) void {
403 const function = options.diagnostic orelse return;
404 function(options.diagnostic_context orelse return, diagnostic);
405 }
406
407 const Cursor = struct {
408 input: []const u8,
409 position: usize = 0,
410
411 fn done(self: Cursor) bool {
412 return self.position >= self.input.len;
413 }
414
415 fn peek(self: Cursor) u8 {
416 return self.input[self.position];
417 }
418
419 fn take(self: *Cursor, expected: u8) bool {
420 if (self.done() or self.peek() != expected) return false;
421 self.position += 1;
422 return true;
423 }
424
425 fn skipSpace(self: *Cursor) void {
426 while (!self.done() and isSpace(self.peek())) {
427 self.position += 1;
428 }
429 }
430
431 fn identifier(self: *Cursor) ?[]const u8 {
432 if (self.done() or !isIdentifierStart(self.peek())) return null;
433 const start = self.position;
434 self.position += 1;
435 while (!self.done() and isIdentifierContinue(self.peek())) self.position += 1;
436 return self.input[start..self.position];
437 }
438 };
439
440 fn isIdentifierStart(byte: u8) bool {
441 return std.ascii.isAlphabetic(byte) or byte == '_';
442 }
443
444 fn isIdentifierContinue(byte: u8) bool {
445 return std.ascii.isAlphanumeric(byte) or byte == '_';
446 }
447
448 fn isSpace(byte: u8) bool {
449 return byte == ' ' or (byte >= '\t' and byte <= '\r');
450 }
451
452 test "parser accepts modifiers strings symbols escapes and line recovery" {
453 const allocator = std.testing.allocator;
454 var storage = try compose.Storage.init(allocator, .{
455 .sequence_symbols = 32,
456 .text_bytes = 64,
457 });
458 defer storage.deinit(allocator);
459 var scratch_storage = try compose.ScratchStorage.init(allocator, compose.default_scratch_limits);
460 defer scratch_storage.deinit(allocator);
461 storage.activate();
462 scratch_storage.activate();
463 try storage.acquire();
464 defer storage.reset();
465 try scratch_storage.acquire();
466 defer scratch_storage.reset();
467 try parse(
468 &storage,
469 &scratch_storage,
470 \\! Shift ~Ctrl <Multi_key> <a> : "A\x42\103" C
471 \\None <dead_acute> <space> : "'" apostrophe
472 \\<NoSuchKeysym> : "ignored"
473 \\<A> : dollar
474 ,
475 "fixture",
476 .{ .paths = .{ .locale = "C" } },
477 0,
478 );
479 const published = storage.publish();
480
481 var iterator_value = published.iterator();
482 const a = iterator_value.next().?;
483 try std.testing.expect(a.text == null);
484 try std.testing.expectEqual(@as(?Keysym, @fromBackingInt(@intCast('$'))), a.symbol);
485 const dead = iterator_value.next().?;
486 try std.testing.expectEqualStrings("'", dead.text.?);
487 const multi = iterator_value.next().?;
488 try std.testing.expectEqualStrings("ABC", multi.text.?);
489 try std.testing.expect(iterator_value.next() == null);
490 }
491
492 test "parser consumes a source BOM and matches four-digit octal escapes" {
493 const allocator = std.testing.allocator;
494 var storage = try compose.Storage.init(allocator, .{
495 .sequence_symbols = 8,
496 .text_bytes = 8,
497 });
498 defer storage.deinit(allocator);
499 var scratch_storage = try compose.ScratchStorage.init(allocator, compose.default_scratch_limits);
500 defer scratch_storage.deinit(allocator);
501 storage.activate();
502 scratch_storage.activate();
503 try storage.acquire();
504 defer storage.reset();
505 try scratch_storage.acquire();
506 defer scratch_storage.reset();
507 try parse(
508 &storage,
509 &scratch_storage,
510 "\xef\xbb\xbf<A> : \"\\0011\" A\n",
511 "fixture",
512 .{ .paths = .{ .locale = "C" } },
513 0,
514 );
515 const published = storage.publish();
516
517 var iterator_value = published.iterator();
518 const entry = iterator_value.next().?;
519 try std.testing.expectEqualStrings("\t", entry.text.?);
520 }
521
522 test "parser rejects zero-prefixed multibyte encodings" {
523 const allocator = std.testing.allocator;
524 var storage = try compose.Storage.init(allocator, .{
525 .sequence_symbols = 8,
526 .text_bytes = 8,
527 });
528 defer storage.deinit(allocator);
529 var scratch_storage = try compose.ScratchStorage.init(allocator, compose.default_scratch_limits);
530 defer scratch_storage.deinit(allocator);
531 storage.activate();
532 scratch_storage.activate();
533 try storage.acquire();
534 defer storage.reset();
535 try scratch_storage.acquire();
536 defer scratch_storage.reset();
537 try std.testing.expectError(
538 error.InvalidEncoding,
539 parse(
540 &storage,
541 &scratch_storage,
542 "<\x00A\x00>\x00 \x00:\x00 \x00X\x00",
543 "fixture",
544 .{ .paths = .{ .locale = "C" } },
545 0,
546 ),
547 );
548 }
549
550 test "parser accepts the pinned ASCII whitespace set" {
551 const allocator = std.testing.allocator;
552 var storage = try compose.Storage.init(allocator, .{
553 .sequence_symbols = 8,
554 .text_bytes = 8,
555 });
556 defer storage.deinit(allocator);
557 var scratch_storage = try compose.ScratchStorage.init(allocator, compose.default_scratch_limits);
558 defer scratch_storage.deinit(allocator);
559 storage.activate();
560 scratch_storage.activate();
561 try storage.acquire();
562 defer storage.reset();
563 try scratch_storage.acquire();
564 defer scratch_storage.reset();
565 try parse(
566 &storage,
567 &scratch_storage,
568 "\x0b<A>\x0c:\x0b\"ok\"\n",
569 "fixture",
570 .{ .paths = .{ .locale = "C" } },
571 0,
572 );
573 const published = storage.publish();
574 var iterator_value = published.iterator();
575 try std.testing.expectEqualStrings("ok", iterator_value.next().?.text.?);
576 }