lib/preserves/src/text/reader.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 //! Reads values from text, whatever the type of their embedded values. A caller needs a text reader
2 //! for its own value type, whatever type its embedded values have. The caller also needs a reader
3 //! that checks every string and symbol as UTF-8 and reads JSON-style surrogate-pair escapes. The
4 //! type of the embedded values decides how an embedded value is spelled in text, and the reader
5 //! knows only what that type tells it. Annotations can hold any value, including embedded values
6 //! the caller's type does not read.
7 //!
8 //! `#:` reads an embedded value through its type's `decodeText`, and fails with
9 //! `EmbeddedNotSupported` when the type lacks one, as `NoEmbedded` and `AnyEmbedded` both do. The
10 //! reader reads each annotation with its own private type for embedded values, checks it like any
11 //! value, then frees it, so `@a v` returns `v`. Sets and dictionaries keep the order the text lists
12 //! them in, after the repeated-item check. The reader copies every string and symbol it keeps, so
13 //! the value owns all its memory and borrows nothing from the text.
14 const std = @import("std");
15 const Allocator = std.mem.Allocator;
16
17 const preserves = @import("../root.zig");
18 const nesting = @import("nesting.zig");
19
20 const value_mod = preserves.value;
21 const integer_mod = preserves.integer_mod;
22
23 /// The errors `decode` and `readValue` return. Code that calls `decode` or `readValue` switches on
24 /// these errors. `UnexpectedEof`: the text ends before a value, inside a string, escape or byte
25 /// string, or before a closing bracket. `UnexpectedTrailingBytes`: `decode` finds anything other
26 /// than whitespace and commas after the value. `BadSyntax`: a delimiter at the start of a value, an
27 /// unknown `#` form, a dictionary key followed by something other than `:`, or a `#xd"…"` double
28 /// not closed after 16 digits. `BadEscape`: an unknown backslash escape, or a `\u` escape that
29 /// names a lone surrogate. `BadHex`: a character other than a hexadecimal digit in `\u`, `\x`,
30 /// `#x"…"` or `#xd"…"`. `BadBase64`: a `#[…]` byte string with invalid base64. `BadNumber`: a token
31 /// shaped like a double that the standard parser rejects. `InvalidUtf8`: a string, quoted symbol or
32 /// bare symbol with invalid UTF-8. `EmbeddedNotSupported`: a `#:` value outside an annotation when
33 /// the type of the embedded values lacks `decodeText`. `NonCanonicalInteger` is listed, and the
34 /// reader never returns it. `DuplicateSetElement` and `DuplicateDictionaryKey`: a set with two
35 /// equal elements, or a dictionary with two equal keys. `NestingLimitExceeded`: values nested more
36 /// than 256 levels deep.
37 pub const DecodeError = Allocator.Error || error{
38 UnexpectedEof,
39 UnexpectedTrailingBytes,
40 BadSyntax,
41 BadEscape,
42 BadHex,
43 BadBase64,
44 BadNumber,
45 InvalidUtf8,
46 EmbeddedNotSupported,
47 NonCanonicalInteger,
48 DuplicateSetElement,
49 DuplicateDictionaryKey,
50 NestingLimitExceeded,
51 };
52
53 const AnnotationEmbedded = struct {
54 value: *value_mod.Value(AnnotationEmbedded),
55
56 pub fn eql(a: AnnotationEmbedded, b: AnnotationEmbedded) bool {
57 return a.value.*.eql(b.value.*);
58 }
59
60 pub fn hash(self: AnnotationEmbedded) u64 {
61 return self.value.*.hash();
62 }
63
64 pub fn order(a: AnnotationEmbedded, b: AnnotationEmbedded) std.math.Order {
65 return a.value.*.compare(b.value.*);
66 }
67
68 pub fn deinit(self: *AnnotationEmbedded, allocator: Allocator) void {
69 self.value.deinit(allocator);
70 allocator.destroy(self.value);
71 }
72
73 pub fn clone(
74 self: AnnotationEmbedded,
75 allocator: Allocator,
76 ) Allocator.Error!AnnotationEmbedded {
77 const V = value_mod.Value(AnnotationEmbedded);
78 const cloned = try allocator.create(V);
79 errdefer allocator.destroy(cloned);
80 cloned.* = try preserves.ownership.cloneValueDeep(
81 AnnotationEmbedded,
82 allocator,
83 self.value.*,
84 );
85 return .{ .value = cloned };
86 }
87 };
88
89 fn decodeAnnotationEmbedded(
90 allocator: Allocator,
91 text: []const u8,
92 index: *usize,
93 level: nesting.Level,
94 ) DecodeError!AnnotationEmbedded {
95 const V = value_mod.Value(AnnotationEmbedded);
96 var value = try readValueAt(AnnotationEmbedded, allocator, text, index, level);
97 errdefer value.deinit(allocator);
98 const owned = try allocator.create(V);
99 owned.* = value;
100 return .{ .value = owned };
101 }
102
103 /// Returns the one value `text` holds, allocated with `allocator`. Code that reads a whole data
104 /// file calls it, so each file yields one owned value. The value's embedded values have the type
105 /// `D`. Whitespace and commas separate values. `decode` reads no comments, so `# ` fails as an
106 /// unknown `#` form. A bare token that is all digits, with an optional sign, becomes an integer of
107 /// any size, a token in decimal or exponent form becomes a double, and any other token becomes a
108 /// symbol.
109 ///
110 /// The value owns all its memory, so `text` may be freed at once, and the caller frees the value
111 /// with `deinit`. On any error the call frees everything it allocated. Its repeated-item checks
112 /// compare each element or key with every earlier one, so their cost grows with the square of the
113 /// count.
114 pub fn decode(comptime D: type, allocator: Allocator, text: []const u8) DecodeError!value_mod.Value(D) {
115 var index: usize = 0;
116 var v = try readValueAt(D, allocator, text, &index, nesting.root);
117 errdefer v.deinit(allocator);
118 skipTrailing(text, &index);
119 if (index != text.len) return error.UnexpectedTrailingBytes;
120 return v;
121 }
122
123 /// Reads one value from `text` at `index.*` and advances `index` past it. Code reading more than
124 /// one value from one text calls it once per value, for each value and the position after it. The
125 /// call leaves any text after the value unread and reports no trailing content. On
126 /// `NestingLimitExceeded`, `index` stops at the opening bracket it rejected. Its value, errors and
127 /// ownership match `decode`.
128 pub fn readValue(
129 comptime D: type,
130 allocator: Allocator,
131 text: []const u8,
132 index: *usize,
133 ) DecodeError!value_mod.Value(D) {
134 return readValueAt(D, allocator, text, index, nesting.root);
135 }
136
137 fn readValueAt(
138 comptime D: type,
139 allocator: Allocator,
140 text: []const u8,
141 index: *usize,
142 level: nesting.Level,
143 ) DecodeError!value_mod.Value(D) {
144 const value_level = try skipAnnotations(allocator, text, index, level);
145 skipWhitespaceAndCommas(text, index);
146 if (index.* >= text.len) return error.UnexpectedEof;
147 const c = text[index.*];
148 return switch (c) {
149 '"' => blk: {
150 index.* += 1;
151 const bytes = try readStringLiteral(allocator, text, index, '"');
152 if (!std.unicode.utf8ValidateSlice(bytes)) {
153 allocator.free(bytes);
154 break :blk error.InvalidUtf8;
155 }
156 break :blk .{ .string = bytes };
157 },
158 '\'' => blk: {
159 index.* += 1;
160 const bytes = try readStringLiteral(allocator, text, index, '\'');
161 if (!std.unicode.utf8ValidateSlice(bytes)) {
162 allocator.free(bytes);
163 break :blk error.InvalidUtf8;
164 }
165 break :blk .{ .symbol = bytes };
166 },
167 '<' => blk: {
168 const nested = try nesting.descend(value_level);
169 index.* += 1;
170 break :blk try readRecord(
171 D,
172 allocator,
173 text,
174 index,
175 nested,
176 );
177 },
178 '[' => blk: {
179 const nested = try nesting.descend(value_level);
180 index.* += 1;
181 break :blk try readSequence(
182 D,
183 allocator,
184 text,
185 index,
186 nested,
187 );
188 },
189 '{' => blk: {
190 const nested = try nesting.descend(value_level);
191 index.* += 1;
192 break :blk try readDictionary(
193 D,
194 allocator,
195 text,
196 index,
197 nested,
198 );
199 },
200 '#' => try readHashForm(D, allocator, text, index, value_level),
201 else => try readBareToken(D, allocator, text, index),
202 };
203 }
204
205 fn readHashForm(
206 comptime D: type,
207 allocator: Allocator,
208 text: []const u8,
209 index: *usize,
210 level: nesting.Level,
211 ) DecodeError!value_mod.Value(D) {
212 if (index.* + 1 >= text.len) return error.UnexpectedEof;
213 const next = text[index.* + 1];
214 switch (next) {
215 't' => {
216 index.* += 2;
217 return value_mod.Value(D).initBoolean(true);
218 },
219 'f' => {
220 index.* += 2;
221 return value_mod.Value(D).initBoolean(false);
222 },
223 '{' => {
224 const nested = try nesting.descend(level);
225 index.* += 2;
226 return try readSet(
227 D,
228 allocator,
229 text,
230 index,
231 nested,
232 );
233 },
234 '"' => {
235 index.* += 2;
236 return try readLiteralByteString(D, allocator, text, index);
237 },
238 '[' => {
239 index.* += 2;
240 return try readBase64ByteString(D, allocator, text, index);
241 },
242 ':' => {
243 const embedded_level = try nesting.descend(level);
244 index.* += 2;
245 if (D == AnnotationEmbedded) {
246 const d = try decodeAnnotationEmbedded(
247 allocator,
248 text,
249 index,
250 embedded_level,
251 );
252 return value_mod.Value(D).initEmbedded(d);
253 }
254 if (!@hasDecl(D, "decodeText")) return error.EmbeddedNotSupported;
255 const d = try D.decodeText(allocator, text, index);
256 return value_mod.Value(D).initEmbedded(d);
257 },
258 'x' => {
259 if (index.* + 2 >= text.len) return error.UnexpectedEof;
260 const follow = text[index.* + 2];
261 if (follow == '"') {
262 index.* += 3;
263 return try readHexByteString(D, allocator, text, index);
264 } else if (follow == 'd') {
265 if (index.* + 3 >= text.len or text[index.* + 3] != '"') return error.BadSyntax;
266 index.* += 4;
267 return try readHexDouble(D, text, index);
268 } else return error.BadSyntax;
269 },
270 else => return error.BadSyntax,
271 }
272 }
273
274 fn readRecord(
275 comptime D: type,
276 allocator: Allocator,
277 text: []const u8,
278 index: *usize,
279 level: nesting.Level,
280 ) DecodeError!value_mod.Value(D) {
281 const V = value_mod.Value(D);
282 skipWhitespaceAndCommas(text, index);
283 var label_value = try readValueAt(D, allocator, text, index, level);
284 errdefer label_value.deinit(allocator);
285
286 var fields: std.ArrayListUnmanaged(V) = .empty;
287 errdefer {
288 for (fields.items) |*f| f.deinit(allocator);
289 fields.deinit(allocator);
290 }
291
292 while (true) {
293 skipWhitespaceAndCommas(text, index);
294 if (index.* >= text.len) return error.UnexpectedEof;
295 if (text[index.*] == '>') {
296 index.* += 1;
297 break;
298 }
299 var f = try readValueAt(D, allocator, text, index, level);
300 errdefer f.deinit(allocator);
301 try fields.append(allocator, f);
302 }
303
304 const owned = try fields.toOwnedSlice(allocator);
305 errdefer {
306 for (owned) |*field| field.deinit(allocator);
307 allocator.free(owned);
308 }
309 return try V.initRecord(allocator, label_value, owned);
310 }
311
312 fn readSequence(
313 comptime D: type,
314 allocator: Allocator,
315 text: []const u8,
316 index: *usize,
317 level: nesting.Level,
318 ) DecodeError!value_mod.Value(D) {
319 const V = value_mod.Value(D);
320 var items: std.ArrayListUnmanaged(V) = .empty;
321 errdefer {
322 for (items.items) |*it| it.deinit(allocator);
323 items.deinit(allocator);
324 }
325 while (true) {
326 skipWhitespaceAndCommas(text, index);
327 if (index.* >= text.len) return error.UnexpectedEof;
328 if (text[index.*] == ']') {
329 index.* += 1;
330 break;
331 }
332 var it = try readValueAt(D, allocator, text, index, level);
333 errdefer it.deinit(allocator);
334 try items.append(allocator, it);
335 }
336 return .{ .sequence = try items.toOwnedSlice(allocator) };
337 }
338
339 fn readSet(
340 comptime D: type,
341 allocator: Allocator,
342 text: []const u8,
343 index: *usize,
344 level: nesting.Level,
345 ) DecodeError!value_mod.Value(D) {
346 const V = value_mod.Value(D);
347 var items: std.ArrayListUnmanaged(V) = .empty;
348 errdefer {
349 for (items.items) |*it| it.deinit(allocator);
350 items.deinit(allocator);
351 }
352 while (true) {
353 skipWhitespaceAndCommas(text, index);
354 if (index.* >= text.len) return error.UnexpectedEof;
355 if (text[index.*] == '}') {
356 index.* += 1;
357 break;
358 }
359 var it = try readValueAt(D, allocator, text, index, level);
360 errdefer it.deinit(allocator);
361 if (V.setContainsElement(items.items, it)) return error.DuplicateSetElement;
362 try items.append(allocator, it);
363 }
364 return .{ .set = try items.toOwnedSlice(allocator) };
365 }
366
367 fn readDictionary(
368 comptime D: type,
369 allocator: Allocator,
370 text: []const u8,
371 index: *usize,
372 level: nesting.Level,
373 ) DecodeError!value_mod.Value(D) {
374 const V = value_mod.Value(D);
375 var entries: std.ArrayListUnmanaged(V.DictionaryEntry) = .empty;
376 errdefer {
377 for (entries.items) |*e| {
378 e.key.deinit(allocator);
379 e.value.deinit(allocator);
380 }
381 entries.deinit(allocator);
382 }
383 while (true) {
384 skipWhitespaceAndCommas(text, index);
385 if (index.* >= text.len) return error.UnexpectedEof;
386 if (text[index.*] == '}') {
387 index.* += 1;
388 break;
389 }
390 var k = try readValueAt(D, allocator, text, index, level);
391 errdefer k.deinit(allocator);
392 skipWhitespaceAndCommas(text, index);
393 if (index.* >= text.len or text[index.*] != ':') return error.BadSyntax;
394 index.* += 1;
395 skipWhitespaceAndCommas(text, index);
396 var v = try readValueAt(D, allocator, text, index, level);
397 errdefer v.deinit(allocator);
398 if (V.dictionaryContainsKey(entries.items, k)) {
399 return error.DuplicateDictionaryKey;
400 }
401 try entries.append(allocator, .{ .key = k, .value = v });
402 }
403 return .{ .dictionary = try entries.toOwnedSlice(allocator) };
404 }
405
406 fn readStringLiteral(
407 allocator: Allocator,
408 text: []const u8,
409 index: *usize,
410 terminator: u8,
411 ) DecodeError![]u8 {
412 var buf: std.ArrayListUnmanaged(u8) = .empty;
413 errdefer buf.deinit(allocator);
414
415 while (true) {
416 if (index.* >= text.len) return error.UnexpectedEof;
417 const c = text[index.*];
418 index.* += 1;
419 if (c == terminator) break;
420 if (c == '\\') {
421 if (index.* >= text.len) return error.UnexpectedEof;
422 const esc = text[index.*];
423 index.* += 1;
424 switch (esc) {
425 '\\' => try buf.append(allocator, '\\'),
426 '/' => try buf.append(allocator, '/'),
427 'b' => try buf.append(allocator, 0x08),
428 'f' => try buf.append(allocator, 0x0c),
429 'n' => try buf.append(allocator, 0x0a),
430 'r' => try buf.append(allocator, 0x0d),
431 't' => try buf.append(allocator, 0x09),
432 'u' => try readUnicodeEscape(allocator, &buf, text, index),
433 else => {
434 if (esc == terminator) {
435 try buf.append(allocator, esc);
436 } else return error.BadEscape;
437 },
438 }
439 } else {
440 try buf.append(allocator, c);
441 }
442 }
443
444 return try buf.toOwnedSlice(allocator);
445 }
446
447 fn readUnicodeEscape(
448 allocator: Allocator,
449 buf: *std.ArrayListUnmanaged(u8),
450 text: []const u8,
451 index: *usize,
452 ) DecodeError!void {
453 const n1 = try readHex4(text, index);
454 var codepoint: u32 = n1;
455 if (n1 >= 0xD800 and n1 <= 0xDBFF) {
456 if (index.* + 2 > text.len or text[index.*] != '\\' or text[index.* + 1] != 'u') {
457 return error.BadEscape;
458 }
459 index.* += 2;
460 const n2 = try readHex4(text, index);
461 if (n2 < 0xDC00 or n2 > 0xDFFF) return error.BadEscape;
462 codepoint = ((n1 - 0xD800) << 10) + (n2 - 0xDC00) + 0x10000;
463 } else if (n1 >= 0xDC00 and n1 <= 0xDFFF) {
464 return error.BadEscape;
465 }
466 var utf8: [4]u8 = undefined;
467 const written = std.unicode.utf8Encode(@intCast(codepoint), &utf8) catch return error.BadEscape;
468 try buf.appendSlice(allocator, utf8[0..written]);
469 }
470
471 fn readHex4(text: []const u8, index: *usize) DecodeError!u32 {
472 if (index.* + 4 > text.len) return error.UnexpectedEof;
473 var v: u32 = 0;
474 var i: usize = 0;
475 while (i < 4) : (i += 1) {
476 const d = hexDigit(text[index.*]) orelse return error.BadHex;
477 v = (v << 4) | d;
478 index.* += 1;
479 }
480 return v;
481 }
482
483 fn readHexByteString(
484 comptime D: type,
485 allocator: Allocator,
486 text: []const u8,
487 index: *usize,
488 ) DecodeError!value_mod.Value(D) {
489 var buf: std.ArrayListUnmanaged(u8) = .empty;
490 errdefer buf.deinit(allocator);
491 while (true) {
492 skipWhitespaceOnly(text, index);
493 if (index.* >= text.len) return error.UnexpectedEof;
494 const c = text[index.*];
495 if (c == '"') {
496 index.* += 1;
497 break;
498 }
499 index.* += 1;
500 if (index.* >= text.len) return error.UnexpectedEof;
501 const c2 = text[index.*];
502 index.* += 1;
503 const h1 = hexDigit(c) orelse return error.BadHex;
504 const h2 = hexDigit(c2) orelse return error.BadHex;
505 try buf.append(allocator, @intCast((h1 << 4) | h2));
506 }
507 return .{ .byte_string = try buf.toOwnedSlice(allocator) };
508 }
509
510 fn readHexDouble(
511 comptime D: type,
512 text: []const u8,
513 index: *usize,
514 ) DecodeError!value_mod.Value(D) {
515 var bytes: [8]u8 = undefined;
516 var filled: usize = 0;
517 while (filled < 8) {
518 skipWhitespaceOnly(text, index);
519 if (index.* + 1 >= text.len) return error.UnexpectedEof;
520 const c1 = text[index.*];
521 const c2 = text[index.* + 1];
522 const h1 = hexDigit(c1) orelse return error.BadHex;
523 const h2 = hexDigit(c2) orelse return error.BadHex;
524 bytes[filled] = @intCast((h1 << 4) | h2);
525 filled += 1;
526 index.* += 2;
527 }
528 skipWhitespaceOnly(text, index);
529 if (index.* >= text.len or text[index.*] != '"') return error.BadSyntax;
530 index.* += 1;
531 const bits = std.mem.readInt(u64, &bytes, .big);
532 return value_mod.Value(D).initDouble(@bitCast(bits));
533 }
534
535 fn readLiteralByteString(
536 comptime D: type,
537 allocator: Allocator,
538 text: []const u8,
539 index: *usize,
540 ) DecodeError!value_mod.Value(D) {
541 var buf: std.ArrayListUnmanaged(u8) = .empty;
542 errdefer buf.deinit(allocator);
543 while (true) {
544 if (index.* >= text.len) return error.UnexpectedEof;
545 const c = text[index.*];
546 index.* += 1;
547 if (c == '"') break;
548 if (c == '\\') {
549 if (index.* >= text.len) return error.UnexpectedEof;
550 const esc = text[index.*];
551 index.* += 1;
552 switch (esc) {
553 '\\' => try buf.append(allocator, '\\'),
554 '"' => try buf.append(allocator, '"'),
555 '/' => try buf.append(allocator, '/'),
556 'b' => try buf.append(allocator, 0x08),
557 'f' => try buf.append(allocator, 0x0c),
558 'n' => try buf.append(allocator, 0x0a),
559 'r' => try buf.append(allocator, 0x0d),
560 't' => try buf.append(allocator, 0x09),
561 'x' => {
562 if (index.* + 2 > text.len) return error.UnexpectedEof;
563 const h1 = hexDigit(text[index.*]) orelse return error.BadHex;
564 const h2 = hexDigit(text[index.* + 1]) orelse return error.BadHex;
565 index.* += 2;
566 try buf.append(allocator, @intCast((h1 << 4) | h2));
567 },
568 else => return error.BadEscape,
569 }
570 } else {
571 try buf.append(allocator, c);
572 }
573 }
574 return .{ .byte_string = try buf.toOwnedSlice(allocator) };
575 }
576
577 fn readBase64ByteString(
578 comptime D: type,
579 allocator: Allocator,
580 text: []const u8,
581 index: *usize,
582 ) DecodeError!value_mod.Value(D) {
583 var enc_buf: std.ArrayListUnmanaged(u8) = .empty;
584 defer enc_buf.deinit(allocator);
585 while (true) {
586 skipWhitespaceOnly(text, index);
587 if (index.* >= text.len) return error.UnexpectedEof;
588 var c = text[index.*];
589 if (c == ']') {
590 index.* += 1;
591 break;
592 }
593 index.* += 1;
594 if (c == '=') continue;
595 if (c == '-') c = '+';
596 if (c == '_') c = '/';
597 try enc_buf.append(allocator, c);
598 }
599
600 const decoder = std.base64.standard_no_pad.Decoder;
601 const expected = decoder.calcSizeForSlice(enc_buf.items) catch return error.BadBase64;
602 const dst = try allocator.alloc(u8, expected);
603 errdefer allocator.free(dst);
604 decoder.decode(dst, enc_buf.items) catch {
605 return error.BadBase64;
606 };
607 return .{ .byte_string = dst };
608 }
609
610 fn readBareToken(
611 comptime D: type,
612 allocator: Allocator,
613 text: []const u8,
614 index: *usize,
615 ) DecodeError!value_mod.Value(D) {
616 const start = index.*;
617 while (index.* < text.len and !isDelimiter(text[index.*])) {
618 index.* += 1;
619 }
620 const tok = text[start..index.*];
621 if (tok.len == 0) return error.BadSyntax;
622 if (looksLikeFloat(tok)) {
623 const f = std.fmt.parseFloat(f64, tok) catch return error.BadNumber;
624 return value_mod.Value(D).initDouble(f);
625 }
626 if (looksLikeInteger(tok)) {
627 const si = try parseDecimalIntoSignedInteger(allocator, tok);
628 return value_mod.Value(D).initSignedInteger(si);
629 }
630 if (!std.unicode.utf8ValidateSlice(tok)) return error.InvalidUtf8;
631 const owned = try allocator.dupe(u8, tok);
632 return .{ .symbol = owned };
633 }
634
635 fn isDelimiter(c: u8) bool {
636 return switch (c) {
637 ' ', '\t', '\r', '\n', ',', '(', ')', '{', '}', '[', ']', '<', '>', '"', '\'', ';', '@', '#', ':' => true,
638 else => false,
639 };
640 }
641
642 fn skipWhitespaceOnly(text: []const u8, index: *usize) void {
643 while (index.* < text.len) {
644 switch (text[index.*]) {
645 ' ', '\t', '\r', '\n' => index.* += 1,
646 else => return,
647 }
648 }
649 }
650
651 fn skipWhitespaceAndCommas(text: []const u8, index: *usize) void {
652 while (index.* < text.len) {
653 switch (text[index.*]) {
654 ' ', '\t', '\r', '\n', ',' => index.* += 1,
655 else => return,
656 }
657 }
658 }
659
660 fn skipTrailing(text: []const u8, index: *usize) void {
661 skipWhitespaceAndCommas(text, index);
662 }
663
664 fn skipAnnotations(
665 allocator: Allocator,
666 text: []const u8,
667 index: *usize,
668 level: nesting.Level,
669 ) DecodeError!nesting.Level {
670 var value_level = level;
671 while (true) {
672 skipWhitespaceAndCommas(text, index);
673 if (index.* >= text.len or text[index.*] != '@') return value_level;
674 const nested = try nesting.descend(value_level);
675 index.* += 1;
676 value_level = nested;
677 var annotation = try readValueAt(
678 AnnotationEmbedded,
679 allocator,
680 text,
681 index,
682 value_level,
683 );
684 annotation.deinit(allocator);
685 }
686 }
687
688 fn hexDigit(c: u8) ?u32 {
689 return switch (c) {
690 '0'...'9' => @as(u32, c - '0'),
691 'a'...'f' => @as(u32, c - 'a' + 10),
692 'A'...'F' => @as(u32, c - 'A' + 10),
693 else => null,
694 };
695 }
696
697 /// The predicate accepts an input consisting of an optional `+` or `-` followed by one or more
698 /// ASCII decimal digits. It rejects an empty input, a sign alone, and underscores. The public
699 /// `parse` and `decode` readers call this predicate before decimal integer conversion. The text
700 /// writer quotes a symbol that the predicate accepts, because a matching symbol takes the quoted
701 /// branch when `looksLikeNumber` recognizes it.
702 pub fn looksLikeInteger(s: []const u8) bool {
703 if (s.len == 0) return false;
704 var i: usize = 0;
705 if (s[i] == '-' or s[i] == '+') i += 1;
706 if (i >= s.len) return false;
707 const digit_start = i;
708 while (i < s.len) : (i += 1) {
709 if (s[i] < '0' or s[i] > '9') return false;
710 }
711 return i > digit_start;
712 }
713
714 /// Accepts an optional `+` or `-`, one or more ASCII decimal digits, and a fraction or exponent or
715 /// both, so `.5` fails. A dot requires at least one following digit, so `5.` fails. An `e` or `E`
716 /// exponent accepts an optional sign and requires at least one digit. The predicate consumes the
717 /// whole token, so `1_2e3` fails.
718 ///
719 /// The public `parse` reader calls this predicate before trying `std.fmt.parseFloat`, then falls
720 /// through to a symbol if conversion fails. The public `decode` reader calls it before float
721 /// conversion and reports `BadNumber` if conversion fails.
722 ///
723 /// The text writer quotes a symbol that this predicate accepts, because a matching symbol takes the
724 /// quoted branch when `looksLikeNumber` recognizes it.
725 pub fn looksLikeFloat(s: []const u8) bool {
726 if (s.len == 0) return false;
727 var i: usize = 0;
728 if (s[i] == '-' or s[i] == '+') i += 1;
729 if (i >= s.len or s[i] < '0' or s[i] > '9') return false;
730 while (i < s.len and s[i] >= '0' and s[i] <= '9') i += 1;
731 var has_decimal_part = false;
732 if (i < s.len and s[i] == '.') {
733 i += 1;
734 const frac_start = i;
735 while (i < s.len and s[i] >= '0' and s[i] <= '9') i += 1;
736 if (i > frac_start) has_decimal_part = true else return false;
737 }
738 var has_exp = false;
739 if (i < s.len and (s[i] == 'e' or s[i] == 'E')) {
740 i += 1;
741 if (i < s.len and (s[i] == '-' or s[i] == '+')) i += 1;
742 const exp_start = i;
743 while (i < s.len and s[i] >= '0' and s[i] <= '9') i += 1;
744 if (i > exp_start) has_exp = true else return false;
745 }
746 return i == s.len and (has_decimal_part or has_exp);
747 }
748
749 /// Returns the integer the decimal string `s` spells, with an optional leading `+` or `-`. `parse`
750 /// and `decode` call it only for tokens accepted by `looksLikeInteger`, which permits an optional
751 /// sign followed by digits. A value that fits 128 bits signed or unsigned is stored inline, and a
752 /// wider one as big-endian two's-complement bytes allocated with `allocator`. Inside the 128-bit
753 /// range, underscores between digits are skipped, so `1_000` returns 1000. The call returns
754 /// `error.BadNumber` for an empty string, a sign alone, or a wide value holding a character other
755 /// than a digit.
756 pub fn parseDecimalIntoSignedInteger(allocator: Allocator, s: []const u8) DecodeError!integer_mod.SignedInteger {
757 if (s.len == 0) return error.BadNumber;
758
759 if (std.fmt.parseInt(i128, s, 10)) |v| {
760 return integer_mod.SignedInteger.fromI128(v);
761 } else |_| {}
762 if (std.fmt.parseInt(u128, s, 10)) |v| {
763 return integer_mod.SignedInteger.fromU128(v);
764 } else |_| {}
765 return try parseDecimalIntoBig(allocator, s);
766 }
767
768 fn parseDecimalIntoBig(allocator: Allocator, s: []const u8) DecodeError!integer_mod.SignedInteger {
769 var i: usize = 0;
770 var is_negative = false;
771 if (s[i] == '-') {
772 is_negative = true;
773 i += 1;
774 } else if (s[i] == '+') i += 1;
775 if (i >= s.len) return error.BadNumber;
776
777 var mag: std.ArrayListUnmanaged(u8) = .empty;
778 defer mag.deinit(allocator);
779 try mag.append(allocator, 0);
780
781 while (i < s.len) : (i += 1) {
782 const d = s[i];
783 if (d < '0' or d > '9') return error.BadNumber;
784 const digit: u8 = @intCast(d - '0');
785 var carry: u16 = digit;
786 var j: usize = mag.items.len;
787 while (j > 0) {
788 j -= 1;
789 const product = @as(u16, mag.items[j]) * 10 + carry;
790 mag.items[j] = @intCast(product & 0xff);
791 carry = product >> 8;
792 }
793 while (carry > 0) {
794 try mag.insert(allocator, 0, @intCast(carry & 0xff));
795 carry >>= 8;
796 }
797 }
798
799 var start: usize = 0;
800 while (start < mag.items.len - 1 and mag.items[start] == 0) start += 1;
801 const stripped = mag.items[start..];
802
803 if (is_negative) {
804 const buf = try allocator.alloc(u8, stripped.len + 1);
805 defer allocator.free(buf);
806 buf[0] = 0x00;
807 @memcpy(buf[1..], stripped);
808 var k: usize = buf.len;
809 var carry: u16 = 1;
810 while (k > 0) {
811 k -= 1;
812 const inv: u16 = @as(u16, ~buf[k]) & 0xff;
813 const sum = inv + carry;
814 buf[k] = @intCast(sum & 0xff);
815 carry = sum >> 8;
816 }
817 const canonical = try canonicalizeBytes(allocator, buf);
818 defer allocator.free(canonical);
819 return try integer_mod.SignedInteger.fromCanonicalBytes(allocator, canonical);
820 } else {
821 if (stripped.len > 0 and (stripped[0] & 0x80) != 0) {
822 const buf = try allocator.alloc(u8, stripped.len + 1);
823 defer allocator.free(buf);
824 buf[0] = 0x00;
825 @memcpy(buf[1..], stripped);
826 return try integer_mod.SignedInteger.fromCanonicalBytes(allocator, buf);
827 }
828 return try integer_mod.SignedInteger.fromCanonicalBytes(allocator, stripped);
829 }
830 }
831
832 fn canonicalizeBytes(allocator: Allocator, bytes: []const u8) DecodeError![]const u8 {
833 var start: usize = 0;
834 while (start + 1 < bytes.len) {
835 const first = bytes[start];
836 const second = bytes[start + 1];
837 if (first == 0xff and (second & 0x80) != 0) {
838 start += 1;
839 continue;
840 }
841 if (first == 0x00 and (second & 0x80) == 0) {
842 start += 1;
843 continue;
844 }
845 break;
846 }
847 return try allocator.dupe(u8, bytes[start..]);
848 }
849
850 fn expectDuplicateWithAllocator(
851 allocator: Allocator,
852 text: []const u8,
853 expected: DecodeError,
854 ) !void {
855 const NE = preserves.domain.NoEmbedded;
856 if (decode(NE, allocator, text)) |decoded| {
857 var value = decoded;
858 value.deinit(allocator);
859 return error.ExpectedDuplicate;
860 } else |err| {
861 if (err == error.OutOfMemory) return err;
862 if (err == expected) return;
863 return err;
864 }
865 }
866
867 fn checkDuplicateAllocationFailures(allocator: Allocator) !void {
868 try expectDuplicateWithAllocator(
869 allocator,
870 "#{<\"label\" 'quoted'> <\"label\" 'quoted'>}",
871 error.DuplicateSetElement,
872 );
873 try expectDuplicateWithAllocator(
874 allocator,
875 "{\"key\": <\"label\" 'first'>, \"key\": <\"label\" 'second'>}",
876 error.DuplicateDictionaryKey,
877 );
878 try expectDuplicateWithAllocator(
879 allocator,
880 "@#:#{[\"owned\"] [\"owned\"]} 3",
881 error.DuplicateSetElement,
882 );
883 }
884
885 test "parse integer" {
886 const allocator = std.testing.allocator;
887 const NE = preserves.domain.NoEmbedded;
888 var v = try decode(NE, allocator, "42");
889 defer v.deinit(allocator);
890 try std.testing.expectEqual(@as(i128, 42), try v.signed_integer.toI128());
891 }
892
893 test "parse negative big integer" {
894 const allocator = std.testing.allocator;
895 const NE = preserves.domain.NoEmbedded;
896 var v = try decode(NE, allocator, "-170141183460469231731687303715884105729");
897 defer v.deinit(allocator);
898 try std.testing.expectEqual(integer_mod.Tier.big, @as(integer_mod.Tier, v.signed_integer.repr));
899 }
900
901 test "parse sequence with commas" {
902 const allocator = std.testing.allocator;
903 const NE = preserves.domain.NoEmbedded;
904 var v = try decode(NE, allocator, "[1, 2, 3]");
905 defer v.deinit(allocator);
906 try std.testing.expectEqual(@as(usize, 3), v.sequence.len);
907 }
908
909 test "parse sequence preserves duplicate values" {
910 const allocator = std.testing.allocator;
911 const NE = preserves.domain.NoEmbedded;
912 var v = try decode(NE, allocator, "[\"same\", \"same\"]");
913 defer v.deinit(allocator);
914
915 try std.testing.expectEqual(@as(usize, 2), v.sequence.len);
916 try std.testing.expect(v.sequence[0].eql(v.sequence[1]));
917 }
918
919 test "parse quoted symbol" {
920 const allocator = std.testing.allocator;
921 const NE = preserves.domain.NoEmbedded;
922 var v = try decode(NE, allocator, "'hello world'");
923 defer v.deinit(allocator);
924 try std.testing.expectEqualStrings("hello world", v.symbol);
925 }
926
927 test "parse rejects duplicate set elements and releases owned values" {
928 const allocator = std.testing.allocator;
929 const NE = preserves.domain.NoEmbedded;
930
931 try std.testing.expectError(
932 error.DuplicateSetElement,
933 decode(NE, allocator, "#{[\"owned\"] [\"owned\"]}"),
934 );
935 }
936
937 test "parse rejects duplicate dictionary keys and releases owned entries" {
938 const allocator = std.testing.allocator;
939 const NE = preserves.domain.NoEmbedded;
940
941 try std.testing.expectError(
942 error.DuplicateDictionaryKey,
943 decode(NE, allocator, "{\"key\": [\"first\"], \"key\": [\"second\"]}"),
944 );
945 }
946
947 test "parse duplicate errors release every allocation failure path" {
948 try std.testing.checkAllAllocationFailures(
949 std.testing.allocator,
950 checkDuplicateAllocationFailures,
951 .{},
952 );
953 }
954
955 test "parse skips structurally nested annotations" {
956 const allocator = std.testing.allocator;
957 const NE = preserves.domain.NoEmbedded;
958 const cases = [_][]const u8{
959 "@[[1] 2] 3",
960 "@{a: {b: c}} 3",
961 "@<r <s>> 3",
962 "@#{[1] {a: 2}} 3",
963 "@{a: @b c} 3",
964 "@\"closing ] } >\" 3",
965 "@#xd\"3ff0000000000000\" 3",
966 "@#[AQI=] 3",
967 };
968 for (cases) |text| {
969 var value = try decode(NE, allocator, text);
970 defer value.deinit(allocator);
971 try std.testing.expectEqual(@as(i128, 3), try value.signed_integer.toI128());
972 }
973 }
974
975 test "parse validates discarded annotations" {
976 const allocator = std.testing.allocator;
977 const NE = preserves.domain.NoEmbedded;
978
979 try std.testing.expectError(
980 error.DuplicateSetElement,
981 decode(NE, allocator, "@#{1 1} 2"),
982 );
983 try std.testing.expectError(
984 error.DuplicateDictionaryKey,
985 decode(NE, allocator, "@{a: 1 a: 2} 3"),
986 );
987 try std.testing.expectError(error.BadSyntax, decode(NE, allocator, "@{a} 3"));
988 try std.testing.expectError(error.BadEscape, decode(NE, allocator, "@\"\\q\" 3"));
989 try std.testing.expectError(error.BadHex, decode(NE, allocator, "@#x\"zz\" 3"));
990 try std.testing.expectError(error.BadBase64, decode(NE, allocator, "@#[%%%] 3"));
991 }
992
993 test "discarded annotations do not enter the application embedded domain" {
994 const allocator = std.testing.allocator;
995 const NE = preserves.domain.NoEmbedded;
996
997 var value = try decode(NE, allocator, "@#:foo 3");
998 defer value.deinit(allocator);
999 try std.testing.expectEqual(@as(i128, 3), try value.signed_integer.toI128());
1000 try std.testing.expectError(
1001 error.DuplicateSetElement,
1002 decode(NE, allocator, "@#:#{1 1} 3"),
1003 );
1004 }