lib/pdf/src/cmap.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2
3 const object = @import("object.zig");
4
5 pub const ParseError = error{
6 BadCMap,
7 OutOfMemory,
8 };
9
10 pub const Space = struct {
11 width: u8,
12 low: u32,
13 high: u32,
14 };
15
16 const Code = struct {
17 width: u8,
18 value: u32,
19 };
20
21 const SteppedMapping = struct {
22 prefix: []const u8,
23 base: u21,
24 };
25
26 const MappedPayload = union(enum) {
27 single: []const u8,
28 stepped: SteppedMapping,
29 listed: []const []const u8,
30 };
31
32 pub const Mapped = struct {
33 width: u8,
34 low: u32,
35 high: u32,
36 payload: MappedPayload,
37 };
38
39 pub const Map = struct {
40 spaces: []const Space,
41 ranges: []const Mapped,
42
43 pub fn appendDecoded(self: Map, allocator: std.mem.Allocator, out: *std.ArrayList(u8), bytes: []const u8) error{OutOfMemory}!void {
44 var pos: usize = 0;
45 for (bytes) |_| {
46 if (pos >= bytes.len) break;
47 const code = self.takeCode(bytes, &pos) orelse {
48 pos += 1;
49 try appendReplacement(allocator, out);
50 continue;
51 };
52 try self.appendCode(allocator, out, code);
53 }
54 }
55
56 fn takeCode(self: Map, bytes: []const u8, pos: *usize) ?Code {
57 var width: u8 = 1;
58 while (width <= 4) : (width += 1) {
59 if (pos.* + width > bytes.len) return null;
60 const value = readCode(bytes[pos.* .. pos.* + width]);
61 if (self.spaceContains(width, value)) {
62 pos.* += width;
63 return .{ .width = width, .value = value };
64 }
65 }
66 return null;
67 }
68
69 fn spaceContains(self: Map, width: u8, value: u32) bool {
70 for (self.spaces) |space| {
71 if (space.width == width and value >= space.low and value <= space.high) return true;
72 }
73 return false;
74 }
75
76 fn appendCode(self: Map, allocator: std.mem.Allocator, out: *std.ArrayList(u8), code: Code) error{OutOfMemory}!void {
77 for (self.ranges) |range| {
78 if (range.width != code.width or code.value < range.low or code.value > range.high) continue;
79 const delta = code.value - range.low;
80 switch (range.payload) {
81 .single => |text| {
82 if (delta != 0) break;
83 out.appendSlice(allocator, text) catch return error.OutOfMemory;
84 return;
85 },
86 .stepped => |stepped| {
87 out.appendSlice(allocator, stepped.prefix) catch return error.OutOfMemory;
88 const scalar = @as(u32, stepped.base) + delta;
89 try appendScalar(allocator, out, scalar);
90 return;
91 },
92 .listed => |texts| {
93 if (delta >= texts.len) break;
94 out.appendSlice(allocator, texts[delta]) catch return error.OutOfMemory;
95 return;
96 },
97 }
98 }
99 try appendReplacement(allocator, out);
100 }
101 };
102
103 fn readCode(bytes: []const u8) u32 {
104 var value: u32 = 0;
105 for (bytes) |byte| value = (value << 8) | byte;
106 return value;
107 }
108
109 fn appendScalar(allocator: std.mem.Allocator, out: *std.ArrayList(u8), scalar: u32) error{OutOfMemory}!void {
110 const valid = scalar <= 0x10FFFF and !(scalar >= 0xD800 and scalar <= 0xDFFF);
111 if (!valid) return appendReplacement(allocator, out);
112 var buffer: [4]u8 = undefined;
113 const len = std.unicode.utf8Encode(@intCast(scalar), &buffer) catch return appendReplacement(allocator, out);
114 out.appendSlice(allocator, buffer[0..len]) catch return error.OutOfMemory;
115 }
116
117 fn appendReplacement(allocator: std.mem.Allocator, out: *std.ArrayList(u8)) error{OutOfMemory}!void {
118 out.appendSlice(allocator, "\u{FFFD}") catch return error.OutOfMemory;
119 }
120
121 pub fn parseAlloc(arena: std.mem.Allocator, bytes: []const u8) ParseError!Map {
122 var parser = object.Parser.init(bytes, 0);
123 var spaces: std.ArrayList(Space) = .empty;
124 var ranges: std.ArrayList(Mapped) = .empty;
125 for (bytes) |_| {
126 parser.skipWhitespace();
127 if (parser.pos >= bytes.len) break;
128 const byte = bytes[parser.pos];
129 if (byte == '<' or byte == '[' or byte == '(' or byte == '/' or std.ascii.isDigit(byte) or byte == '+' or byte == '-' or byte == '.') {
130 _ = parser.parseValue(arena) catch {
131 parser.pos += 1;
132 };
133 continue;
134 }
135 const start = parser.pos;
136 while (parser.pos < bytes.len and !object.delimiterOrWhitespace(bytes[parser.pos])) parser.pos += 1;
137 if (parser.pos == start) {
138 parser.pos += 1;
139 continue;
140 }
141 const keyword = bytes[start..parser.pos];
142 if (std.mem.eql(u8, keyword, "begincodespacerange")) {
143 try parseSpaces(arena, &parser, &spaces);
144 } else if (std.mem.eql(u8, keyword, "beginbfchar")) {
145 try parseChars(arena, &parser, &ranges);
146 } else if (std.mem.eql(u8, keyword, "beginbfrange")) {
147 try parseRanges(arena, &parser, &ranges);
148 }
149 }
150 if (ranges.items.len == 0) return error.BadCMap;
151 if (spaces.items.len == 0) try deriveSpaces(arena, ranges.items, &spaces);
152 return .{
153 .spaces = spaces.toOwnedSlice(arena) catch return error.OutOfMemory,
154 .ranges = ranges.toOwnedSlice(arena) catch return error.OutOfMemory,
155 };
156 }
157
158 fn deriveSpaces(arena: std.mem.Allocator, ranges: []const Mapped, spaces: *std.ArrayList(Space)) ParseError!void {
159 var widths = @as([5]bool, @splat(false));
160 for (ranges) |range| widths[range.width] = true;
161 for (widths, 0..) |present, width| {
162 if (!present or width == 0) continue;
163 const high: u32 = if (width >= 4) std.math.maxInt(u32) else (@as(u32, 1) << @intCast(width * 8)) - 1;
164 spaces.append(arena, .{ .width = @intCast(width), .low = 0, .high = high }) catch return error.OutOfMemory;
165 }
166 }
167
168 const entry_cap = 1 << 16;
169
170 fn parseSpaces(arena: std.mem.Allocator, parser: *object.Parser, spaces: *std.ArrayList(Space)) ParseError!void {
171 for (parser.bytes[parser.pos..]) |_| {
172 if (spaces.items.len >= entry_cap) return;
173 const low = nextHexToken(arena, parser, "endcodespacerange") orelse return;
174 const high = nextHexToken(arena, parser, "endcodespacerange") orelse return;
175 const low_code = hexCode(low) orelse continue;
176 const high_code = hexCode(high) orelse continue;
177 if (low_code.width != high_code.width) continue;
178 spaces.append(arena, .{ .width = low_code.width, .low = low_code.value, .high = high_code.value }) catch return error.OutOfMemory;
179 }
180 }
181
182 fn parseChars(arena: std.mem.Allocator, parser: *object.Parser, ranges: *std.ArrayList(Mapped)) ParseError!void {
183 for (parser.bytes[parser.pos..]) |_| {
184 if (ranges.items.len >= entry_cap) return;
185 const src = nextHexToken(arena, parser, "endbfchar") orelse return;
186 const dst = nextHexToken(arena, parser, "endbfchar") orelse return;
187 const src_code = hexCode(src) orelse continue;
188 const text = utf8FromHexUtf16Alloc(arena, dst) orelse continue;
189 ranges.append(arena, .{
190 .width = src_code.width,
191 .low = src_code.value,
192 .high = src_code.value,
193 .payload = .{ .single = text },
194 }) catch return error.OutOfMemory;
195 }
196 }
197
198 fn parseRanges(arena: std.mem.Allocator, parser: *object.Parser, ranges: *std.ArrayList(Mapped)) ParseError!void {
199 for (parser.bytes[parser.pos..]) |_| {
200 if (ranges.items.len >= entry_cap) return;
201 const low = nextHexToken(arena, parser, "endbfrange") orelse return;
202 const high = nextHexToken(arena, parser, "endbfrange") orelse return;
203 parser.skipWhitespace();
204 if (parser.pos >= parser.bytes.len) return;
205 const low_code = hexCode(low) orelse return;
206 const high_code = hexCode(high) orelse return;
207 if (parser.bytes[parser.pos] == '[') {
208 const value = parser.parseValue(arena) catch return;
209 const items = switch (value) {
210 .array => |array| array,
211 else => continue,
212 };
213 if (low_code.width != high_code.width or high_code.value < low_code.value) continue;
214 const texts = arena.alloc([]const u8, items.len) catch return error.OutOfMemory;
215 for (items, 0..) |item, index| {
216 texts[index] = switch (item) {
217 .string => |string| utf8FromHexUtf16Alloc(arena, string) orelse "\u{FFFD}",
218 else => "\u{FFFD}",
219 };
220 }
221 ranges.append(arena, .{
222 .width = low_code.width,
223 .low = low_code.value,
224 .high = high_code.value,
225 .payload = .{ .listed = texts },
226 }) catch return error.OutOfMemory;
227 continue;
228 }
229 const dst = nextHexToken(arena, parser, "endbfrange") orelse return;
230 if (low_code.width != high_code.width or high_code.value < low_code.value) continue;
231 try appendSteppedRange(arena, ranges, low_code, high_code, dst);
232 }
233 }
234
235 fn appendSteppedRange(arena: std.mem.Allocator, ranges: *std.ArrayList(Mapped), low: Code, high: Code, dst: object.String) ParseError!void {
236 var buffer: [64]u8 = undefined;
237 const raw_len = hexToBytes(dst.raw, &buffer) orelse return;
238 var scalars: [32]u21 = undefined;
239 const count = utf16BeScalars(buffer[0..raw_len], &scalars) orelse return;
240 if (count == 0) return;
241 if (low.value == high.value) {
242 const text = utf8Alloc(arena, scalars[0..count]) orelse return;
243 ranges.append(arena, .{
244 .width = low.width,
245 .low = low.value,
246 .high = high.value,
247 .payload = .{ .single = text },
248 }) catch return error.OutOfMemory;
249 return;
250 }
251 const prefix = utf8Alloc(arena, scalars[0 .. count - 1]) orelse return;
252 ranges.append(arena, .{
253 .width = low.width,
254 .low = low.value,
255 .high = high.value,
256 .payload = .{ .stepped = .{ .prefix = prefix, .base = scalars[count - 1] } },
257 }) catch return error.OutOfMemory;
258 }
259
260 fn nextHexToken(arena: std.mem.Allocator, parser: *object.Parser, comptime end_keyword: []const u8) ?object.String {
261 parser.skipWhitespace();
262 if (parser.pos >= parser.bytes.len) return null;
263 if (parser.bytes[parser.pos] != '<') {
264 if (parser.atKeyword(end_keyword)) {
265 parser.pos += end_keyword.len;
266 }
267 return null;
268 }
269 const value = parser.parseValue(arena) catch return null;
270 return switch (value) {
271 .string => |string| if (string.kind == .hex) string else null,
272 else => null,
273 };
274 }
275
276 fn hexCode(string: object.String) ?Code {
277 var buffer: [4]u8 = undefined;
278 const len = hexToBytes(string.raw, &buffer) orelse return null;
279 if (len == 0) return null;
280 return .{ .width = @intCast(len), .value = readCode(buffer[0..len]) };
281 }
282
283 fn hexToBytes(raw: []const u8, buffer: []u8) ?usize {
284 var count: usize = 0;
285 var high: ?u8 = null;
286 for (raw) |char| {
287 const nibble: u8 = switch (char) {
288 '0'...'9' => char - '0',
289 'a'...'f' => char - 'a' + 10,
290 'A'...'F' => char - 'A' + 10,
291 ' ', '\t', '\r', '\n', 0, 0x0C => continue,
292 else => return null,
293 };
294 if (high) |value| {
295 if (count >= buffer.len) return null;
296 buffer[count] = (value << 4) | nibble;
297 count += 1;
298 high = null;
299 } else {
300 high = nibble;
301 }
302 }
303 if (high) |value| {
304 if (count >= buffer.len) return null;
305 buffer[count] = value << 4;
306 count += 1;
307 }
308 return count;
309 }
310
311 fn utf16BeScalars(bytes: []const u8, scalars: []u21) ?usize {
312 if (bytes.len % 2 != 0) return null;
313 var count: usize = 0;
314 var index: usize = 0;
315 while (index < bytes.len) : (index += 2) {
316 const unit = (@as(u16, bytes[index]) << 8) | bytes[index + 1];
317 if (count >= scalars.len) return null;
318 if (unit >= 0xD800 and unit <= 0xDBFF) {
319 if (index + 3 >= bytes.len) return null;
320 const low = (@as(u16, bytes[index + 2]) << 8) | bytes[index + 3];
321 if (low < 0xDC00 or low > 0xDFFF) return null;
322 scalars[count] = 0x10000 + (@as(u21, unit - 0xD800) << 10) + (low - 0xDC00);
323 index += 2;
324 } else if (unit >= 0xDC00 and unit <= 0xDFFF) {
325 return null;
326 } else {
327 scalars[count] = unit;
328 }
329 count += 1;
330 }
331 return count;
332 }
333
334 fn utf8Alloc(arena: std.mem.Allocator, scalars: []const u21) ?[]const u8 {
335 var out: std.ArrayList(u8) = .empty;
336 for (scalars) |scalar| {
337 var buffer: [4]u8 = undefined;
338 const len = std.unicode.utf8Encode(scalar, &buffer) catch return null;
339 out.appendSlice(arena, buffer[0..len]) catch return null;
340 }
341 return out.toOwnedSlice(arena) catch null;
342 }
343
344 fn utf8FromHexUtf16Alloc(arena: std.mem.Allocator, string: object.String) ?[]const u8 {
345 var buffer: [64]u8 = undefined;
346 const raw_len = hexToBytes(string.raw, &buffer) orelse return null;
347 var scalars: [32]u21 = undefined;
348 const count = utf16BeScalars(buffer[0..raw_len], &scalars) orelse return null;
349 return utf8Alloc(arena, scalars[0..count]);
350 }
351
352 const test_cmap =
353 "/CIDInit /ProcSet findresource begin\n" ++
354 "12 dict begin\nbegincmap\n" ++
355 "/CIDSystemInfo << /Registry (TeX) /Ordering (x) /Supplement 0 >> def\n" ++
356 "1 begincodespacerange\n<00> <FF>\nendcodespacerange\n" ++
357 "2 beginbfrange\n<61> <7A> <0061>\n<20> <26> <0020>\nendbfrange\n" ++
358 "2 beginbfchar\n<02> <00660069>\n<14> <006600660069>\nendbfchar\n" ++
359 "endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n";
360
361 test "cmap parses tex shaped tounicode and decodes codes" {
362 var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
363 defer arena_state.deinit();
364 const map = try parseAlloc(arena_state.allocator(), test_cmap);
365 var out = std.ArrayList(u8).empty;
366 defer out.deinit(std.testing.allocator);
367 try map.appendDecoded(std.testing.allocator, &out, "Arti\x02cial \x14x");
368 try std.testing.expectEqualStrings("\u{FFFD}rtificial ffix", out.items);
369 }
370
371 test "cmap stepped ranges honor multi unit prefixes and lists" {
372 const body =
373 "1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n" ++
374 "2 beginbfrange\n<0005> <0007> <00660066006C>\n<0010> <0011> [<0041> <00420043>]\nendbfrange\n";
375 var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
376 defer arena_state.deinit();
377 const map = try parseAlloc(arena_state.allocator(), body);
378 var out = std.ArrayList(u8).empty;
379 defer out.deinit(std.testing.allocator);
380 try map.appendDecoded(std.testing.allocator, &out, &.{ 0, 6, 0, 0x10, 0, 0x11, 0, 0x12 });
381 try std.testing.expectEqualStrings("ffmABC\u{FFFD}", out.items);
382 }
383
384 test "cmap surrogate pairs and empty maps stay honest" {
385 const body = "1 beginbfchar\n<01> <D83DDE00>\nendbfchar\n";
386 var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
387 defer arena_state.deinit();
388 const map = try parseAlloc(arena_state.allocator(), body);
389 var out = std.ArrayList(u8).empty;
390 defer out.deinit(std.testing.allocator);
391 try map.appendDecoded(std.testing.allocator, &out, "\x01");
392 try std.testing.expectEqualStrings("\u{1F600}", out.items);
393 try std.testing.expectError(error.BadCMap, parseAlloc(arena_state.allocator(), "nothing here"));
394 const malformed =
395 "junk 2 begincodespacerange <GG> <HH> <00> <FF> endcodespacerange " ++
396 "2 beginbfchar <GG> <0042> <01> <0041> endbfchar";
397 const bounded = try parseAlloc(arena_state.allocator(), malformed);
398 out.clearRetainingCapacity();
399 try bounded.appendDecoded(std.testing.allocator, &out, "\x01");
400 try std.testing.expectEqualStrings("A", out.items);
401 }