lib/ui/src/text/segment.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

 1 const std = @import("std");
 2 const unicode = @import("unicode");
 3 
 4 pub const Boundary = struct {
 5     byte_offset: u32,
 6     action: unicode.LineBreakAction,
 7 };
 8 
 9 const Case = struct {
10     text: []const u8,
11     actions: []const unicode.LineBreakAction,
12 };
13 
14 pub fn boundaries(
15     text: []const u8,
16     output: []Boundary,
17 ) error{ OutputTooSmall, TextTooLong }![]Boundary {
18     if (text.len > std.math.maxInt(u32)) return error.TextTooLong;
19     std.debug.assert(std.unicode.utf8ValidateSlice(text));
20     var graphemes = unicode.GraphemeIterator.init(text);
21     var line: unicode.LineBreakState = .{};
22     var count: usize = 0;
23     while (graphemes.next()) |cluster| {
24         std.debug.assert(cluster.valid_utf8);
25         var cursor: usize = 0;
26         var action: unicode.LineBreakAction = .prohibited;
27         while (cursor < cluster.text.len) {
28             const len = std.unicode.utf8ByteSequenceLength(cluster.text[cursor]) catch unreachable;
29             const codepoint = std.unicode.utf8Decode(cluster.text[cursor..][0..len]) catch unreachable;
30             const next = line.consume(codepoint);
31             if (cursor == 0) action = next;
32             cursor += len;
33         }
34         if (count >= output.len) return error.OutputTooSmall;
35         output[count] = .{ .byte_offset = @intCast(cluster.start), .action = action };
36         count += 1;
37     }
38     if (count >= output.len) return error.OutputTooSmall;
39     output[count] = .{ .byte_offset = @intCast(text.len), .action = line.finish() };
40     return output[0 .. count + 1];
41 }
42 
43 test "line boundaries follow selected Unicode 17 conformance cases" {
44     const cases = [_]Case{
45         .{ .text = "❗ ❗", .actions = &.{ .prohibited, .prohibited, .opportunity, .mandatory } },
46         .{ .text = "❗\u{0308}❗", .actions = &.{ .prohibited, .prohibited, .mandatory } },
47         .{
48             .text = "❗\u{200b}❗",
49             .actions = &.{ .prohibited, .prohibited, .opportunity, .mandatory },
50         },
51         .{ .text = "🇦🇦🇦🇦", .actions = &.{ .prohibited, .opportunity, .mandatory } },
52         .{ .text = "a b", .actions = &.{ .prohibited, .prohibited, .opportunity, .mandatory } },
53     };
54     for (cases) |case| {
55         var output: [16]Boundary = undefined;
56         const found = try boundaries(case.text, &output);
57         try std.testing.expectEqual(case.actions.len, found.len);
58         for (found, case.actions) |actual, expected| {
59             try std.testing.expectEqual(expected, actual.action);
60         }
61     }
62 }
63 
64 test "line boundaries never split a combining grapheme" {
65     var output: [4]Boundary = undefined;
66     const found = try boundaries("a\u{0308}b", &output);
67     try std.testing.expectEqual(@as(usize, 3), found.len);
68     try std.testing.expectEqual(@as(u32, 0), found[0].byte_offset);
69     try std.testing.expectEqual(@as(u32, 3), found[1].byte_offset);
70     try std.testing.expectEqual(@as(u32, 4), found[2].byte_offset);
71 }
72 
73 test "empty text has one mandatory end boundary" {
74     var output: [1]Boundary = undefined;
75     const found = try boundaries("", &output);
76     try std.testing.expectEqual(@as(usize, 1), found.len);
77     try std.testing.expectEqual(@as(u32, 0), found[0].byte_offset);
78     try std.testing.expectEqual(unicode.LineBreakAction.mandatory, found[0].action);
79 }