tiny.css.token

1/// Whether a hash token could name an identifier, which decides whether `#a`2/// is an id selector or a hex color.3pub const HashKind = enum(u8) {4    unrestricted,5    id,6};

1/// The token kinds CSS Syntax Level 3 section 4 produces. Comments are2/// consumed rather than emitted, which is what the parsing algorithms expect.3pub const Kind = enum(u8) {4    eof,5    whitespace,6    ident,7    function,8    at_keyword,9    hash,10    string,11    bad_string,12    url,13    bad_url,14    delim,15    number,16    percentage,17    dimension,18    cdo,19    cdc,20    colon,21    semicolon,22    comma,23    left_square,24    right_square,25    left_paren,26    right_paren,27    left_curly,28    right_curly,29};

1/// Whether a numeric token was written without a fraction or an exponent.2pub const NumericKind = enum(u8) {3    integer,4    number,5};

1/// One token as a span over the caller's source bytes. `value` narrows to the2/// part a consumer reads: an identifier without its sigil, a string without3/// its quotes, a dimension without its unit.4pub const Token = struct {5    kind: Kind,6    start: u32,7    end: u32,8    value_start: u32,9    value_end: u32,10    unit_start: u32 = 0,11    unit_end: u32 = 0,12    number: f64 = 0,13    numeric: NumericKind = .integer,14    hash_kind: HashKind = .unrestricted,15    escaped: bool = false,1617    /// The whole token including any sigil, quotes, or unit.18    pub fn text(self: Token, source: []const u8) []const u8 {19        return source[self.start..self.end];20    }2122    /// The consumer facing span described above.23    pub fn value(self: Token, source: []const u8) []const u8 {24        return source[self.value_start..self.value_end];25    }2627    /// The dimension unit, empty for every other kind.28    pub fn unit(self: Token, source: []const u8) []const u8 {29        return source[self.unit_start..self.unit_end];30    }31};

1/// A forward tokenizer over borrowed source bytes. It allocates nothing and2/// every `next` either advances `index` or returns `eof`.3pub const Tokenizer = struct {4    source: []const u8,5    index: u32 = 0,67    pub fn init(source: []const u8) Tokenizer {8        std.debug.assert(source.len <= std.math.maxInt(u32));9        return .{ .source = source };10    }1112    /// Consumes and returns the next token.13    pub fn next(self: *Tokenizer) Token {14        self.skipComments();15        const start = self.index;16        if (start >= self.source.len) return self.single(.eof, start);17        const byte = self.source[start];18        if (isWhitespace(byte)) return self.whitespace(start);19        if (byte == '"' or byte == '\'') return self.string(start, byte);20        if (byte == '#') return self.hash(start);21        if (byte == '+' or byte == '.') {22            if (startsNumber(self.source, start)) return self.numeric(start);23            return self.delim(start);24        }25        if (byte == '-') return self.minus(start);26        if (byte == '<') return self.lessThan(start);27        if (byte == '@') return self.atKeyword(start);28        if (byte == '\\') {29            if (validEscape(self.source, start)) return self.identLike(start);30            return self.delim(start);31        }32        if (isDigit(byte)) return self.numeric(start);33        if (isIdentStart(byte)) return self.identLike(start);34        if (punctuation(byte)) |kind| return self.single(kind, start);35        return self.delim(start);36    }3738    fn skipComments(self: *Tokenizer) void {39        var guard: usize = 0;40        while (guard <= self.source.len) : (guard += 1) {41            const index = self.index;42            if (index + 1 >= self.source.len) return;43            if (self.source[index] != '/' or self.source[index + 1] != '*') return;44            const rest = self.source[index + 2 ..];45            const close = std.mem.indexOf(u8, rest, "*/") orelse {46                self.index = @intCast(self.source.len);47                return;48            };49            self.index = index + 2 + @as(u32, @intCast(close)) + 2;50        }51        unreachable;52    }5354    fn single(self: *Tokenizer, kind: Kind, start: u32) Token {55        const end = if (kind == .eof) start else start + 1;56        self.index = end;57        return .{ .kind = kind, .start = start, .end = end, .value_start = start, .value_end = end };58    }5960    fn delim(self: *Tokenizer, start: u32) Token {61        return self.single(.delim, start);62    }6364    fn whitespace(self: *Tokenizer, start: u32) Token {65        var index = start;66        while (index < self.source.len and isWhitespace(self.source[index])) index += 1;67        self.index = index;68        return .{69            .kind = .whitespace,70            .start = start,71            .end = index,72            .value_start = start,73            .value_end = index,74        };75    }7677    fn hash(self: *Tokenizer, start: u32) Token {78        const after = start + 1;79        if (after < self.source.len and80            (isIdent(self.source[after]) or validEscape(self.source, after)))81        {82            const kind: HashKind = if (startsIdent(self.source, after)) .id else .unrestricted;83            const end = consumeIdent(self.source, after);84            self.index = end;85            return .{86                .kind = .hash,87                .start = start,88                .end = end,89                .value_start = after,90                .value_end = end,91                .hash_kind = kind,92                .escaped = hasEscape(self.source[after..end]),93            };94        }95        return self.delim(start);96    }9798    fn minus(self: *Tokenizer, start: u32) Token {99        if (startsNumber(self.source, start)) return self.numeric(start);100        if (start + 2 < self.source.len and std.mem.eql(u8, self.source[start..][0..3], "-->")) {101            self.index = start + 3;102            return .{103                .kind = .cdc,104                .start = start,105                .end = start + 3,106                .value_start = start,107                .value_end = start + 3,108            };109        }110        if (startsIdent(self.source, start)) return self.identLike(start);111        return self.delim(start);112    }113114    fn lessThan(self: *Tokenizer, start: u32) Token {115        if (start + 3 < self.source.len and std.mem.eql(u8, self.source[start..][0..4], "<!--")) {116            self.index = start + 4;117            return .{118                .kind = .cdo,119                .start = start,120                .end = start + 4,121                .value_start = start,122                .value_end = start + 4,123            };124        }125        return self.delim(start);126    }127128    fn atKeyword(self: *Tokenizer, start: u32) Token {129        const after = start + 1;130        if (!startsIdent(self.source, after)) return self.delim(start);131        const end = consumeIdent(self.source, after);132        self.index = end;133        return .{134            .kind = .at_keyword,135            .start = start,136            .end = end,137            .value_start = after,138            .value_end = end,139            .escaped = hasEscape(self.source[after..end]),140        };141    }142143    fn identLike(self: *Tokenizer, start: u32) Token {144        const end = consumeIdent(self.source, start);145        const name = self.source[start..end];146        if (end < self.source.len and self.source[end] == '(') {147            if (std.ascii.eqlIgnoreCase(name, "url") and !quotedUrl(self.source, end + 1)) {148                return self.url(start, end + 1);149            }150            self.index = end + 1;151            return .{152                .kind = .function,153                .start = start,154                .end = end + 1,155                .value_start = start,156                .value_end = end,157                .escaped = hasEscape(name),158            };159        }160        self.index = end;161        return .{162            .kind = .ident,163            .start = start,164            .end = end,165            .value_start = start,166            .value_end = end,167            .escaped = hasEscape(name),168        };169    }170171    fn url(self: *Tokenizer, start: u32, body: u32) Token {172        var index = body;173        while (index < self.source.len and isWhitespace(self.source[index])) index += 1;174        const value_start = index;175        while (index < self.source.len and self.source[index] != ')') {176            if (self.source[index] == '\\' and index + 1 < self.source.len) {177                index += 2;178                continue;179            }180            index += 1;181        }182        const value_end = trimTrailingSpace(self.source, value_start, index);183        const closed = index < self.source.len;184        self.index = if (closed) index + 1 else index;185        return .{186            .kind = if (closed) .url else .bad_url,187            .start = start,188            .end = self.index,189            .value_start = value_start,190            .value_end = value_end,191            .escaped = hasEscape(self.source[value_start..value_end]),192        };193    }194195    fn string(self: *Tokenizer, start: u32, quote: u8) Token {196        var index = start + 1;197        while (index < self.source.len) {198            const byte = self.source[index];199            if (byte == quote) {200                self.index = index + 1;201                return .{202                    .kind = .string,203                    .start = start,204                    .end = index + 1,205                    .value_start = start + 1,206                    .value_end = index,207                    .escaped = hasEscape(self.source[start + 1 .. index]),208                };209            }210            if (byte == '\n') break;211            if (byte == '\\' and index + 1 < self.source.len) {212                index += 2;213                continue;214            }215            index += 1;216        }217        self.index = index;218        return .{219            .kind = .bad_string,220            .start = start,221            .end = index,222            .value_start = start + 1,223            .value_end = index,224        };225    }226227    fn numeric(self: *Tokenizer, start: u32) Token {228        const scanned = consumeNumber(self.source, start);229        var token = Token{230            .kind = .number,231            .start = start,232            .end = scanned.end,233            .value_start = start,234            .value_end = scanned.end,235            .number = std.fmt.parseFloat(f64, self.source[start..scanned.end]) catch 0,236            .numeric = scanned.kind,237        };238        if (startsIdent(self.source, scanned.end)) {239            const unit_end = consumeIdent(self.source, scanned.end);240            token.kind = .dimension;241            token.unit_start = scanned.end;242            token.unit_end = unit_end;243            token.end = unit_end;244        } else if (scanned.end < self.source.len and self.source[scanned.end] == '%') {245            token.kind = .percentage;246            token.end = scanned.end + 1;247        }248        self.index = token.end;249        return token;250    }251};

1/// Consumes an identifier starting at `index` and returns its end.2pub fn consumeIdent(source: []const u8, index: u32) u32 {3    var cursor = index;4    var guard: usize = 0;5    while (cursor < source.len and guard <= source.len) : (guard += 1) {6        if (isIdent(source[cursor])) {7            cursor += 1;8            continue;9        }10        if (validEscape(source, cursor)) {11            cursor += 2;12            if (isHex(source[cursor - 1])) {13                var digits: usize = 1;14                while (cursor < source.len and digits < 6 and isHex(source[cursor])) : (digits += 1) {15                    cursor += 1;16                }17                if (cursor < source.len and isWhitespace(source[cursor])) cursor += 1;18            }19            continue;20        }21        break;22    }23    return @min(cursor, @as(u32, @intCast(source.len)));24}

1/// Whether `text` carries an escape and therefore needs `unescape`.2pub fn hasEscape(text: []const u8) bool {3    return std.mem.indexOfScalar(u8, text, '\\') != null;4}

1/// Whether `byte` may continue an identifier.2pub fn isIdent(byte: u8) bool {3    return isIdentStart(byte) or std.ascii.isDigit(byte) or byte == '-';4}

1/// Whether `byte` may start an identifier.2pub fn isIdentStart(byte: u8) bool {3    return std.ascii.isAlphabetic(byte) or byte == '_' or byte >= 0x80;4}

1/// Whether `byte` is one of the five CSS whitespace bytes.2pub fn isWhitespace(byte: u8) bool {3    return byte == ' ' or byte == '\t' or byte == '\n' or byte == '\r' or byte == 0x0C;4}

1/// Whether an identifier starts at `index`, per section 4.3.9.2pub fn startsIdent(source: []const u8, index: u32) bool {3    if (index >= source.len) return false;4    const byte = source[index];5    if (isIdentStart(byte)) return true;6    if (byte == '-') {7        if (index + 1 >= source.len) return false;8        const after = source[index + 1];9        return isIdentStart(after) or after == '-' or validEscape(source, index + 1);10    }11    return validEscape(source, index);12}

1/// Whether a number starts at `index`, per section 4.3.10.2pub fn startsNumber(source: []const u8, index: u32) bool {3    if (index >= source.len) return false;4    const byte = source[index];5    if (isDigit(byte)) return true;6    if (byte == '.') return index + 1 < source.len and isDigit(source[index + 1]);7    if (byte != '+' and byte != '-') return false;8    if (index + 1 >= source.len) return false;9    if (isDigit(source[index + 1])) return true;10    return source[index + 1] == '.' and index + 2 < source.len and isDigit(source[index + 2]);11}

1/// Decodes CSS escapes from `text` into `out`, dropping escaped newlines.2/// Returns null when the decoded identifier does not fit.3pub fn unescape(text: []const u8, out: []u8) ?[]const u8 {4    var read: usize = 0;5    var written: usize = 0;6    while (read < text.len) {7        const byte = text[read];8        if (byte != '\\') {9            if (written >= out.len) return null;10            out[written] = byte;11            written += 1;12            read += 1;13            continue;14        }15        read += 1;16        if (read >= text.len) {17            if (written + 3 > out.len) return null;18            written += std.unicode.utf8Encode(0xFFFD, out[written..]) catch return null;19            break;20        }21        if (text[read] == '\n') {22            read += 1;23            continue;24        }25        if (!isHex(text[read])) {26            const start = read;27            read += std.unicode.utf8ByteSequenceLength(text[read]) catch 1;28            const end = @min(read, text.len);29            if (written + (end - start) > out.len) return null;30            @memcpy(out[written..][0 .. end - start], text[start..end]);31            written += end - start;32            continue;33        }34        var point: u32 = 0;35        var digits: usize = 0;36        while (read < text.len and digits < 6 and isHex(text[read])) : (digits += 1) {37            point = point * 16 + hexValue(text[read]);38            read += 1;39        }40        if (read < text.len and isWhitespace(text[read])) read += 1;41        const scalar = if (point == 0 or point > 0x10FFFF or42            (point >= 0xD800 and point <= 0xDFFF)) 0xFFFD else point;43        written += std.unicode.utf8Encode(@intCast(scalar), out[written..]) catch return null;44    }45    return out[0..written];46}

1/// Whether a valid escape starts at `index`, per section 4.3.8.2pub fn validEscape(source: []const u8, index: u32) bool {3    if (index >= source.len or source[index] != '\\') return false;4    return index + 1 < source.len and source[index + 1] != '\n';5}

1pub const token = @import("token/root.zig");

1//! The CSS Syntax Level 3 tokenizer.2//!3//! Every selector, declaration value, and at-rule prelude in this package is4//! read through `Tokenizer`, so escapes, strings, comments, and numeric units5//! are handled once instead of at each byte scanning site.6//!7//! The tokenizer borrows the caller's source bytes and allocates nothing. A8//! token is a pair of spans over that source, so a consumer that needs decoded9//! text calls `unescape` with its own buffer.10//!11//! Identifiers need no Unicode tables. Section 4.2 makes every byte at or above12//! 0x80 an identifier code point, so UTF-8 identifiers scan byte wise.1314const scan = @import("scan.zig");1516pub const HashKind = scan.HashKind;17pub const Kind = scan.Kind;18pub const NumericKind = scan.NumericKind;19pub const Token = scan.Token;20pub const Tokenizer = scan.Tokenizer;21pub const consumeIdent = scan.consumeIdent;22pub const hasEscape = scan.hasEscape;23pub const isIdent = scan.isIdent;24pub const isIdentStart = scan.isIdentStart;25pub const isWhitespace = scan.isWhitespace;26pub const startsIdent = scan.startsIdent;27pub const startsNumber = scan.startsNumber;28pub const unescape = scan.unescape;29pub const validEscape = scan.validEscape;