lib/css/src/token/root.zig

daab053ee43316e1809a84551d573ddd1e5bf3d2

 1 //! The CSS Syntax Level 3 tokenizer.
 2 //!
 3 //! Every selector, declaration value, and at-rule prelude in this package is
 4 //! read through `Tokenizer`, so escapes, strings, comments, and numeric units
 5 //! are handled once instead of at each byte scanning site.
 6 //!
 7 //! The tokenizer borrows the caller's source bytes and allocates nothing. A
 8 //! token is a pair of spans over that source, so a consumer that needs decoded
 9 //! text calls `unescape` with its own buffer.
10 //!
11 //! Identifiers need no Unicode tables. Section 4.2 makes every byte at or above
12 //! 0x80 an identifier code point, so UTF-8 identifiers scan byte wise.
13 
14 const scan = @import("scan.zig");
15 
16 pub const HashKind = scan.HashKind;
17 pub const Kind = scan.Kind;
18 pub const NumericKind = scan.NumericKind;
19 pub const Token = scan.Token;
20 pub const Tokenizer = scan.Tokenizer;
21 pub const consumeIdent = scan.consumeIdent;
22 pub const hasEscape = scan.hasEscape;
23 pub const isIdent = scan.isIdent;
24 pub const isIdentStart = scan.isIdentStart;
25 pub const isWhitespace = scan.isWhitespace;
26 pub const startsIdent = scan.startsIdent;
27 pub const startsNumber = scan.startsNumber;
28 pub const unescape = scan.unescape;
29 pub const validEscape = scan.validEscape;