tiny.python.source.token

1pub const token = @import("token.zig");

1//! A token is one name, number, string, keyword or symbol of a program's text. A token records its2//! kind and the byte range it covers in the text.3//!4//! The parser decides what to build from a token's kind alone. The parser reads a token's5//! characters only for names, numbers and strings.6//!7//! The kinds follow the categories of lexical analysis in the [Python 3.14 language8//! reference](https://docs.python.org/3.14/reference/): names, keywords, literals, operators and9//! delimiters. The kinds also include the reference's newline, indent and dedent tokens, which10//! carry the line and block structure.11//!12//! The kinds cover the package's subset: two literal kinds (integers and strings), 19 keywords, 1913//! operators and delimiters, and a kind that marks the end of the input. A token stores its byte14//! range, `Span`, as two offsets into the text. A token keeps no copy of the text, so it has a15//! fixed size and owns no memory. A caller reads a token's characters by passing the same text back16//! to `Span.text`.17const std = @import("std");1819/// The kind of a token. The parser switches and matches on it to decide what each token starts.20/// Keyword kinds are named for their keyword with `_kw` added, as `if_kw` is `if`. `true_kw`,21/// `false_kw` and `none_kw` are `True`, `False` and `None`. The lexer gives a keyword kind to these22/// 19 words alone. Every other word gets the kind `identifier`. Operator and delimiter kinds are23/// named for their characters, as `plus` is `+`, `less_equal` is `<=` and `lparen` is `(`.24/// `newline`, `indent`, `dedent` and `eof` carry the line and block structure and the end of the25/// input.26pub const Tag = enum {27    /// A name that is none of the 19 keywords. A name starts with an ASCII letter or underscore and28    /// continues with ASCII letters, digits and underscores. Python's other reserved words, such as29    /// `class` and `import`, get this kind too.30    identifier,31    /// A run of decimal digits. The token holds no sign, underscore or base prefix: `-5` is a32    /// `minus` token and an `integer` token. The parser reads the digits as a signed 128-bit33    /// integer.34    integer,35    /// A string literal between single or double quotes, on one line. Its byte range includes both36    /// quotes. A backslash has no special meaning, so the first quote that matches the opening one37    /// ends the literal.38    string,39    and_kw,40    break_kw,41    continue_kw,42    del_kw,43    def_kw,44    elif_kw,45    else_kw,46    for_kw,47    if_kw,48    in_kw,49    is_kw,50    not_kw,51    or_kw,52    pass_kw,53    return_kw,54    while_kw,55    true_kw,56    false_kw,57    none_kw,58    plus,59    minus,60    star,61    /// A single `=`, the assignment sign. `==` has its own kind, `equal_equal`.62    equal,63    equal_equal,64    bang_equal,65    less,66    less_equal,67    greater,68    greater_equal,69    comma,70    colon,71    dot,72    lparen,73    rparen,74    lbracket,75    rbracket,76    lbrace,77    rbrace,78    /// One line break: `\n`, `\r\n` or a lone `\r`. Its byte range covers the bytes of the break.79    /// The lexer emits one for every line break, including the break that ends a blank or80    /// comment-only line. A line break inside parentheses, brackets or braces yields one as well.81    newline,82    /// The start of a block. The lexer emits one before the first token of a line indented deeper83    /// than the enclosing block. Its byte range covers that line's leading spaces.84    indent,85    /// The end of a block. The lexer emits one for each indentation level that a less indented line86    /// closes. Its byte range covers the leading spaces of that line. At the end of the input, the87    /// lexer emits one for each level still open, and each has an empty range at the end of the88    /// text.89    dedent,90    /// The last token of every list that `tokenize` returns, after any `dedent` tokens that close91    /// the open blocks. Its byte range is empty and sits at the end of the text.92    eof,93};9495/// A byte range of the source text: the offset of its first byte and the offset one past its last96/// byte. The parser reads a token's characters back out of the source text through it. A span holds97/// two offsets and no pointer, so it borrows nothing.98pub const Span = struct {99    /// The offset of the range's first byte in the source text.100    start: usize,101    /// The offset one past the range's last byte. `end - start` is the range's length. An empty102    /// range has `start == end`.103    end: usize,104105    /// Returns the bytes of the range, as a slice of the given source text. The parser reads names,106    /// integer digits and string literals through it. The caller passes the text the range was107    /// taken from, and the range has to lie inside it. The slice borrows the text and allocates108    /// nothing.109    pub fn text(self: Span, source: []const u8) []const u8 {110        return source[self.start..self.end];111    }112};113114/// One token: its kind and its byte range in the source text. `tokenize` returns a list of them,115/// and the parser walks that list. A token owns no memory. A caller reads its characters back with116/// `span.text(source)`.117pub const Token = struct {118    /// The token's kind.119    tag: Tag,120    /// The token's byte range in the source text.121    span: Span,122};123124test "span returns source text" {125    const source = "alpha";126    try std.testing.expectEqualStrings("alp", (Span{ .start = 0, .end = 3 }).text(source));127}