tiny.python.source

1pub const source = @import("source/root.zig");

1//! The first stage of the package turns the text of a Python program into a list of tokens. Each2//! token records its kind and the byte range it covers in the text. The parser needs the program as3//! a sequence of names, numbers, strings, keywords and symbols, with the start and end of every4//! indented block marked.5//!6//! One file, `token`, defines the token and its kinds. The other file, `lexer`, produces the list7//! through one function, `tokenize`. The namespace re-exports `Token`, `Tag` and `Span` from8//! `token`, and `Stream` and `tokenize` from `lexer`. The package's `execute` calls `tokenize`9//! before anything else.10pub const token = @import("token.zig");11pub const lexer = @import("lexer.zig");1213pub const Token = token.Token;14pub const Tag = token.Tag;15pub const Span = token.Span;16pub const Stream = lexer.Stream;17pub const tokenize = lexer.tokenize;