Tokenizer

export interface Tokenizer {
  errors: ErrorCollector
  stream(): Generator<Token>
  tokenize(nestGroups?: boolean): Token[]
}

Instantiate a Tokenizer with makeTokenizer.

Properties

errors

errors: ErrorCollector

stream

stream(): Generator<Token>

tokenize

tokenize(nestGroups?: boolean): Token[]