Tokenizer

export interface Tokenizer {
  errors: ErrorCollector
  stream(): Generator<Token>
  tokenize(nestGroups: boolean): Token[]
}

Properties

errors

tokenizer.errors satisfies ErrorCollector

stream

tokenizer.stream satisfies () => Generator<Token>

tokenize

tokenizer.tokenize satisfies (nestGroups: boolean) => Token[]