Skip to main content

makeTokenizer

export function makeTokenizer(
  source: Source,
  options: {
    allowLeadingZeroes?: boolean
    errorCollector?: ErrorCollector
    extraValidFirstLetters?: string
    preserveComments?: boolean
    preserveNewlines?: boolean
    realPrecision?: number
    sourceMap?: SourceMap
    tokenizeReal?: boolean
  }
): Tokenizer

Arguments​

NameTypeDescription
sourceSource
options{   allowLeadingZeroes?: boolean   errorCollector?: ErrorCollector   extraValidFirstLetters?: string   preserveComments?: boolean   preserveNewlines?: boolean   realPrecision?: number   sourceMap?: SourceMap   tokenizeReal?: boolean }

Return value​

Tokenizer