Skip to main content

makeTokenizer

export function makeTokenizer(
  source: Source,
  options: {
    allowLeadingZeroes??: boolean
    errorCollector??: ErrorCollector
    extraValidFirstLetters??: string
    preserveComments??: boolean
    preserveNewlines??: boolean
    realPrecision??: number
    sourceMap??: SourceMap
    tokenizeReal??: boolean
  }
): Tokenizer

Arguments​

1. source​

source: Source

2. options​

options: {
  allowLeadingZeroes??: boolean
  errorCollector??: ErrorCollector
  extraValidFirstLetters??: string
  preserveComments??: boolean
  preserveNewlines??: boolean
  realPrecision??: number
  sourceMap??: SourceMap
  tokenizeReal??: boolean
}

Returns​

Tokenizer