import { FjsonError, type Span } from "./errors.ts"; type TokenData = | { kind: "{" | "}" | "[" | "]" | ":" | "," | "eof" | "invalid" } | { kind: "string" | "identifier"; value: string } | { kind: "number"; value: number; raw: string }; export type Token = TokenData & { readonly span: Span; readonly newlineBefore: boolean; }; function isDigitCode(code: number): boolean { return code >= 48 && code <= 57; } function isIdentifierStart(ch: string | undefined): boolean { if (ch === undefined) return false; const code = ch.charCodeAt(0); return ch === "_" || ch === "$" || (code >= 65 && code <= 90) || (code >= 97 && code <= 122); } function isIdentifierChar(ch: string | undefined): boolean { return ch !== undefined && (isIdentifierStart(ch) || isDigitCode(ch.charCodeAt(0))); } const escapes = new Map([ ['"', '"'], ["'", "'"], ["\\", "\\"], ["/", "/"], ["b", "\b"], ["f", "\f"], ["n", "\n"], ["r", "\r"], ["t", "\t"], ]); export class Scanner { #pos = 0; readonly #source: string; readonly #buffer: Token[] = []; constructor(source: string) { this.#source = source; } peek(offset: 0 | 1 = 0): Token { while (this.#buffer.length <= offset) this.#buffer.push(this.#scan()); return this.#buffer[offset]; } take(): Token { const token = this.peek(); this.#buffer.shift(); return token; } #scan(): Token { const newlineBefore = this.#trivia(); const start = this.#pos; const data = this.#token(); return { ...data, span: { start, end: this.#pos }, newlineBefore }; } #token(): TokenData { const ch = this.#source[this.#pos]; if (ch === undefined) return { kind: "eof" }; if (ch === '"' || ch === "'") { return { kind: "string", value: this.#string() }; } if (isIdentifierStart(ch)) { return { kind: "identifier", value: this.#identifier() }; } if (ch === "-" || isDigitCode(ch.charCodeAt(0))) { return { kind: "number", ...this.#number() }; } this.#pos += 1; switch (ch) { case "{": case "}": case "[": case "]": case ":": case ",": return { kind: ch }; default: return { kind: "invalid" }; } } #error(message: string): FjsonError { return new FjsonError("syntax", message, { start: this.#pos, end: Math.min(this.#pos + 1, this.#source.length), }); } #trivia(): boolean { let newline = false; for (;;) { const ch = this.#source[this.#pos]; if (ch === " " || ch === "\t") { this.#pos += 1; } else if (ch === "\r" || ch === "\n") { newline = true; this.#pos += 1; } else if (ch === "/" && this.#source[this.#pos + 1] === "/") { while ( this.#pos < this.#source.length && this.#source[this.#pos] !== "\r" && this.#source[this.#pos] !== "\n" ) { this.#pos += 1; } } else if (ch === "/" && this.#source[this.#pos + 1] === "*") { const start = this.#pos; const end = this.#source.indexOf("*/", this.#pos + 2); if (end === -1) { throw new FjsonError("syntax", "Unterminated block comment", { start, end: start + 2, }); } // newlines inside block comments aren't implicit commas this.#pos = end + 2; } else { return newline; } } } #identifier(): string { const start = this.#pos; while (isIdentifierChar(this.#source[this.#pos])) this.#pos += 1; return this.#source.slice(start, this.#pos); } #number(): { value: number; raw: string } { const start = this.#pos; if (this.#source[this.#pos] === "-") this.#pos += 1; const first = this.#source.charCodeAt(this.#pos); if (first === 48) { this.#pos += 1; if (isDigitCode(this.#source.charCodeAt(this.#pos))) { throw this.#error("Leading zeros are not allowed"); } } else if (first >= 49 && first <= 57) { this.#digits(); } else { throw this.#error("Expected a digit"); } if (this.#source[this.#pos] === ".") { this.#pos += 1; this.#digits(); } const exponent = this.#source[this.#pos]; if (exponent === "e" || exponent === "E") { this.#pos += 1; const sign = this.#source[this.#pos]; if (sign === "+" || sign === "-") this.#pos += 1; this.#digits(); } const next = this.#source[this.#pos]; if (isIdentifierChar(next) || next === ".") { throw this.#error("Invalid decimal number"); } const raw = this.#source.slice(start, this.#pos); return { value: Number(raw), raw }; } #digits(): void { if (!isDigitCode(this.#source.charCodeAt(this.#pos))) { throw this.#error("Expected a digit"); } while (isDigitCode(this.#source.charCodeAt(this.#pos))) this.#pos += 1; } #string(): string { const start = this.#pos; const quote = this.#source[this.#pos]; this.#pos += 1; let value = ""; for (;;) { if (this.#pos >= this.#source.length) { throw new FjsonError("syntax", "Unterminated string", { start, end: start + 1, }); } const ch = this.#source[this.#pos]; if (ch === quote) { this.#pos += 1; return value; } if (ch === "\\") { this.#pos += 1; const escaped = this.#source[this.#pos]; if (escaped === "u") { this.#pos += 1; value += this.#unicodeEscape(); } else { const decoded = escapes.get(escaped); if (decoded === undefined) throw this.#error("Invalid string escape"); value += decoded; this.#pos += 1; } continue; } if (this.#source.charCodeAt(this.#pos) <= 0x1f) { throw this.#error("Control characters must be escaped in strings"); } value += ch; this.#pos += 1; } } #hexQuad(): number { let value = 0; for (let i = 0; i < 4; i += 1) { const digit = parseInt(this.#source[this.#pos], 16); if (Number.isNaN(digit) || this.#source[this.#pos] === undefined) { throw this.#error("Expected four hexadecimal digits in Unicode escape"); } this.#pos += 1; value = value * 16 + digit; } return value; } #unicodeEscape(): string { const quadStart = this.#pos; const first = this.#hexQuad(); if (first >= 0xd800 && first <= 0xdbff) { if (!this.#source.startsWith("\\u", this.#pos)) { throw new FjsonError("unicode", "Unpaired Unicode surrogate", { start: quadStart, end: this.#pos, }); } this.#pos += 2; const secondStart = this.#pos; const second = this.#hexQuad(); if (second < 0xdc00 || second > 0xdfff) { throw new FjsonError("unicode", "Expected a low Unicode surrogate", { start: secondStart, end: this.#pos, }); } return String.fromCodePoint( 0x10000 + ((first - 0xd800) << 10) + (second - 0xdc00), ); } if (first >= 0xdc00 && first <= 0xdfff) { throw new FjsonError("unicode", "Unexpected low Unicode surrogate", { start: quadStart, end: this.#pos, }); } return String.fromCharCode(first); } }