char/fjson

friendly JSON implementation in TypeScript

git clone https://git.t4t.associates/char/fjson

Charlotte Somsplit scanner from parser17b13b5

main
branchesmainall refs →
7.2 KiB265 linesraw
1import { FjsonError, type Span } from "./errors.ts";
2
3type TokenData =
4  | { kind: "{" | "}" | "[" | "]" | ":" | "," | "eof" | "invalid" }
5  | { kind: "string" | "identifier"; value: string }
6  | { kind: "number"; value: number; raw: string };
7
8export type Token = TokenData & {
9  readonly span: Span;
10  readonly newlineBefore: boolean;
11};
12
13function isDigitCode(code: number): boolean {
14  return code >= 48 && code <= 57;
15}
16
17function isIdentifierStart(ch: string | undefined): boolean {
18  if (ch === undefined) return false;
19  const code = ch.charCodeAt(0);
20  return ch === "_" || ch === "$" ||
21    (code >= 65 && code <= 90) ||
22    (code >= 97 && code <= 122);
23}
24
25function isIdentifierChar(ch: string | undefined): boolean {
26  return ch !== undefined &&
27    (isIdentifierStart(ch) || isDigitCode(ch.charCodeAt(0)));
28}
29
30const escapes = new Map([
31  ['"', '"'],
32  ["'", "'"],
33  ["\\", "\\"],
34  ["/", "/"],
35  ["b", "\b"],
36  ["f", "\f"],
37  ["n", "\n"],
38  ["r", "\r"],
39  ["t", "\t"],
40]);
41
42export class Scanner {
43  #pos = 0;
44  readonly #source: string;
45  readonly #buffer: Token[] = [];
46
47  constructor(source: string) {
48    this.#source = source;
49  }
50
51  peek(offset: 0 | 1 = 0): Token {
52    while (this.#buffer.length <= offset) this.#buffer.push(this.#scan());
53    return this.#buffer[offset];
54  }
55
56  take(): Token {
57    const token = this.peek();
58    this.#buffer.shift();
59    return token;
60  }
61
62  #scan(): Token {
63    const newlineBefore = this.#trivia();
64    const start = this.#pos;
65    const data = this.#token();
66    return { ...data, span: { start, end: this.#pos }, newlineBefore };
67  }
68
69  #token(): TokenData {
70    const ch = this.#source[this.#pos];
71    if (ch === undefined) return { kind: "eof" };
72    if (ch === '"' || ch === "'") {
73      return { kind: "string", value: this.#string() };
74    }
75    if (isIdentifierStart(ch)) {
76      return { kind: "identifier", value: this.#identifier() };
77    }
78    if (ch === "-" || isDigitCode(ch.charCodeAt(0))) {
79      return { kind: "number", ...this.#number() };
80    }
81    this.#pos += 1;
82    switch (ch) {
83      case "{":
84      case "}":
85      case "[":
86      case "]":
87      case ":":
88      case ",":
89        return { kind: ch };
90      default:
91        return { kind: "invalid" };
92    }
93  }
94
95  #error(message: string): FjsonError {
96    return new FjsonError("syntax", message, {
97      start: this.#pos,
98      end: Math.min(this.#pos + 1, this.#source.length),
99    });
100  }
101
102  #trivia(): boolean {
103    let newline = false;
104    for (;;) {
105      const ch = this.#source[this.#pos];
106      if (ch === " " || ch === "\t") {
107        this.#pos += 1;
108      } else if (ch === "\r" || ch === "\n") {
109        newline = true;
110        this.#pos += 1;
111      } else if (ch === "/" && this.#source[this.#pos + 1] === "/") {
112        while (
113          this.#pos < this.#source.length &&
114          this.#source[this.#pos] !== "\r" &&
115          this.#source[this.#pos] !== "\n"
116        ) {
117          this.#pos += 1;
118        }
119      } else if (ch === "/" && this.#source[this.#pos + 1] === "*") {
120        const start = this.#pos;
121        const end = this.#source.indexOf("*/", this.#pos + 2);
122        if (end === -1) {
123          throw new FjsonError("syntax", "Unterminated block comment", {
124            start,
125            end: start + 2,
126          });
127        }
128        // newlines inside block comments aren't implicit commas
129        this.#pos = end + 2;
130      } else {
131        return newline;
132      }
133    }
134  }
135
136  #identifier(): string {
137    const start = this.#pos;
138    while (isIdentifierChar(this.#source[this.#pos])) this.#pos += 1;
139    return this.#source.slice(start, this.#pos);
140  }
141
142  #number(): { value: number; raw: string } {
143    const start = this.#pos;
144    if (this.#source[this.#pos] === "-") this.#pos += 1;
145    const first = this.#source.charCodeAt(this.#pos);
146    if (first === 48) {
147      this.#pos += 1;
148      if (isDigitCode(this.#source.charCodeAt(this.#pos))) {
149        throw this.#error("Leading zeros are not allowed");
150      }
151    } else if (first >= 49 && first <= 57) {
152      this.#digits();
153    } else {
154      throw this.#error("Expected a digit");
155    }
156    if (this.#source[this.#pos] === ".") {
157      this.#pos += 1;
158      this.#digits();
159    }
160    const exponent = this.#source[this.#pos];
161    if (exponent === "e" || exponent === "E") {
162      this.#pos += 1;
163      const sign = this.#source[this.#pos];
164      if (sign === "+" || sign === "-") this.#pos += 1;
165      this.#digits();
166    }
167    const next = this.#source[this.#pos];
168    if (isIdentifierChar(next) || next === ".") {
169      throw this.#error("Invalid decimal number");
170    }
171    const raw = this.#source.slice(start, this.#pos);
172    return { value: Number(raw), raw };
173  }
174
175  #digits(): void {
176    if (!isDigitCode(this.#source.charCodeAt(this.#pos))) {
177      throw this.#error("Expected a digit");
178    }
179    while (isDigitCode(this.#source.charCodeAt(this.#pos))) this.#pos += 1;
180  }
181
182  #string(): string {
183    const start = this.#pos;
184    const quote = this.#source[this.#pos];
185    this.#pos += 1;
186    let value = "";
187    for (;;) {
188      if (this.#pos >= this.#source.length) {
189        throw new FjsonError("syntax", "Unterminated string", {
190          start,
191          end: start + 1,
192        });
193      }
194      const ch = this.#source[this.#pos];
195      if (ch === quote) {
196        this.#pos += 1;
197        return value;
198      }
199      if (ch === "\\") {
200        this.#pos += 1;
201        const escaped = this.#source[this.#pos];
202        if (escaped === "u") {
203          this.#pos += 1;
204          value += this.#unicodeEscape();
205        } else {
206          const decoded = escapes.get(escaped);
207          if (decoded === undefined) throw this.#error("Invalid string escape");
208          value += decoded;
209          this.#pos += 1;
210        }
211        continue;
212      }
213      if (this.#source.charCodeAt(this.#pos) <= 0x1f) {
214        throw this.#error("Control characters must be escaped in strings");
215      }
216      value += ch;
217      this.#pos += 1;
218    }
219  }
220
221  #hexQuad(): number {
222    let value = 0;
223    for (let i = 0; i < 4; i += 1) {
224      const digit = parseInt(this.#source[this.#pos], 16);
225      if (Number.isNaN(digit) || this.#source[this.#pos] === undefined) {
226        throw this.#error("Expected four hexadecimal digits in Unicode escape");
227      }
228      this.#pos += 1;
229      value = value * 16 + digit;
230    }
231    return value;
232  }
233
234  #unicodeEscape(): string {
235    const quadStart = this.#pos;
236    const first = this.#hexQuad();
237    if (first >= 0xd800 && first <= 0xdbff) {
238      if (!this.#source.startsWith("\\u", this.#pos)) {
239        throw new FjsonError("unicode", "Unpaired Unicode surrogate", {
240          start: quadStart,
241          end: this.#pos,
242        });
243      }
244      this.#pos += 2;
245      const secondStart = this.#pos;
246      const second = this.#hexQuad();
247      if (second < 0xdc00 || second > 0xdfff) {
248        throw new FjsonError("unicode", "Expected a low Unicode surrogate", {
249          start: secondStart,
250          end: this.#pos,
251        });
252      }
253      return String.fromCodePoint(
254        0x10000 + ((first - 0xd800) << 10) + (second - 0xdc00),
255      );
256    }
257    if (first >= 0xdc00 && first <= 0xdfff) {
258      throw new FjsonError("unicode", "Unexpected low Unicode surrogate", {
259        start: quadStart,
260        end: this.#pos,
261      });
262    }
263    return String.fromCharCode(first);
264  }
265}