xbintsc 0.1.6 → 0.1.9

package/src/lexer/scanner.ts

@@ -1,115 +1,15 @@
 import { DiagnosticBag, DiagnosticCode } from "../diagnostics/diagnostic.js";
 import type { SourceFile } from "../diagnostics/source.js";
 import { KEYWORDS, Token, TokenKind } from "./token.js";
-
-const enum Char {
-  Null = 0,
-  Tab = 9,
-  LineFeed = 10,
-  VerticalTab = 11,
-  FormFeed = 12,
-  CarriageReturn = 13,
-  Space = 32,
-  Exclamation = 33,
-  DoubleQuote = 34,
-  Hash = 35,
-  Dollar = 36,
-  Percent = 37,
-  Ampersand = 38,
-  SingleQuote = 39,
-  OpenParen = 40,
-  CloseParen = 41,
-  Asterisk = 42,
-  Plus = 43,
-  Comma = 44,
-  Minus = 45,
-  Dot = 46,
-  Slash = 47,
-  Zero = 48,
-  Nine = 57,
-  Colon = 58,
-  Semicolon = 59,
-  LessThan = 60,
-  Equals = 61,
-  GreaterThan = 62,
-  Question = 63,
-  At = 64,
-  UpperA = 65,
-  UpperB = 66,
-  UpperE = 69,
-  UpperF = 70,
-  UpperO = 79,
-  UpperX = 88,
-  UpperZ = 90,
-  OpenBracket = 91,
-  Backslash = 92,
-  CloseBracket = 93,
-  Caret = 94,
-  Underscore = 95,
-  Backtick = 96,
-  LowerA = 97,
-  LowerB = 98,
-  LowerE = 101,
-  LowerF = 102,
-  LowerN = 110,
-  LowerO = 111,
-  LowerR = 114,
-  LowerT = 116,
-  LowerU = 117,
-  LowerX = 120,
-  LowerZ = 122,
-  OpenBrace = 123,
-  Bar = 124,
-  CloseBrace = 125,
-  Tilde = 126,
-  NonBreakingSpace = 0xa0,
-  LineSeparator = 0x2028,
-  ParagraphSeparator = 0x2029,
-  ByteOrderMark = 0xfeff,
-}
-
-function isLineBreak(ch: number): boolean {
-  return ch === Char.LineFeed || ch === Char.CarriageReturn || ch === Char.LineSeparator || ch === Char.ParagraphSeparator;
-}
-
-function isWhiteSpace(ch: number): boolean {
-  return (
-    ch === Char.Space ||
-    ch === Char.Tab ||
-    ch === Char.VerticalTab ||
-    ch === Char.FormFeed ||
-    ch === Char.NonBreakingSpace ||
-    ch === Char.ByteOrderMark ||
-    (ch > 0x1680 && WHITESPACE_UNICODE.has(ch))
-  );
-}
-
-const WHITESPACE_UNICODE = new Set<number>([
-  0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200a, 0x202f, 0x205f,
-  0x3000,
-]);
-
-function isDigit(ch: number): boolean {
-  return ch >= Char.Zero && ch <= Char.Nine;
-}
-
-function isHexDigit(ch: number): boolean {
-  return isDigit(ch) || (ch >= Char.LowerA && ch <= Char.LowerF) || (ch >= Char.UpperA && ch <= Char.UpperF);
-}
-
-function isIdentifierStart(ch: number): boolean {
-  return (
-    (ch >= Char.LowerA && ch <= Char.LowerZ) ||
-    (ch >= Char.UpperA && ch <= Char.UpperZ) ||
-    ch === Char.Dollar ||
-    ch === Char.Underscore ||
-    ch > 0x7f
-  );
-}
-
-function isIdentifierPart(ch: number): boolean {
-  return isIdentifierStart(ch) || isDigit(ch) || ch === Char.Backslash || ch > 0x7f;
-}
+import {
+  Char,
+  isDigit,
+  isHexDigit,
+  isIdentifierPart,
+  isIdentifierStart,
+  isLineBreak,
+  isWhiteSpace,
+} from "./char.js";
 
 /**
  * Hand-written scanner for the TypeScript language. It produces a flat token

package/src/parser/context.ts

@@ -0,0 +1,196 @@
+/**
+ * Shared state and low-level token plumbing for the parser.
+ *
+ * The parser's method groups (statements, expressions, types, modules) are
+ * defined as separate modules and installed onto `Parser.prototype`. This class
+ * holds the state they all operate on plus the token accessors every group
+ * needs.
+ */
+
+import { DiagnosticBag, DiagnosticCode } from "../diagnostics/diagnostic.js";
+import type { SourceFile } from "../diagnostics/source.js";
+import { Scanner } from "../lexer/scanner.js";
+import { Token, TokenKind } from "../lexer/token.js";
+import { SyntaxKind } from "../ast/kinds.js";
+import type { Identifier, StringLiteral } from "../ast/nodes.js";
+import { SpeculationError } from "./speculation.js";
+
+export interface ParserState {
+  index: number;
+  tokens: Token[];
+  pos: number;
+  consumed: number;
+}
+
+export class ParserContext {
+  readonly scanner: Scanner;
+  readonly diagnostics: DiagnosticBag;
+  readonly source: SourceFile;
+  tokens: Token[];
+  index = 0;
+  /** Non-zero while a speculative parse is running; errors abort the attempt. */
+  speculating = 0;
+  /** Monotonic count of tokens consumed; used to detect lack of progress. */
+  consumed = 0;
+
+  constructor(source: SourceFile, diagnostics: DiagnosticBag) {
+    this.source = source;
+    this.diagnostics = diagnostics;
+    this.scanner = new Scanner(source, diagnostics);
+    this.tokens = [this.scanner.nextToken()];
+  }
+
+  // -- token access ---------------------------------------------------------
+
+  get token(): Token {
+    return this.tokens[this.index]!;
+  }
+
+  /**
+   * Compare the current token kind without triggering TypeScript's control-flow
+   * narrowing of `this.token.kind`, which would otherwise be invalidated by the
+   * calls to `nextToken()` that occur between checks.
+   */
+  at(kind: TokenKind): boolean {
+    return this.tokens[this.index]!.kind === kind;
+  }
+
+  atAhead(k: number, kind: TokenKind): boolean {
+    return this.lookAhead(k).kind === kind;
+  }
+
+  lookAhead(k: number): Token {
+    while (this.tokens.length <= this.index + k) {
+      this.tokens.push(this.scanner.nextToken());
+    }
+    return this.tokens[this.index + k]!;
+  }
+
+  nextToken(): Token {
+    this.consumed++;
+    const current = this.token;
+    if (this.index + 1 < this.tokens.length) {
+      this.index++;
+    } else {
+      this.tokens = [this.scanner.nextToken()];
+      this.index = 0;
+    }
+    return current;
+  }
+
+  parseExpected(kind: TokenKind, message?: string): Token {
+    if (this.token.kind === kind) return this.nextToken();
+    this.error(
+      DiagnosticCode.ExpectedToken,
+      message ?? `Expected '${kind}' but found '${this.token.text || this.token.kind}'`,
+      this.token,
+    );
+    return this.token;
+  }
+
+  error(code: DiagnosticCode, message: string, token: Token = this.token): void {
+    this.diagnostics.error(code, message, { start: token.start, end: token.end }, this.source.fileName);
+    if (this.speculating > 0) throw new SpeculationError();
+  }
+
+  /**
+   * Run `fn` speculatively. Any diagnostic raised while it runs (which aborts
+   * the attempt via `SpeculationError`) is rolled back along with the scanner
+   * position, so the parser can try a different production cleanly.
+   */
+  tryParse<T>(fn: () => T): T | undefined {
+    const mark = this.diagnostics.mark();
+    const state = this.saveState();
+    this.speculating++;
+    let result: T | undefined;
+    let ok = false;
+    try {
+      result = fn();
+      ok = true;
+    } catch (error) {
+      if (!(error instanceof SpeculationError)) throw error;
+    } finally {
+      this.speculating--;
+    }
+    if (!ok) {
+      this.diagnostics.reset(mark);
+      this.restoreState(state);
+      return undefined;
+    }
+    return result;
+  }
+
+  isIdentifierLike(token: Token): boolean {
+    if (token.kind === TokenKind.Identifier) return true;
+    switch (token.kind) {
+      case TokenKind.AsKeyword:
+      case TokenKind.SatisfiesKeyword:
+      case TokenKind.FromKeyword:
+      case TokenKind.TypeKeyword:
+      case TokenKind.GetKeyword:
+      case TokenKind.SetKeyword:
+      case TokenKind.AsyncKeyword:
+        return true;
+      default:
+        return false;
+    }
+  }
+
+  parseIdentifier(allowKeywords = false): Identifier {
+    const token = this.token;
+    if (token.kind === TokenKind.Identifier || (allowKeywords && this.isIdentifierLike(token))) {
+      this.nextToken();
+      return { kind: SyntaxKind.Identifier, text: token.text, start: token.start, end: token.end };
+    }
+    this.error(DiagnosticCode.ExpectedIdentifier, `Expected identifier but found '${token.text || token.kind}'`);
+    if (token.kind !== TokenKind.EndOfFile) this.nextToken();
+    return { kind: SyntaxKind.Identifier, text: token.text || "missing", start: token.start, end: token.end };
+  }
+
+  parseIdentifierName(): Identifier {
+    return this.parseIdentifier(/* allowKeywords */ true);
+  }
+
+  parseSemicolon(): void {
+    if (this.at(TokenKind.Semicolon)) {
+      this.nextToken();
+      return;
+    }
+    if (this.canInsertSemicolon()) return;
+    this.error(DiagnosticCode.ExpectedToken, "Expected ';'", this.token);
+  }
+
+  canInsertSemicolon(): boolean {
+    return this.at(TokenKind.EndOfFile) || this.at(TokenKind.CloseBrace) || this.token.precededByLineBreak;
+  }
+
+  parseDelimitedList<T>(close: TokenKind, parseElement: () => T): T[] {
+    const items: T[] = [];
+    while (this.token.kind !== close && !this.at(TokenKind.EndOfFile)) {
+      const before = this.consumed;
+      items.push(parseElement());
+      if (this.at(TokenKind.Comma)) this.nextToken();
+      else break;
+      if (this.consumed === before && this.token.kind !== close) this.nextToken();
+    }
+    this.parseExpected(close);
+    return items;
+  }
+
+  stringLiteralFromToken(token: Token): StringLiteral {
+    return { kind: SyntaxKind.StringLiteral, text: token.text, raw: token.text, value: String(token.value ?? ""), start: token.start, end: token.end };
+  }
+
+  // -- state save / restore (for speculative parsing) -----------------------
+
+  saveState(): ParserState {
+    return { index: this.index, tokens: this.tokens.slice(), pos: this.scanner.position, consumed: this.consumed };
+  }
+
+  restoreState(state: ParserState): void {
+    this.index = state.index;
+    this.tokens = state.tokens;
+    this.scanner.seek(state.pos);
+    this.consumed = state.consumed;
+  }
+}