xbintsc 0.1.3 → 0.1.8

package/src/driver/compiler.ts

@@ -139,7 +139,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   const runtimeDir = findRuntimeDir();
   mkdirSync(cacheDir, { recursive: true });
 
-  const { runtimeObject, extensionObjects } = ensureRuntimeObjects(runner, clang, runtimeDir, cacheDir, registry);
+  const { runtimeObjects, extensionObjects } = ensureRuntimeObjects(runner, clang, runtimeDir, cacheDir, registry);
 
   const objectPath = emit === "obj" ? outputPath : resolve(outDir, baseName + ".o");
   compileIr(runner, { clang, irPath, objectPath, optimize });
@@ -152,7 +152,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 
   link(runner, {
     clang,
-    objectPaths: [objectPath, runtimeObject, ...extensionObjects],
+    objectPaths: [objectPath, ...runtimeObjects, ...extensionObjects],
     outputPath,
     linkerFlags: [...(process.platform === "win32" ? [] : ["-lm"]), ...registry.linkerFlags()],
     optimize,
@@ -164,10 +164,20 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 }
 
 interface RuntimeObjects {
-  readonly runtimeObject: string;
+  readonly runtimeObjects: readonly string[];
   readonly extensionObjects: readonly string[];
 }
 
+/** Core runtime translation units (each compiled and cached independently). */
+const RUNTIME_SOURCES = [
+  "xt_alloc.c",
+  "xt_values.c",
+  "xt_containers.c",
+  "xt_stdlib.c",
+  "xt_builtins.c",
+  "xt_io.c",
+] as const;
+
 /**
  * Compile the core runtime and every extension source, reusing cached object
  * files keyed on the C source hash. Returns the object paths to link.
@@ -189,9 +199,11 @@ function ensureRuntimeObjects(
     return objectPath;
   };
 
-  const runtimeObject = compileOne(join(runtimeDir, "xt_runtime.c"), "xt_runtime");
+  const runtimeObjects = RUNTIME_SOURCES.map((source) =>
+    compileOne(join(runtimeDir, source), basename(source, ".c")),
+  );
   const extensionObjects = registry
     .runtimeSources()
     .map((sourcePath, index) => compileOne(sourcePath, `ext_${index}_${basename(sourcePath, ".c")}`));
-  return { runtimeObject, extensionObjects };
+  return { runtimeObjects, extensionObjects };
 }

package/src/lexer/char.ts

@@ -0,0 +1,116 @@
+/**
+ * Character classification used by the scanner.
+ *
+ * Keeping the code-point constants and the `is*` predicates in their own module
+ * keeps the scanner focused on tokenization and makes the low-level rules easy
+ * to test in isolation.
+ */
+
+export const enum Char {
+  Null = 0,
+  Tab = 9,
+  LineFeed = 10,
+  VerticalTab = 11,
+  FormFeed = 12,
+  CarriageReturn = 13,
+  Space = 32,
+  Exclamation = 33,
+  DoubleQuote = 34,
+  Hash = 35,
+  Dollar = 36,
+  Percent = 37,
+  Ampersand = 38,
+  SingleQuote = 39,
+  OpenParen = 40,
+  CloseParen = 41,
+  Asterisk = 42,
+  Plus = 43,
+  Comma = 44,
+  Minus = 45,
+  Dot = 46,
+  Slash = 47,
+  Zero = 48,
+  Nine = 57,
+  Colon = 58,
+  Semicolon = 59,
+  LessThan = 60,
+  Equals = 61,
+  GreaterThan = 62,
+  Question = 63,
+  At = 64,
+  UpperA = 65,
+  UpperB = 66,
+  UpperE = 69,
+  UpperF = 70,
+  UpperO = 79,
+  UpperX = 88,
+  UpperZ = 90,
+  OpenBracket = 91,
+  Backslash = 92,
+  CloseBracket = 93,
+  Caret = 94,
+  Underscore = 95,
+  Backtick = 96,
+  LowerA = 97,
+  LowerB = 98,
+  LowerE = 101,
+  LowerF = 102,
+  LowerN = 110,
+  LowerO = 111,
+  LowerR = 114,
+  LowerT = 116,
+  LowerU = 117,
+  LowerX = 120,
+  LowerZ = 122,
+  OpenBrace = 123,
+  Bar = 124,
+  CloseBrace = 125,
+  Tilde = 126,
+  NonBreakingSpace = 0xa0,
+  LineSeparator = 0x2028,
+  ParagraphSeparator = 0x2029,
+  ByteOrderMark = 0xfeff,
+}
+
+export function isLineBreak(ch: number): boolean {
+  return ch === Char.LineFeed || ch === Char.CarriageReturn || ch === Char.LineSeparator || ch === Char.ParagraphSeparator;
+}
+
+const WHITESPACE_UNICODE = new Set<number>([
+  0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200a, 0x202f, 0x205f,
+  0x3000,
+]);
+
+export function isWhiteSpace(ch: number): boolean {
+  return (
+    ch === Char.Space ||
+    ch === Char.Tab ||
+    ch === Char.VerticalTab ||
+    ch === Char.FormFeed ||
+    ch === Char.NonBreakingSpace ||
+    ch === Char.ByteOrderMark ||
+    (ch > 0x1680 && WHITESPACE_UNICODE.has(ch))
+  );
+}
+
+export function isDigit(ch: number): boolean {
+  return ch >= Char.Zero && ch <= Char.Nine;
+}
+
+export function isHexDigit(ch: number): boolean {
+  return isDigit(ch) || (ch >= Char.LowerA && ch <= Char.LowerF) || (ch >= Char.UpperA && ch <= Char.UpperF);
+}
+
+export function isIdentifierStart(ch: number): boolean {
+  return (
+    (ch >= Char.LowerA && ch <= Char.LowerZ) ||
+    (ch >= Char.UpperA && ch <= Char.UpperZ) ||
+    ch === Char.Dollar ||
+    ch === Char.Underscore ||
+    ch > 0x7f
+  );
+}
+
+export function isIdentifierPart(ch: number): boolean {
+  return isIdentifierStart(ch) || isDigit(ch) || ch === Char.Backslash || ch > 0x7f;
+}

package/src/lexer/scanner.ts

@@ -1,115 +1,15 @@
 import { DiagnosticBag, DiagnosticCode } from "../diagnostics/diagnostic.js";
 import type { SourceFile } from "../diagnostics/source.js";
 import { KEYWORDS, Token, TokenKind } from "./token.js";
-
-const enum Char {
-  Null = 0,
-  Tab = 9,
-  LineFeed = 10,
-  VerticalTab = 11,
-  FormFeed = 12,
-  CarriageReturn = 13,
-  Space = 32,
-  Exclamation = 33,
-  DoubleQuote = 34,
-  Hash = 35,
-  Dollar = 36,
-  Percent = 37,
-  Ampersand = 38,
-  SingleQuote = 39,
-  OpenParen = 40,
-  CloseParen = 41,
-  Asterisk = 42,
-  Plus = 43,
-  Comma = 44,
-  Minus = 45,
-  Dot = 46,
-  Slash = 47,
-  Zero = 48,
-  Nine = 57,
-  Colon = 58,
-  Semicolon = 59,
-  LessThan = 60,
-  Equals = 61,
-  GreaterThan = 62,
-  Question = 63,
-  At = 64,
-  UpperA = 65,
-  UpperB = 66,
-  UpperE = 69,
-  UpperF = 70,
-  UpperO = 79,
-  UpperX = 88,
-  UpperZ = 90,
-  OpenBracket = 91,
-  Backslash = 92,
-  CloseBracket = 93,
-  Caret = 94,
-  Underscore = 95,
-  Backtick = 96,
-  LowerA = 97,
-  LowerB = 98,
-  LowerE = 101,
-  LowerF = 102,
-  LowerN = 110,
-  LowerO = 111,
-  LowerR = 114,
-  LowerT = 116,
-  LowerU = 117,
-  LowerX = 120,
-  LowerZ = 122,
-  OpenBrace = 123,
-  Bar = 124,
-  CloseBrace = 125,
-  Tilde = 126,
-  NonBreakingSpace = 0xa0,
-  LineSeparator = 0x2028,
-  ParagraphSeparator = 0x2029,
-  ByteOrderMark = 0xfeff,
-}
-
-function isLineBreak(ch: number): boolean {
-  return ch === Char.LineFeed || ch === Char.CarriageReturn || ch === Char.LineSeparator || ch === Char.ParagraphSeparator;
-}
-
-function isWhiteSpace(ch: number): boolean {
-  return (
-    ch === Char.Space ||
-    ch === Char.Tab ||
-    ch === Char.VerticalTab ||
-    ch === Char.FormFeed ||
-    ch === Char.NonBreakingSpace ||
-    ch === Char.ByteOrderMark ||
-    (ch > 0x1680 && WHITESPACE_UNICODE.has(ch))
-  );
-}
-
-const WHITESPACE_UNICODE = new Set<number>([
-  0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200a, 0x202f, 0x205f,
-  0x3000,
-]);
-
-function isDigit(ch: number): boolean {
-  return ch >= Char.Zero && ch <= Char.Nine;
-}
-
-function isHexDigit(ch: number): boolean {
-  return isDigit(ch) || (ch >= Char.LowerA && ch <= Char.LowerF) || (ch >= Char.UpperA && ch <= Char.UpperF);
-}
-
-function isIdentifierStart(ch: number): boolean {
-  return (
-    (ch >= Char.LowerA && ch <= Char.LowerZ) ||
-    (ch >= Char.UpperA && ch <= Char.UpperZ) ||
-    ch === Char.Dollar ||
-    ch === Char.Underscore ||
-    ch > 0x7f
-  );
-}
-
-function isIdentifierPart(ch: number): boolean {
-  return isIdentifierStart(ch) || isDigit(ch) || ch === Char.Backslash || ch > 0x7f;
-}
+import {
+  Char,
+  isDigit,
+  isHexDigit,
+  isIdentifierPart,
+  isIdentifierStart,
+  isLineBreak,
+  isWhiteSpace,
+} from "./char.js";
 
 /**
  * Hand-written scanner for the TypeScript language. It produces a flat token

package/src/parser/context.ts

@@ -0,0 +1,196 @@
+/**
+ * Shared state and low-level token plumbing for the parser.
+ *
+ * The parser's method groups (statements, expressions, types, modules) are
+ * defined as separate modules and installed onto `Parser.prototype`. This class
+ * holds the state they all operate on plus the token accessors every group
+ * needs.
+ */
+
+import { DiagnosticBag, DiagnosticCode } from "../diagnostics/diagnostic.js";
+import type { SourceFile } from "../diagnostics/source.js";
+import { Scanner } from "../lexer/scanner.js";
+import { Token, TokenKind } from "../lexer/token.js";
+import { SyntaxKind } from "../ast/kinds.js";
+import type { Identifier, StringLiteral } from "../ast/nodes.js";
+import { SpeculationError } from "./speculation.js";
+
+export interface ParserState {
+  index: number;
+  tokens: Token[];
+  pos: number;
+  consumed: number;
+}
+
+export class ParserContext {
+  readonly scanner: Scanner;
+  readonly diagnostics: DiagnosticBag;
+  readonly source: SourceFile;
+  tokens: Token[];
+  index = 0;
+  /** Non-zero while a speculative parse is running; errors abort the attempt. */
+  speculating = 0;
+  /** Monotonic count of tokens consumed; used to detect lack of progress. */
+  consumed = 0;
+
+  constructor(source: SourceFile, diagnostics: DiagnosticBag) {
+    this.source = source;
+    this.diagnostics = diagnostics;
+    this.scanner = new Scanner(source, diagnostics);
+    this.tokens = [this.scanner.nextToken()];
+  }
+
+  // -- token access ---------------------------------------------------------
+
+  get token(): Token {
+    return this.tokens[this.index]!;
+  }
+
+  /**
+   * Compare the current token kind without triggering TypeScript's control-flow
+   * narrowing of `this.token.kind`, which would otherwise be invalidated by the
+   * calls to `nextToken()` that occur between checks.
+   */
+  at(kind: TokenKind): boolean {
+    return this.tokens[this.index]!.kind === kind;
+  }
+
+  atAhead(k: number, kind: TokenKind): boolean {
+    return this.lookAhead(k).kind === kind;
+  }
+
+  lookAhead(k: number): Token {
+    while (this.tokens.length <= this.index + k) {
+      this.tokens.push(this.scanner.nextToken());
+    }
+    return this.tokens[this.index + k]!;
+  }
+
+  nextToken(): Token {
+    this.consumed++;
+    const current = this.token;
+    if (this.index + 1 < this.tokens.length) {
+      this.index++;
+    } else {
+      this.tokens = [this.scanner.nextToken()];
+      this.index = 0;
+    }
+    return current;
+  }
+
+  parseExpected(kind: TokenKind, message?: string): Token {
+    if (this.token.kind === kind) return this.nextToken();
+    this.error(
+      DiagnosticCode.ExpectedToken,
+      message ?? `Expected '${kind}' but found '${this.token.text || this.token.kind}'`,
+      this.token,
+    );
+    return this.token;
+  }
+
+  error(code: DiagnosticCode, message: string, token: Token = this.token): void {
+    this.diagnostics.error(code, message, { start: token.start, end: token.end }, this.source.fileName);
+    if (this.speculating > 0) throw new SpeculationError();
+  }
+
+  /**
+   * Run `fn` speculatively. Any diagnostic raised while it runs (which aborts
+   * the attempt via `SpeculationError`) is rolled back along with the scanner
+   * position, so the parser can try a different production cleanly.
+   */
+  tryParse<T>(fn: () => T): T | undefined {
+    const mark = this.diagnostics.mark();
+    const state = this.saveState();
+    this.speculating++;
+    let result: T | undefined;
+    let ok = false;
+    try {
+      result = fn();
+      ok = true;
+    } catch (error) {
+      if (!(error instanceof SpeculationError)) throw error;
+    } finally {
+      this.speculating--;
+    }
+    if (!ok) {
+      this.diagnostics.reset(mark);
+      this.restoreState(state);
+      return undefined;
+    }
+    return result;
+  }
+
+  isIdentifierLike(token: Token): boolean {
+    if (token.kind === TokenKind.Identifier) return true;
+    switch (token.kind) {
+      case TokenKind.AsKeyword:
+      case TokenKind.SatisfiesKeyword:
+      case TokenKind.FromKeyword:
+      case TokenKind.TypeKeyword:
+      case TokenKind.GetKeyword:
+      case TokenKind.SetKeyword:
+      case TokenKind.AsyncKeyword:
+        return true;
+      default:
+        return false;
+    }
+  }
+
+  parseIdentifier(allowKeywords = false): Identifier {
+    const token = this.token;
+    if (token.kind === TokenKind.Identifier || (allowKeywords && this.isIdentifierLike(token))) {
+      this.nextToken();
+      return { kind: SyntaxKind.Identifier, text: token.text, start: token.start, end: token.end };
+    }
+    this.error(DiagnosticCode.ExpectedIdentifier, `Expected identifier but found '${token.text || token.kind}'`);
+    if (token.kind !== TokenKind.EndOfFile) this.nextToken();
+    return { kind: SyntaxKind.Identifier, text: token.text || "missing", start: token.start, end: token.end };
+  }
+
+  parseIdentifierName(): Identifier {
+    return this.parseIdentifier(/* allowKeywords */ true);
+  }
+
+  parseSemicolon(): void {
+    if (this.at(TokenKind.Semicolon)) {
+      this.nextToken();
+      return;
+    }
+    if (this.canInsertSemicolon()) return;
+    this.error(DiagnosticCode.ExpectedToken, "Expected ';'", this.token);
+  }
+
+  canInsertSemicolon(): boolean {
+    return this.at(TokenKind.EndOfFile) || this.at(TokenKind.CloseBrace) || this.token.precededByLineBreak;
+  }
+
+  parseDelimitedList<T>(close: TokenKind, parseElement: () => T): T[] {
+    const items: T[] = [];
+    while (this.token.kind !== close && !this.at(TokenKind.EndOfFile)) {
+      const before = this.consumed;
+      items.push(parseElement());
+      if (this.at(TokenKind.Comma)) this.nextToken();
+      else break;
+      if (this.consumed === before && this.token.kind !== close) this.nextToken();
+    }
+    this.parseExpected(close);
+    return items;
+  }
+
+  stringLiteralFromToken(token: Token): StringLiteral {
+    return { kind: SyntaxKind.StringLiteral, text: token.text, raw: token.text, value: String(token.value ?? ""), start: token.start, end: token.end };
+  }
+
+  // -- state save / restore (for speculative parsing) -----------------------
+
+  saveState(): ParserState {
+    return { index: this.index, tokens: this.tokens.slice(), pos: this.scanner.position, consumed: this.consumed };
+  }
+
+  restoreState(state: ParserState): void {
+    this.index = state.index;
+    this.tokens = state.tokens;
+    this.scanner.seek(state.pos);
+    this.consumed = state.consumed;
+  }
+}