xbintsc 0.1.6 → 0.1.9

package/src/driver/compiler.ts

@@ -13,6 +13,8 @@ import { DiagnosticBag, type Diagnostic } from "../diagnostics/diagnostic.js";
 import { SourceFile } from "../diagnostics/source.js";
 import { Parser } from "../parser/parser.js";
 import { generate } from "../codegen/llvm.js";
+import { SyntaxKind } from "../ast/nodes.js";
+import { bundleModules } from "./modules.js";
 import { createDefaultRegistry, type ExtensionRegistry } from "../extensions/registry.js";
 import { BuildCache, hashParts } from "./cache.js";
 import { findRuntimeDir } from "./paths.js";
@@ -94,9 +96,31 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
         ? resolve(options.output ?? join(outDir, baseName + ".o"))
         : executableName(absoluteEntry, outDir, options.output);
 
+  // -- front end -----------------------------------------------------------
+  const file = new SourceFile(absoluteEntry, sourceText);
+  const diagnostics = new DiagnosticBag();
+  const parser = new Parser(file, diagnostics);
+  let sourceFile = parser.parseSourceFile();
+
+  // Lower `import`/`export` by merging every reachable module into one file.
+  const isModule = sourceFile.statements.some(
+    (statement) =>
+      statement.kind === SyntaxKind.ImportDeclaration ||
+      statement.kind === SyntaxKind.ExportDeclaration ||
+      statement.kind === SyntaxKind.ExportAssignment,
+  );
+  let cacheText = sourceText;
+  if (isModule) {
+    const bundled = bundleModules(absoluteEntry, diagnostics);
+    if (bundled) {
+      sourceFile = bundled.sourceFile;
+      cacheText = bundled.text;
+    }
+  }
+
   const cacheKey = hashParts([
     COMPILER_VERSION,
-    sourceText,
+    cacheText,
     emit,
     optimize,
     process.platform,
@@ -105,15 +129,10 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 
   const cache = new BuildCache(cacheDir);
   const outputs = [outputPath];
-  if (!options.force && cache.isFresh(cacheKey, outputs)) {
+  if (!options.force && !diagnostics.hasErrors && cache.isFresh(cacheKey, outputs)) {
     return { outputPath, cached: true, diagnostics: [] };
   }
 
-  // -- front end -----------------------------------------------------------
-  const file = new SourceFile(absoluteEntry, sourceText);
-  const diagnostics = new DiagnosticBag();
-  const parser = new Parser(file, diagnostics);
-  const sourceFile = parser.parseSourceFile();
   const { ir } = generate(sourceFile, diagnostics, { builtins: registry.builtins() });
 
   if (options.verbose) {
@@ -139,7 +158,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   const runtimeDir = findRuntimeDir();
   mkdirSync(cacheDir, { recursive: true });
 
-  const { runtimeObject, extensionObjects } = ensureRuntimeObjects(runner, clang, runtimeDir, cacheDir, registry);
+  const { runtimeObjects, extensionObjects } = ensureRuntimeObjects(runner, clang, runtimeDir, cacheDir, registry);
 
   const objectPath = emit === "obj" ? outputPath : resolve(outDir, baseName + ".o");
   compileIr(runner, { clang, irPath, objectPath, optimize });
@@ -152,7 +171,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 
   link(runner, {
     clang,
-    objectPaths: [objectPath, runtimeObject, ...extensionObjects],
+    objectPaths: [objectPath, ...runtimeObjects, ...extensionObjects],
     outputPath,
     linkerFlags: [...(process.platform === "win32" ? [] : ["-lm"]), ...registry.linkerFlags()],
     optimize,
@@ -164,10 +183,22 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 }
 
 interface RuntimeObjects {
-  readonly runtimeObject: string;
+  readonly runtimeObjects: readonly string[];
   readonly extensionObjects: readonly string[];
 }
 
+/** Core runtime translation units (each compiled and cached independently). */
+const RUNTIME_SOURCES = [
+  "xt_alloc.c",
+  "xt_values.c",
+  "xt_containers.c",
+  "xt_stdlib.c",
+  "xt_stdlib2.c",
+  "xt_promise.c",
+  "xt_builtins.c",
+  "xt_io.c",
+] as const;
+
 /**
  * Compile the core runtime and every extension source, reusing cached object
  * files keyed on the C source hash. Returns the object paths to link.
@@ -189,9 +220,11 @@ function ensureRuntimeObjects(
     return objectPath;
   };
 
-  const runtimeObject = compileOne(join(runtimeDir, "xt_runtime.c"), "xt_runtime");
+  const runtimeObjects = RUNTIME_SOURCES.map((source) =>
+    compileOne(join(runtimeDir, source), basename(source, ".c")),
+  );
   const extensionObjects = registry
     .runtimeSources()
     .map((sourcePath, index) => compileOne(sourcePath, `ext_${index}_${basename(sourcePath, ".c")}`));
-  return { runtimeObject, extensionObjects };
+  return { runtimeObjects, extensionObjects };
 }

package/src/driver/modules.ts

@@ -0,0 +1,377 @@
+/**
+ * Source-level module bundler.
+ *
+ * xbintsc does not run a linker for modules, so `import`/`export` are lowered
+ * at the driver by merging every reachable module into a single source file.
+ * Each module's top-level bindings are renamed to a globally unique name and
+ * imported names are rewritten to the (renamed) exported binding.
+ *
+ * This is intentionally simple: it supports named imports/exports, `export
+ * default` for declarations, and `export ... from`. Namespace imports and
+ * `export *` are approximated, and circular graphs are reported as an error.
+ */
+
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import {
+  ModifierKind,
+  SyntaxKind,
+  type ExportAssignment,
+  type ExportDeclaration,
+  type Identifier,
+  type ImportDeclaration,
+  type Node,
+  type SourceFileNode,
+  type Statement,
+  type VariableDeclaration,
+  type VariableDeclarationList,
+  type VariableStatement,
+} from "../ast/nodes.js";
+import { bind, SymbolKind, type BindResult, type SymbolInfo } from "../binder/binder.js";
+import { DiagnosticBag, DiagnosticCode } from "../diagnostics/diagnostic.js";
+import { SourceFile } from "../diagnostics/source.js";
+import { Parser } from "../parser/parser.js";
+
+interface ModuleRecord {
+  readonly path: string;
+  readonly text: string;
+  readonly sourceFile: SourceFileNode;
+  readonly bind: BindResult;
+  readonly prefix: string;
+  /** exported name -> final (renamed) local name */
+  readonly exports: Map<string, string>;
+  /** original local name of every top-level symbol */
+  readonly originalNames: Map<number, string>;
+  /** final name of every top-level symbol */
+  readonly finalNames: Map<number, string>;
+}
+
+export interface BundleResult {
+  readonly sourceFile: SourceFileNode;
+  readonly text: string;
+  readonly moduleCount: number;
+}
+
+const RESOLVE_SUFFIXES = ["", ".ts", ".tsx", ".mts", ".cts", "/index.ts", "/index.tsx"];
+
+function resolveModule(fromDir: string, specifier: string): string | undefined {
+  const base = resolve(fromDir, specifier);
+  for (const suffix of RESOLVE_SUFFIXES) {
+    const candidate = base + suffix;
+    if (existsSync(candidate) && statSync(candidate).isFile()) return candidate;
+  }
+  return undefined;
+}
+
+function hasModifier(node: Node, kind: ModifierKind): boolean {
+  const modifiers = (node as { modifiers?: { modifierKind: ModifierKind }[] }).modifiers;
+  return !!modifiers && modifiers.some((modifier) => modifier.modifierKind === kind);
+}
+
+function declarationName(node: Node): Identifier | undefined {
+  switch (node.kind) {
+    case SyntaxKind.FunctionDeclaration:
+    case SyntaxKind.ClassDeclaration:
+    case SyntaxKind.EnumDeclaration:
+    case SyntaxKind.InterfaceDeclaration:
+    case SyntaxKind.TypeAliasDeclaration:
+    case SyntaxKind.ModuleDeclaration:
+      return (node as { name?: Identifier }).name;
+    case SyntaxKind.VariableDeclaration:
+      return (node as VariableDeclaration).name;
+    default:
+      return undefined;
+  }
+}
+
+/** Load, parse and bind the entry module and every module it imports. */
+function loadGraph(entryPath: string, diagnostics: DiagnosticBag): ModuleRecord[] | undefined {
+  const records = new Map<string, ModuleRecord>();
+  const order: ModuleRecord[] = [];
+  const visiting = new Set<string>();
+  let counter = 0;
+  let failed = false;
+
+  const load = (path: string): ModuleRecord | undefined => {
+    const existing = records.get(path);
+    if (existing) return existing;
+    if (visiting.has(path)) {
+      diagnostics.error(DiagnosticCode.CodegenError, `Circular import detected involving '${path}'`);
+      failed = true;
+      return undefined;
+    }
+    if (!existsSync(path)) {
+      diagnostics.error(DiagnosticCode.CodegenError, `Cannot find module '${path}'`);
+      failed = true;
+      return undefined;
+    }
+    visiting.add(path);
+    const text = readFileSync(path, "utf8");
+    const file = new SourceFile(path, text);
+    const parser = new Parser(file, diagnostics);
+    const sourceFile = parser.parseSourceFile();
+    const bindResult = bind(sourceFile);
+    const record: ModuleRecord = {
+      path,
+      text,
+      sourceFile,
+      bind: bindResult,
+      prefix: `m${counter++}_`,
+      exports: new Map(),
+      originalNames: new Map(),
+      finalNames: new Map(),
+    };
+    records.set(path, record);
+
+    for (const statement of sourceFile.statements) {
+      const specifier = moduleSpecifierOf(statement);
+      if (!specifier) continue;
+      const dependency = resolveModule(dirname(path), specifier);
+      if (!dependency) {
+        diagnostics.error(
+          DiagnosticCode.CodegenError,
+          `Cannot resolve module '${specifier}' from '${path}'`,
+          statement,
+          path,
+        );
+        failed = true;
+        continue;
+      }
+      load(dependency);
+    }
+    visiting.delete(path);
+    order.push(record);
+    return record;
+  };
+
+  load(entryPath);
+  if (failed) return undefined;
+  return order;
+}
+
+function moduleSpecifierOf(statement: Statement): string | undefined {
+  if (statement.kind === SyntaxKind.ImportDeclaration) {
+    return (statement as ImportDeclaration).moduleSpecifier.value;
+  }
+  if (statement.kind === SyntaxKind.ExportDeclaration) {
+    const specifier = (statement as ExportDeclaration).moduleSpecifier;
+    if (specifier) return specifier.value;
+  }
+  return undefined;
+}
+
+/** Rewrite a symbol's declaration and every reference to `finalName`. */
+function renameSymbol(symbol: SymbolInfo, finalName: string): void {
+  for (const declaration of symbol.declarations) {
+    const name = declarationName(declaration);
+    if (name) (name as { text: string }).text = finalName;
+  }
+  for (const reference of symbol.references) (reference as { text: string }).text = finalName;
+}
+
+/** Rewrite just the references of a symbol (used for imported names). */
+function renameReferences(symbol: SymbolInfo, finalName: string): void {
+  for (const reference of symbol.references) (reference as { text: string }).text = finalName;
+}
+
+function moduleScope(record: ModuleRecord) {
+  return record.bind.scopes.get(record.sourceFile);
+}
+
+function topLevelSymbols(record: ModuleRecord): SymbolInfo[] {
+  const scope = moduleScope(record);
+  if (!scope) return [];
+  return [...scope.symbols.values()];
+}
+
+/**
+ * Merge every module's statements into a single file. Dependencies come first
+ * (post-order over the import graph), the entry module last.
+ */
+export function bundleModules(entryPath: string, diagnostics: DiagnosticBag): BundleResult | undefined {
+  const records = loadGraph(entryPath, diagnostics);
+  if (!records) return undefined;
+
+  const byPath = new Map(records.map((record) => [record.path, record]));
+  const dependencyOf = (record: ModuleRecord, specifier: string): ModuleRecord | undefined => {
+    const resolved = resolveModule(dirname(record.path), specifier);
+    return resolved ? byPath.get(resolved) : undefined;
+  };
+
+  // Phase 1: record every top-level binding and pick a unique final name.
+  for (const record of records) {
+    for (const symbol of topLevelSymbols(record)) {
+      record.originalNames.set(symbol.id, symbol.name);
+      if (symbol.kind === SymbolKind.Import) continue;
+      record.finalNames.set(symbol.id, record.prefix + symbol.name);
+    }
+  }
+
+  // Phase 2: collect each module's exported names.
+  for (const record of records) {
+    const scope = moduleScope(record);
+    if (!scope) continue;
+    const symbolByName = scope.symbols;
+    for (const statement of record.sourceFile.statements) {
+      if (statement.kind === SyntaxKind.ExportDeclaration) {
+        const declaration = statement as ExportDeclaration;
+        if (declaration.exportClause && declaration.exportClause.kind === SyntaxKind.NamedExports) {
+          for (const specifier of declaration.exportClause.elements) {
+            const localName = specifier.propertyName?.text ?? specifier.name.text;
+            const exportedName = specifier.name.text;
+            if (declaration.moduleSpecifier) {
+              const dependency = dependencyOf(record, declaration.moduleSpecifier.value);
+              const target = dependency?.exports.get(localName);
+              if (target) record.exports.set(exportedName, target);
+            } else {
+              const symbol = symbolByName.get(localName);
+              if (symbol) {
+                const finalName = record.finalNames.get(symbol.id) ?? symbol.name;
+                record.exports.set(exportedName, finalName);
+              }
+            }
+          }
+        } else if (!declaration.exportClause && declaration.moduleSpecifier) {
+          // `export * from "..."`: copy every re-exported name.
+          const dependency = dependencyOf(record, declaration.moduleSpecifier.value);
+          if (dependency) for (const [name, value] of dependency.exports) record.exports.set(name, value);
+        }
+        continue;
+      }
+      if (statement.kind === SyntaxKind.ExportAssignment) {
+        record.exports.set("default", `${record.prefix}default`);
+        continue;
+      }
+      if (hasModifier(statement, ModifierKind.Default)) {
+        const name = declarationName(statement) ?? declarationName((statement as { declaration?: Node }).declaration ?? statement);
+        if (name) {
+          const symbol = symbolByName.get(name.text);
+          if (symbol) record.exports.set("default", record.finalNames.get(symbol.id) ?? symbol.name);
+        }
+        continue;
+      }
+      if (hasModifier(statement, ModifierKind.Export)) {
+        const names = exportedNames(statement);
+        for (const name of names) {
+          const symbol = symbolByName.get(name);
+          if (symbol) record.exports.set(name, record.finalNames.get(symbol.id) ?? symbol.name);
+        }
+      }
+    }
+  }
+
+  // Phase 3: rewrite imported names and record re-exports.
+  for (const record of records) {
+    const scope = moduleScope(record);
+    if (!scope) continue;
+    for (const statement of record.sourceFile.statements) {
+      if (statement.kind !== SyntaxKind.ImportDeclaration) continue;
+      const declaration = statement as ImportDeclaration;
+      const dependency = dependencyOf(record, declaration.moduleSpecifier.value);
+      const clause = declaration.importClause;
+      if (!clause || !dependency) continue;
+      if (clause.name) {
+        const target = dependency.exports.get("default") ?? dependency.exports.get("default");
+        const local = scope.symbols.get(clause.name.text);
+        if (target && local) renameReferences(local, target);
+      }
+      const bindings = clause.namedBindings;
+      if (bindings && bindings.kind === SyntaxKind.NamedImports) {
+        for (const specifier of bindings.elements) {
+          const importedName = specifier.propertyName?.text ?? specifier.name.text;
+          const target = dependency.exports.get(importedName);
+          const local = scope.symbols.get(specifier.name.text);
+          if (target && local) renameReferences(local, target);
+        }
+      }
+    }
+  }
+
+  // Phase 4: concatenate statements, dropping imports and pure export wrappers.
+  const merged: Statement[] = [];
+  const first = records[records.length - 1]!;
+  for (const record of records) {
+    for (const statement of record.sourceFile.statements) {
+      if (statement.kind === SyntaxKind.ImportDeclaration) continue;
+      if (statement.kind === SyntaxKind.ExportDeclaration) {
+        const declaration = statement as ExportDeclaration;
+        const inner = (declaration as unknown as { declaration?: Statement }).declaration;
+        if (inner) merged.push(inner);
+        continue;
+      }
+      if (statement.kind === SyntaxKind.ExportAssignment) {
+        const assignment = statement as ExportAssignment;
+        merged.push(exportAssignmentToStatement(record, assignment));
+        continue;
+      }
+      merged.push(statement);
+    }
+  }
+
+  // Phase 5: rename every top-level binding (declaration + references) to its
+  // unique final name, now that import references have been rewritten.
+  for (const record of records) {
+    for (const symbol of topLevelSymbols(record)) {
+      if (symbol.kind === SymbolKind.Import) continue;
+      const finalName = record.finalNames.get(symbol.id);
+      if (finalName) renameSymbol(symbol, finalName);
+    }
+  }
+
+  const sourceFile: SourceFileNode = {
+    kind: SyntaxKind.SourceFile,
+    statements: merged,
+    fileName: first.path,
+    text: records.map((record) => record.text).join("\n"),
+    start: 0,
+    end: 0,
+  };
+  return { sourceFile, text: sourceFile.text, moduleCount: records.length };
+}
+
+function exportedNames(statement: Statement): string[] {
+  const names: string[] = [];
+  if (statement.kind === SyntaxKind.VariableStatement) {
+    for (const declaration of (statement as VariableStatement).declarationList.declarations) {
+      if (declaration.name) names.push(declaration.name.text);
+    }
+  } else {
+    const name = declarationName(statement);
+    if (name) names.push(name.text);
+  }
+  return names;
+}
+
+/** Turn `export default expr` into a synthetic `const` declaration. */
+function exportAssignmentToStatement(record: ModuleRecord, assignment: ExportAssignment): Statement {
+  const name: Identifier = {
+    kind: SyntaxKind.Identifier,
+    text: `${record.prefix}default`,
+    start: assignment.start,
+    end: assignment.start,
+  };
+  record.exports.set("default", name.text);
+  const declaration: VariableDeclaration = {
+    kind: SyntaxKind.VariableDeclaration,
+    name,
+    exclamation: false,
+    initializer: assignment.expression,
+    start: assignment.start,
+    end: assignment.end,
+  };
+  const list: VariableDeclarationList = {
+    kind: SyntaxKind.VariableDeclarationList,
+    declarationKind: "const" as const,
+    declarations: [declaration],
+    start: assignment.start,
+    end: assignment.end,
+  };
+  const statement: VariableStatement = {
+    kind: SyntaxKind.VariableStatement,
+    declarationList: list,
+    modifiers: [],
+    start: assignment.start,
+    end: assignment.end,
+  };
+  return statement;
+}

package/src/lexer/char.ts

@@ -0,0 +1,116 @@
+/**
+ * Character classification used by the scanner.
+ *
+ * Keeping the code-point constants and the `is*` predicates in their own module
+ * keeps the scanner focused on tokenization and makes the low-level rules easy
+ * to test in isolation.
+ */
+
+export const enum Char {
+  Null = 0,
+  Tab = 9,
+  LineFeed = 10,
+  VerticalTab = 11,
+  FormFeed = 12,
+  CarriageReturn = 13,
+  Space = 32,
+  Exclamation = 33,
+  DoubleQuote = 34,
+  Hash = 35,
+  Dollar = 36,
+  Percent = 37,
+  Ampersand = 38,
+  SingleQuote = 39,
+  OpenParen = 40,
+  CloseParen = 41,
+  Asterisk = 42,
+  Plus = 43,
+  Comma = 44,
+  Minus = 45,
+  Dot = 46,
+  Slash = 47,
+  Zero = 48,
+  Nine = 57,
+  Colon = 58,
+  Semicolon = 59,
+  LessThan = 60,
+  Equals = 61,
+  GreaterThan = 62,
+  Question = 63,
+  At = 64,
+  UpperA = 65,
+  UpperB = 66,
+  UpperE = 69,
+  UpperF = 70,
+  UpperO = 79,
+  UpperX = 88,
+  UpperZ = 90,
+  OpenBracket = 91,
+  Backslash = 92,
+  CloseBracket = 93,
+  Caret = 94,
+  Underscore = 95,
+  Backtick = 96,
+  LowerA = 97,
+  LowerB = 98,
+  LowerE = 101,
+  LowerF = 102,
+  LowerN = 110,
+  LowerO = 111,
+  LowerR = 114,
+  LowerT = 116,
+  LowerU = 117,
+  LowerX = 120,
+  LowerZ = 122,
+  OpenBrace = 123,
+  Bar = 124,
+  CloseBrace = 125,
+  Tilde = 126,
+  NonBreakingSpace = 0xa0,
+  LineSeparator = 0x2028,
+  ParagraphSeparator = 0x2029,
+  ByteOrderMark = 0xfeff,
+}
+
+export function isLineBreak(ch: number): boolean {
+  return ch === Char.LineFeed || ch === Char.CarriageReturn || ch === Char.LineSeparator || ch === Char.ParagraphSeparator;
+}
+
+const WHITESPACE_UNICODE = new Set<number>([
+  0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200a, 0x202f, 0x205f,
+  0x3000,
+]);
+
+export function isWhiteSpace(ch: number): boolean {
+  return (
+    ch === Char.Space ||
+    ch === Char.Tab ||
+    ch === Char.VerticalTab ||
+    ch === Char.FormFeed ||
+    ch === Char.NonBreakingSpace ||
+    ch === Char.ByteOrderMark ||
+    (ch > 0x1680 && WHITESPACE_UNICODE.has(ch))
+  );
+}
+
+export function isDigit(ch: number): boolean {
+  return ch >= Char.Zero && ch <= Char.Nine;
+}
+
+export function isHexDigit(ch: number): boolean {
+  return isDigit(ch) || (ch >= Char.LowerA && ch <= Char.LowerF) || (ch >= Char.UpperA && ch <= Char.UpperF);
+}
+
+export function isIdentifierStart(ch: number): boolean {
+  return (
+    (ch >= Char.LowerA && ch <= Char.LowerZ) ||
+    (ch >= Char.UpperA && ch <= Char.UpperZ) ||
+    ch === Char.Dollar ||
+    ch === Char.Underscore ||
+    ch > 0x7f
+  );
+}
+
+export function isIdentifierPart(ch: number): boolean {
+  return isIdentifierStart(ch) || isDigit(ch) || ch === Char.Backslash || ch > 0x7f;
+}