xbintsc 0.3.81 → 0.5.6

package/src/driver/bundler/merge.ts

@@ -169,6 +169,32 @@ export function bundleModules(
 
   // Phase 4: concatenate statements, dropping imports and pure export wrappers.
   const merged: Statement[] = [];
+  /* Every node keeps the module it was parsed from, so a later codegen
+   * diagnostic can name the right file and line instead of pointing into the
+   * entry module (whose text happens to start at the same offsets). */
+  const origins = new Map<object, string>();
+  const trackOrigin = (record: ModuleRecord, root: Node): void => {
+    const visit = (node: Node): void => {
+      origins.set(node, record.path);
+      const fields = node as unknown as Record<string, unknown>;
+      for (const key of Object.keys(fields)) {
+        if (key === "kind" || key === "start" || key === "end" || key === "text" || key === "value" || key === "raw") {
+          continue;
+        }
+        const value = fields[key];
+        if (Array.isArray(value)) {
+          for (const child of value) if (isNode(child)) visit(child);
+        } else if (isNode(value)) {
+          visit(value);
+        }
+      }
+    };
+    visit(root);
+  };
+  const push = (record: ModuleRecord, statement: Statement): void => {
+    trackOrigin(record, statement);
+    merged.push(statement);
+  };
   const first = records[records.length - 1]!;
   for (const record of records) {
     for (const statement of record.sourceFile.statements) {
@@ -181,25 +207,25 @@ export function bundleModules(
           externalSpecifiers,
         );
         if (resolution.kind === "external") {
-          merged.push(statement);
+          push(record, statement);
         } else {
           const synthetic = namespaceStatements.get(statement);
-          if (synthetic) merged.push(synthetic);
+          if (synthetic) push(record, synthetic);
         }
         continue;
       }
       if (statement.kind === SyntaxKind.ExportDeclaration) {
         const declaration = statement as ExportDeclaration;
         const inner = (declaration as unknown as { declaration?: Statement }).declaration;
-        if (inner) merged.push(inner);
+        if (inner) push(record, inner);
         continue;
       }
       if (statement.kind === SyntaxKind.ExportAssignment) {
         const assignment = statement as ExportAssignment;
-        merged.push(exportAssignmentToStatement(record, assignment));
+        push(record, exportAssignmentToStatement(record, assignment));
         continue;
       }
-      merged.push(statement);
+      push(record, statement);
     }
   }
 
@@ -217,6 +243,9 @@ export function bundleModules(
     kind: SyntaxKind.SourceFile,
     statements: merged,
     fileName: first.path,
+    origins,
+    moduleTexts: new Map(records.map((record) => [record.path, record.text])),
+    modulePrefixes: records.map((record) => record.prefix),
     text: records.map((record) => record.text).join("\n"),
     start: 0,
     end: 0,
@@ -248,6 +277,11 @@ function collectBindingIdentifiers(name: BindingName, out: string[]): void {
   }
 }
 
+/** True for anything shaped like an AST node (constructed nodes carry no trivia). */
+function isNode(value: unknown): value is Node {
+  return !!value && typeof value === "object" && "kind" in (value as object);
+}
+
 /** Lower `import * as ns from "./mod"` to `const m0_ns = { ...exports }`. */
 function namespaceImportToStatement(
   statement: Statement,

package/src/driver/compiler.ts

@@ -18,6 +18,7 @@ import { SyntaxKind } from "../ast/nodes.js";
 import { bundleModules } from "./modules.js";
 import { createDefaultRegistry, type ExtensionRegistry } from "../extensions/registry.js";
 import { BuildCache, hashParts } from "./cache.js";
+import { compilerFingerprint, registryFingerprint, runtimeFingerprint } from "./fingerprint.js";
 import { findRuntimeDir } from "./paths.js";
 import { findRuntimeLibrary } from "./runtime-lib.js";
 import { resolveToolchain } from "./toolchain-provider.js";
@@ -31,10 +32,17 @@ import {
   compileIr,
   link,
   realRunner,
+  RUNTIME_CFLAGS,
   type Runner,
 } from "./toolchain.js";
 
-export const COMPILER_VERSION = "0.1.0";
+/**
+ * The compiler's own version. It is part of the incremental cache key, so it
+ * must be bumped for every release (a test enforces that it matches
+ * `package.json`); a stale version would let an upgraded compiler reuse binaries
+ * produced by the previous one.
+ */
+export const COMPILER_VERSION = "0.5.6";
 
 export type EmitKind = "exe" | "obj" | "ir";
 
@@ -57,6 +65,14 @@ export interface BuildOptions {
   readonly icon?: string;
   /** Application metadata: name, icon and macOS bundle settings. */
   readonly app?: AppConfig;
+  /** Emit LLVM debug metadata so `-g` produces DWARF/CodeView (default false). */
+  readonly debug?: boolean;
+}
+
+/** One phase of a build, in milliseconds (reported by `--time`). */
+export interface PhaseTiming {
+  readonly phase: "frontend" | "codegen" | "runtime" | "clang" | "link";
+  readonly ms: number;
 }
 
 export interface BuildResult {
@@ -67,6 +83,8 @@ export interface BuildResult {
   readonly ir?: string;
   /** macOS `.app` bundle produced by `app.bundle` (darwin only). */
   readonly bundlePath?: string;
+  /** Per-phase timings, collected for `xbintsc build --time`. */
+  readonly timings?: readonly PhaseTiming[];
 }
 
 export interface CompileStringResult {
@@ -75,7 +93,12 @@ export interface CompileStringResult {
 }
 
 /** Compile TypeScript source text to LLVM IR without touching the toolchain. */
-export function compileString(source: string, fileName = "input.ts", extensions?: ExtensionRegistry): CompileStringResult {
+export function compileString(
+  source: string,
+  fileName = "input.ts",
+  extensions?: ExtensionRegistry,
+  options: { readonly debug?: boolean } = {},
+): CompileStringResult {
   const file = new SourceFile(fileName, source);
   const diagnostics = new DiagnosticBag();
   const parser = new Parser(file, diagnostics);
@@ -85,6 +108,7 @@ export function compileString(source: string, fileName = "input.ts", extensions?
     builtins: registry.builtins(),
     modules: registry.modules(),
     moduleHints: registry.moduleHints(),
+    debug: options.debug === true,
   });
   return { ir, diagnostics: diagnostics.diagnostics };
 }
@@ -106,7 +130,11 @@ function externalModuleSpecifiers(registry: ExtensionRegistry): Set<string> {
  * Compile a file to LLVM IR, bundling every reachable relative module first so
  * that imports from other TypeScript sources resolve exactly like `build`.
  */
-export function compileEntry(entryPath: string, extensions?: ExtensionRegistry): CompileStringResult {
+export function compileEntry(
+  entryPath: string,
+  extensions?: ExtensionRegistry,
+  options: { readonly debug?: boolean } = {},
+): CompileStringResult {
   const absoluteEntry = resolve(entryPath);
   const sourceText = sourceTextBytes(readFileSync(absoluteEntry, "utf8"));
   const file = new SourceFile(absoluteEntry, sourceText);
@@ -128,6 +156,7 @@ export function compileEntry(entryPath: string, extensions?: ExtensionRegistry):
     builtins: registry.builtins(),
     modules: registry.modules(),
     moduleHints: registry.moduleHints(),
+    debug: options.debug === true,
   });
   return { ir, diagnostics: diagnostics.diagnostics };
 }
@@ -173,6 +202,8 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 
   const absoluteEntry = resolve(entryPath);
   const sourceText = sourceTextBytes(readFileSync(absoluteEntry, "utf8"));
+  const buildStarted = Date.now();
+  const timings: PhaseTiming[] = [];
 
   const baseName = basename(absoluteEntry, extname(absoluteEntry));
   const outputPath =
@@ -203,6 +234,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
       cacheText = bundled.text;
     }
   }
+  timings.push({ phase: "frontend", ms: Date.now() - buildStarted });
 
   const app = options.app ?? {};
   const iconPath = options.icon ?? app.icon;
@@ -222,6 +254,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   const runtimeDir = findRuntimeDir();
   const cacheKey = hashParts([
     COMPILER_VERSION,
+    compilerFingerprint(),
     cacheText,
     emit,
     optimize,
@@ -232,6 +265,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     app.name ?? "",
     String(app.bundle ?? false),
     app.bundleId ?? "",
+    String(options.debug === true),
   ]);
 
   // A `.app` bundle is deterministic from the output path and name, so it can be
@@ -243,21 +277,24 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   const cache = new BuildCache(cacheDir);
   const outputs = bundle ? [outputPath, bundle] : [outputPath];
   if (!options.force && !diagnostics.hasErrors && cache.isFresh(cacheKey, outputs)) {
-    return { outputPath, cached: true, diagnostics: [], ...(bundle ? { bundlePath: bundle } : {}) };
+    return { outputPath, cached: true, diagnostics: [], timings, ...(bundle ? { bundlePath: bundle } : {}) };
   }
 
+  const codegenStarted = Date.now();
   const { ir } = generate(sourceFile, diagnostics, {
     builtins: registry.builtins(),
     modules: registry.modules(),
     moduleHints: registry.moduleHints(),
+    debug: options.debug === true,
   });
+  timings.push({ phase: "codegen", ms: Date.now() - codegenStarted });
 
   if (options.verbose) {
     process.stderr.write(`xbintsc: generated ${ir.length} bytes of LLVM IR\n`);
   }
 
   if (diagnostics.hasErrors) {
-    return { outputPath, cached: false, diagnostics: diagnostics.diagnostics, ir };
+    return { outputPath, cached: false, diagnostics: diagnostics.diagnostics, ir, timings };
   }
 
   mkdirSync(outDir, { recursive: true });
@@ -272,10 +309,11 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   if (emit === "ir") {
     cache.record(cacheKey, outputs);
     cache.save();
-    return { outputPath, irPath, cached: false, diagnostics: [], ir };
+    return { outputPath, irPath, cached: false, diagnostics: [], ir, timings };
   }
 
   // -- toolchain -----------------------------------------------------------
+  const clangStarted = Date.now();
   const toolchain = options.clang
     ? { clang: options.clang, linkerArgs: [] as string[], env: {} as Record<string, string> }
     : resolveToolchain(runner);
@@ -283,14 +321,16 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   mkdirSync(cacheDir, { recursive: true });
 
   const objectPath = emit === "obj" ? outputPath : resolve(outDir, baseName + ".o");
-  compileIr(runner, { clang, irPath, objectPath, optimize, env: toolchain.env });
+  compileIr(runner, { clang, irPath, objectPath, optimize, env: toolchain.env, debug: options.debug === true });
+  timings.push({ phase: "clang", ms: Date.now() - clangStarted });
 
   if (emit === "obj") {
     cache.record(cacheKey, outputs);
     cache.save();
-    return { outputPath, irPath, cached: false, diagnostics: [], ir };
+    return { outputPath, irPath, cached: false, diagnostics: [], ir, timings };
   }
 
+  const runtimeStarted = Date.now();
   const { runtimeObjects, extensionObjects, nativeObjects } = ensureRuntimeObjects(
     runner,
     clang,
@@ -300,6 +340,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     resolvePreferPrebuilt(options.preferPrebuilt),
     toolchain.env,
   );
+  timings.push({ phase: "runtime", ms: Date.now() - runtimeStarted });
 
   /* On Windows a GUI program links as a **Windows-subsystem** executable, so
    * launching it does not flash a console window. That subsystem's C runtime
@@ -336,6 +377,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     }
   }
 
+  const linkStarted = Date.now();
   link(runner, {
     clang,
     objectPaths: [
@@ -360,6 +402,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     optimize,
     env: toolchain.env,
   });
+  timings.push({ phase: "link", ms: Date.now() - linkStarted });
 
   const bundlePath =
     bundle !== undefined
@@ -375,7 +418,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 
   cache.record(cacheKey, outputs);
   cache.save();
-  return { outputPath, irPath, cached: false, diagnostics: [], ir, ...(bundlePath ? { bundlePath } : {}) };
+  return { outputPath, irPath, cached: false, diagnostics: [], ir, timings, ...(bundlePath ? { bundlePath } : {}) };
 }
 
 /** Default `CFBundleIdentifier` for a bundle when none is configured. */
@@ -401,59 +444,17 @@ export const RUNTIME_SOURCES = [
   "xt_stdlib.c",
   "xt_stdlib2.c",
   "xt_promise.c",
+  "xt_async.c",
   "xt_loop.c",
   "xt_generator.c",
+  "xt_coroutine.c",
   "xt_fetch.c",
   "xt_symbol.c",
   "xt_builtins.c",
+  "xt_constructors.c",
   "xt_io.c",
 ] as const;
 
-/**
- * Fingerprint every runtime artifact that can change a linked executable: the
- * C sources, shared headers and `#include`d `.inc` fragments plus the shipped
- * prebuilt archives. It feeds the executable cache key, so a runtime change
- * invalidates a cached binary even when the entry source is untouched — the
- * per-object cache cannot express that, because the freshness check runs before
- * any compilation.
- */
-function runtimeFingerprint(runtimeDir: string): string {
-  if (!existsSync(runtimeDir)) return "";
-  const parts: string[] = [];
-  const walk = (dir: string): void => {
-    for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
-      const full = join(dir, entry.name);
-      if (entry.isDirectory()) {
-        walk(full);
-      } else if (/\.(c|h|inc)$/.test(entry.name)) {
-        parts.push(entry.name, readFileSync(full, "utf8"));
-      } else {
-        parts.push(entry.name, readFileSync(full).toString("base64"));
-      }
-    }
-  };
-  walk(runtimeDir);
-  return hashParts(parts);
-}
-
-/**
- * Fingerprint everything an extension contributes that can change the linked
- * executable but does not live under `runtime/`: its name, linker flags and the
- * contents of any pre-built C++/Rust objects. Keeps a cached binary fresh when
- * a native extension is rebuilt, or when `--ext-native` flags change.
- */
-function registryFingerprint(registry: ExtensionRegistry): string {
-  const parts: string[] = [];
-  for (const extension of registry.all()) {
-    parts.push(extension.name);
-    parts.push(...(extension.linkerFlags?.() ?? []));
-    for (const object of extension.nativeObjects?.() ?? []) {
-      parts.push(object, existsSync(object) ? readFileSync(object).toString("base64") : "");
-    }
-  }
-  return hashParts(parts);
-}
-
 /**
  * Compile the core runtime and every extension source, reusing cached object
  * files keyed on the C source hash. Returns the object paths to link.
@@ -487,7 +488,16 @@ function ensureRuntimeObjects(
 
   const compileOne = (sourcePath: string, tag: string): string => {
     const body = readFileSync(sourcePath, "utf8");
-    const key = hashParts(["runtime", COMPILER_VERSION, tag, headerStamp, body, clang, process.platform]);
+    const key = hashParts([
+      "runtime",
+      COMPILER_VERSION,
+      RUNTIME_CFLAGS.join(" "),
+      tag,
+      headerStamp,
+      body,
+      clang,
+      process.platform,
+    ]);
     const objectPath = join(cacheDir, `${tag}-${key}.o`);
     if (!existsSync(objectPath)) {
       compileC(runner, clang, sourcePath, objectPath, runtimeDir, env);

package/src/driver/fingerprint.ts

@@ -0,0 +1,85 @@
+/**
+ * Content fingerprints for the incremental cache.
+ *
+ * The executable cache is keyed on everything that can change the linked
+ * binary but is not part of the entry source: the C runtime, the compiler's own
+ * sources, and whatever a native extension contributes. The per-object cache
+ * cannot express these, because the freshness check runs before compilation.
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { ExtensionRegistry } from "../extensions/registry.js";
+import { hashParts } from "./cache.js";
+
+/** Runtime sources, headers and `#include`d `.inc` fragments, plus prebuilt archives. */
+export function runtimeFingerprint(runtimeDir: string): string {
+  if (!existsSync(runtimeDir)) return "";
+  const parts: string[] = [];
+  const walk = (dir: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(full);
+      } else if (/\.(c|h|inc)$/.test(entry.name)) {
+        parts.push(entry.name, readFileSync(full, "utf8"));
+      } else {
+        parts.push(entry.name, readFileSync(full).toString("base64"));
+      }
+    }
+  };
+  walk(runtimeDir);
+  return hashParts(parts);
+}
+
+/**
+ * Content hash of the compiler's own TypeScript sources, when they are present.
+ *
+ * `COMPILER_VERSION` is bumped per release, but a developer editing codegen
+ * between releases would otherwise reuse a binary produced by the previous
+ * compiler for the same input. Hashing `src/` closes that hole in a source
+ * checkout. A released archive ships only `bin/` and `runtime/`, so the marker
+ * check fails there and the version alone keys the cache — which is correct,
+ * because release automation keeps it in sync with package.json.
+ */
+export function compilerFingerprint(): string {
+  try {
+    // In a checkout this module is `<root>/src/driver/fingerprint.ts`.
+    const sourceRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+    if (!existsSync(join(sourceRoot, "codegen", "llvm.ts"))) return "";
+    if (!existsSync(join(sourceRoot, "driver", "compiler.ts"))) return "";
+    const parts: string[] = [];
+    const walk = (dir: string): void => {
+      for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+          walk(full);
+        } else if (entry.name.endsWith(".ts")) {
+          parts.push(relative(sourceRoot, full).replace(/\\/g, "/"), readFileSync(full, "utf8"));
+        }
+      }
+    };
+    walk(sourceRoot);
+    return hashParts(parts);
+  } catch {
+    // A self-hosted or packaged compiler may not expose its sources.
+    return "";
+  }
+}
+
+/**
+ * Everything an extension contributes besides its C sources: name, linker flags
+ * and the contents of pre-built C++/Rust objects.
+ */
+export function registryFingerprint(registry: ExtensionRegistry): string {
+  const parts: string[] = [];
+  for (const extension of registry.all()) {
+    parts.push(extension.name);
+    parts.push(...(extension.linkerFlags?.() ?? []));
+    for (const object of extension.nativeObjects?.() ?? []) {
+      parts.push(object, existsSync(object) ? readFileSync(object).toString("base64") : "");
+    }
+  }
+  return hashParts(parts);
+}

package/src/driver/toolchain-provider.ts

@@ -6,18 +6,23 @@
  * *which* driver to use, in a fixed priority order:
  *
  *   1. an explicit `xbintsc_CLANG` / `xbintsc_TOOLCHAIN`
- *   2. the system `PATH` (`findClang`)
+ *   2. the system `PATH` (`findClang`), preferring a clang >= 16
+ *   3. the LLVM build bundled with Visual Studio on Windows
  *
  * xbintsc does not ship a compiler. Every host must provide a clang-compatible
  * toolchain (see `doc/requirements.md`): the Xcode Command Line Tools on macOS,
  * the distribution's clang/lld on Linux, and LLVM plus the Visual Studio C++
  * build tools (the MSVC ABI) on Windows.
  *
+ * The version is checked here, not left to the first failed C compile: the
+ * emitted IR needs LLVM 16+ (opaque pointers), and a "clang 15 is too old"
+ * error up front is far more actionable than a wall of IR type errors later.
+ *
  * Extra linker arguments (for example `-fuse-ld=lld`) can be supplied through
  * `xbintsc_LINKER_ARGS`.
  */
 
-import { findClang, realRunner, type Runner } from "./toolchain.js";
+import { locateClang, realRunner, type ClangVersion, type Runner } from "./toolchain.js";
 
 export type ToolchainSource = "env" | "system";
 
@@ -26,6 +31,8 @@ export interface ResolvedToolchain {
   readonly clang: string;
   /** Where the driver came from. */
   readonly source: ToolchainSource;
+  /** Parsed `clang --version`, when the driver prints one. */
+  readonly version?: ClangVersion;
   /** Extra arguments to pass to the link step. */
   readonly linkerArgs: readonly string[];
   /** Extra environment for the toolchain subprocesses, merged over the current
@@ -33,14 +40,6 @@ export interface ResolvedToolchain {
   readonly env: Record<string, string>;
 }
 
-function works(runner: Runner, command: string): boolean {
-  try {
-    return runner.run(command, ["--version"]).status === 0;
-  } catch {
-    return false;
-  }
-}
-
 function linkerArgs(fallback: readonly string[]): string[] {
   const raw = process.env.xbintsc_LINKER_ARGS;
   if (raw) return raw.split(" ").filter((part) => part.length > 0);
@@ -49,10 +48,12 @@ function linkerArgs(fallback: readonly string[]): string[] {
 
 /** Resolve the toolchain that `build` should use. See the module docblock. */
 export function resolveToolchain(runner: Runner = realRunner): ResolvedToolchain {
-  const explicit = process.env.xbintsc_CLANG ?? process.env.xbintsc_TOOLCHAIN;
-  if (explicit && works(runner, explicit)) {
-    return { clang: explicit, source: "env", linkerArgs: linkerArgs([]), env: {} };
-  }
-
-  return { clang: findClang(runner), source: "system", linkerArgs: linkerArgs([]), env: {} };
+  const location = locateClang(runner);
+  return {
+    clang: location.command,
+    source: location.source,
+    ...(location.version ? { version: location.version } : {}),
+    linkerArgs: linkerArgs([]),
+    env: {},
+  };
 }