xbintsc 0.3.6 → 0.3.8

package/src/ast/expressions.ts

@@ -40,7 +40,8 @@ export interface NumericLiteral extends Node {
 export interface BigIntLiteral extends Node {
   readonly kind: SyntaxKind.BigIntLiteral;
   readonly text: string;
-  readonly value: number;
+  /** Exact value; BigInt literals are arbitrary precision. */
+  readonly value: bigint;
 }
 
 export interface StringLiteral extends Node {

package/src/cli/main.ts

@@ -5,6 +5,7 @@
  * available programmatically through the driver API:
  *
  *   xbintsc build <file> [-o out] [--emit ir|obj|exe] [-O0..3] [--ext node]
+ *                        [--ext-native manifest.json]
  *   xbintsc run   <file> [-- args...]
  *   xbintsc emit  <file>            # print LLVM IR to stdout
  *   xbintsc doctor                  # report the resolved toolchain
@@ -23,6 +24,7 @@ import { resolveToolchain } from "../driver/toolchain-provider.js";
 import { realRunner } from "../driver/toolchain.js";
 import { createDefaultRegistry, type ExtensionRegistry } from "../extensions/registry.js";
 import { nodeExtension } from "../extensions/node/index.js";
+import { nativeExtensionFromManifest } from "../extensions/native.js";
 
 export interface CliIo {
   readonly stdout: (text: string) => void;
@@ -41,7 +43,7 @@ interface ParsedArgs {
   readonly passthrough: string[];
 }
 
-const VALUE_FLAGS = new Set(["output", "out", "emit", "optimize", "ext"]);
+const VALUE_FLAGS = new Set(["output", "out", "emit", "optimize", "ext", "ext-native"]);
 
 function parseArgs(argv: readonly string[]): ParsedArgs {
   const positionals: string[] = [];
@@ -102,6 +104,14 @@ function buildRegistry(flags: Map<string, string | boolean>): ExtensionRegistry
       else throw new Error(`Unknown extension '${name}'`);
     }
   }
+  // Each `--ext-native` value is a manifest path describing a pre-built C++ or
+  // Rust library; several manifests may be comma separated.
+  const native = flags.get("ext-native");
+  if (typeof native === "string") {
+    for (const manifest of native.split(",").map((n) => n.trim()).filter(Boolean)) {
+      registry.register(nativeExtensionFromManifest(manifest));
+    }
+  }
   return registry;
 }
 
@@ -133,6 +143,8 @@ Options:
       --emit <kind>     exe | obj | ir (default: exe)
   -O0..-O3              Optimization level (default: -O2)
       --ext <names>     Comma separated extensions (e.g. node)
+      --ext-native <m>  Register a C++/Rust extension from a JSON manifest
+                        (comma separated for several)
       --force           Ignore the incremental cache
       --verbose         Print progress information
 `;

package/src/driver/compiler.ts

@@ -160,7 +160,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     emit,
     optimize,
     process.platform,
-    registry.all().map((e) => e.name).join(","),
+    registryFingerprint(registry),
     runtimeFingerprint(runtimeDir),
   ]);
 
@@ -209,7 +209,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     return { outputPath, irPath, cached: false, diagnostics: [], ir };
   }
 
-  const { runtimeObjects, extensionObjects } = ensureRuntimeObjects(
+  const { runtimeObjects, extensionObjects, nativeObjects } = ensureRuntimeObjects(
     runner,
     clang,
     runtimeDir,
@@ -221,7 +221,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 
   link(runner, {
     clang,
-    objectPaths: [objectPath, ...runtimeObjects, ...extensionObjects],
+    objectPaths: [objectPath, ...runtimeObjects, ...extensionObjects, ...nativeObjects],
     outputPath,
     linkerFlags: [
       ...toolchain.linkerArgs,
@@ -240,6 +240,8 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
 interface RuntimeObjects {
   readonly runtimeObjects: readonly string[];
   readonly extensionObjects: readonly string[];
+  /** Pre-built C++/Rust objects and archives contributed by native extensions. */
+  readonly nativeObjects: readonly string[];
 }
 
 /** Core runtime translation units (each compiled and cached independently). */
@@ -285,6 +287,24 @@ function runtimeFingerprint(runtimeDir: string): string {
   return hashParts(parts);
 }
 
+/**
+ * Fingerprint everything an extension contributes that can change the linked
+ * executable but does not live under `runtime/`: its name, linker flags and the
+ * contents of any pre-built C++/Rust objects. Keeps a cached binary fresh when
+ * a native extension is rebuilt, or when `--ext-native` flags change.
+ */
+function registryFingerprint(registry: ExtensionRegistry): string {
+  const parts: string[] = [];
+  for (const extension of registry.all()) {
+    parts.push(extension.name);
+    parts.push(...(extension.linkerFlags?.() ?? []));
+    for (const object of extension.nativeObjects?.() ?? []) {
+      parts.push(object, existsSync(object) ? readFileSync(object).toString("base64") : "");
+    }
+  }
+  return hashParts(parts);
+}
+
 /**
  * Compile the core runtime and every extension source, reusing cached object
  * files keyed on the C source hash. Returns the object paths to link.
@@ -348,5 +368,6 @@ function ensureRuntimeObjects(
       extensionObjects.push(compileOne(sourcePath, `ext_${extensionObjects.length}_${basename(sourcePath, ".c")}`));
     }
   }
-  return { runtimeObjects, extensionObjects };
+  // A native archive may be shared by several manifests; link each once.
+  return { runtimeObjects, extensionObjects, nativeObjects: [...new Set(registry.nativeObjects())] };
 }

package/src/extensions/native.ts

@@ -0,0 +1,270 @@
+/**
+ * Native (C++ / Rust) extensions.
+ *
+ * xbintsc can consume code that was compiled outside the TypeScript pipeline,
+ * as long as it exposes `extern "C"` entry points with the runtime calling
+ * convention:
+ *
+ *     xt_value my_fn(int32_t argc, xt_value *argv);
+ *
+ * A C++ or Rust project is built into an object file or static archive, and a
+ * small JSON manifest describes how those symbols map onto global functions
+ * (`builtins`) and importable modules (`modules`). The driver links the
+ * artifacts verbatim and the code generator wires the bindings exactly like it
+ * does for the built-in `node` extension — the core compiler never needs to
+ * know the extension was written in C++ or Rust.
+ *
+ * ```jsonc
+ * {
+ *   "name": "mathx",
+ *   "description": "C++ math helpers",
+ *   "objects": ["build/libmathx.a"],
+ *   "linkerFlags": ["-lm"],
+ *   "linkerFlagsByPlatform": { "linux": ["-lstdc++"], "darwin": ["-lc++"] },
+ *   "builtins": { "fastAdd": { "symbol": "mathx_add" } },
+ *   "modules": {
+ *     "mathx": {
+ *       "exports": {
+ *         "add": { "symbol": "mathx_add" },
+ *         "reverse": { "symbol": "mathx_reverse" }
+ *       }
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * `objects` paths are resolved relative to the manifest file, so a manifest can
+ * live next to the project it describes.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, isAbsolute, resolve } from "node:path";
+import type { Extension, ExtensionModule, ModuleExport, ModuleExports } from "./registry.js";
+
+/** A global function an extension provides (callable without an import). */
+export interface NativeBuiltin {
+  /** Exported C symbol, called as `xt_value symbol(int32_t argc, xt_value *argv)`. */
+  readonly symbol: string;
+  readonly returnVoid?: boolean;
+}
+
+/** One binding inside a `modules` entry; mirrors the core `ModuleExport`. */
+export type NativeModuleExport = ModuleExport;
+
+/** An importable module supplied by the native library. */
+export interface NativeModule {
+  /** Named exports for `import { x } from "..."`. */
+  readonly exports?: Readonly<Record<string, NativeModuleExport>>;
+  /** Namespace name for `import * as ns` / `import ns from`. */
+  readonly namespace?: string;
+}
+
+/** The on-disk schema of a native extension manifest. */
+export interface NativeManifest {
+  /** Unique extension name (used by `--ext`/diagnostics). */
+  readonly name: string;
+  readonly description?: string;
+  /**
+   * Object files (`.o`) or static archives (`.a`/`.lib`) to link. Resolved
+   * relative to the manifest unless absolute.
+   */
+  readonly objects?: readonly string[];
+  /** Linker flags appended verbatim, e.g. `["-lm"]`. */
+  readonly linkerFlags?: readonly string[];
+  /**
+   * Linker flags added only on the given platform (`darwin`, `linux`, `win32`),
+   * for C++ standard libraries or Rust's native dependencies that differ per OS.
+   */
+  readonly linkerFlagsByPlatform?: Readonly<Record<string, readonly string[]>>;
+  /** Global functions the extension binds. */
+  readonly builtins?: Readonly<Record<string, NativeBuiltin>>;
+  /** Modules importable as `import ... from "<specifier>"`. */
+  readonly modules?: Readonly<Record<string, NativeModule>>;
+}
+
+/** Raised for a malformed manifest or a missing build artifact. */
+export class NativeExtensionError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "NativeExtensionError";
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stringArray(value: unknown, field: string): readonly string[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+    throw new NativeExtensionError(`Native manifest field '${field}' must be an array of strings`);
+  }
+  return value as readonly string[];
+}
+
+function parseBuiltins(value: unknown): Readonly<Record<string, NativeBuiltin>> | undefined {
+  if (value === undefined) return undefined;
+  if (!isRecord(value)) throw new NativeExtensionError("Native manifest field 'builtins' must be an object");
+  const builtins: Record<string, NativeBuiltin> = {};
+  for (const [name, entry] of Object.entries(value)) {
+    if (!isRecord(entry) || typeof entry.symbol !== "string" || entry.symbol.length === 0) {
+      throw new NativeExtensionError(`Native manifest builtin '${name}' needs a non-empty 'symbol'`);
+    }
+    builtins[name] = {
+      symbol: entry.symbol,
+      ...(entry.returnVoid === true ? { returnVoid: true } : {}),
+    };
+  }
+  return builtins;
+}
+
+function parseExports(value: unknown, moduleName: string): ModuleExports | undefined {
+  if (value === undefined) return undefined;
+  if (!isRecord(value)) {
+    throw new NativeExtensionError(`Native manifest module '${moduleName}'.exports must be an object`);
+  }
+  const exports: Record<string, ModuleExport> = {};
+  for (const [name, entry] of Object.entries(value)) {
+    if (!isRecord(entry)) {
+      throw new NativeExtensionError(`Native manifest export '${moduleName}.${name}' must be an object`);
+    }
+    const symbol = entry.symbol;
+    const namespace = entry.namespace;
+    const method = entry.method;
+    if (symbol !== undefined && typeof symbol !== "string") {
+      throw new NativeExtensionError(`Native manifest export '${moduleName}.${name}'.symbol must be a string`);
+    }
+    if (namespace !== undefined && typeof namespace !== "string") {
+      throw new NativeExtensionError(`Native manifest export '${moduleName}.${name}'.namespace must be a string`);
+    }
+    if (method !== undefined && typeof method !== "string") {
+      throw new NativeExtensionError(`Native manifest export '${moduleName}.${name}'.method must be a string`);
+    }
+    if (symbol === undefined && namespace === undefined) {
+      throw new NativeExtensionError(
+        `Native manifest export '${moduleName}.${name}' needs a 'symbol' or a 'namespace'`,
+      );
+    }
+    exports[name] = {
+      ...(symbol !== undefined ? { symbol } : {}),
+      ...(namespace !== undefined ? { namespace } : {}),
+      ...(method !== undefined ? { method } : {}),
+      ...(entry.returnVoid === true ? { returnVoid: true } : {}),
+      ...(entry.isConstructor === true ? { isConstructor: true } : {}),
+    };
+  }
+  return exports;
+}
+
+function parseModules(value: unknown): Readonly<Record<string, ExtensionModule>> | undefined {
+  if (value === undefined) return undefined;
+  if (!isRecord(value)) throw new NativeExtensionError("Native manifest field 'modules' must be an object");
+  const modules: Record<string, ExtensionModule> = {};
+  for (const [specifier, entry] of Object.entries(value)) {
+    if (!isRecord(entry)) {
+      throw new NativeExtensionError(`Native manifest module '${specifier}' must be an object`);
+    }
+    if (entry.namespace !== undefined && typeof entry.namespace !== "string") {
+      throw new NativeExtensionError(`Native manifest module '${specifier}'.namespace must be a string`);
+    }
+    modules[specifier] = {
+      ...(entry.namespace !== undefined ? { namespace: entry.namespace } : {}),
+      ...(entry.exports !== undefined ? { exports: parseExports(entry.exports, specifier) } : {}),
+    };
+  }
+  return modules;
+}
+
+/** Validate parsed JSON and normalise it into a {@link NativeManifest}. */
+export function parseNativeManifest(value: unknown, source = "<manifest>"): NativeManifest {
+  if (!isRecord(value)) throw new NativeExtensionError(`Native manifest ${source} must contain a JSON object`);
+  if (typeof value.name !== "string" || value.name.trim().length === 0) {
+    throw new NativeExtensionError(`Native manifest ${source} needs a non-empty 'name'`);
+  }
+  if (value.description !== undefined && typeof value.description !== "string") {
+    throw new NativeExtensionError(`Native manifest '${value.name}'.description must be a string`);
+  }
+
+  let linkerFlagsByPlatform: Record<string, readonly string[]> | undefined;
+  if (value.linkerFlagsByPlatform !== undefined) {
+    if (!isRecord(value.linkerFlagsByPlatform)) {
+      throw new NativeExtensionError("Native manifest field 'linkerFlagsByPlatform' must be an object");
+    }
+    linkerFlagsByPlatform = {};
+    for (const [platform, flags] of Object.entries(value.linkerFlagsByPlatform)) {
+      linkerFlagsByPlatform[platform] = stringArray(flags, `linkerFlagsByPlatform.${platform}`);
+    }
+  }
+
+  const builtins = parseBuiltins(value.builtins);
+  const modules = parseModules(value.modules);
+
+  return {
+    name: value.name,
+    ...(value.description !== undefined ? { description: value.description } : {}),
+    objects: stringArray(value.objects, "objects"),
+    linkerFlags: stringArray(value.linkerFlags, "linkerFlags"),
+    ...(linkerFlagsByPlatform ? { linkerFlagsByPlatform } : {}),
+    ...(builtins ? { builtins } : {}),
+    ...(modules ? { modules } : {}),
+  };
+}
+
+/** Read and validate a manifest from disk. */
+export function loadNativeManifest(manifestPath: string): NativeManifest {
+  const absolute = resolve(manifestPath);
+  let text: string;
+  try {
+    text = readFileSync(absolute, "utf8");
+  } catch (error) {
+    throw new NativeExtensionError(`Unable to read native extension manifest '${absolute}': ${String(error)}`);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch (error) {
+    throw new NativeExtensionError(`Native extension manifest '${absolute}' is not valid JSON: ${String(error)}`);
+  }
+  return parseNativeManifest(parsed, absolute);
+}
+
+function resolveArtifact(baseDir: string, path: string): string {
+  return isAbsolute(path) ? path : resolve(baseDir, path);
+}
+
+/**
+ * Turn a manifest into an {@link Extension}: resolve and verify every native
+ * object, then expose the manifest's builtins/modules to the code generator.
+ *
+ * The manifest is read eagerly so a typo surfaces at registration time rather
+ * than at link time.
+ */
+export function nativeExtensionFromManifest(manifestPath: string): Extension {
+  const absolute = resolve(manifestPath);
+  const manifest = loadNativeManifest(absolute);
+  const baseDir = dirname(absolute);
+
+  const objects = (manifest.objects ?? []).map((object) => resolveArtifact(baseDir, object));
+  for (const object of objects) {
+    if (!existsSync(object)) {
+      throw new NativeExtensionError(
+        `Native extension '${manifest.name}' references missing object '${object}'.\n` +
+          `Build the C++/Rust library first (see the project next to ${absolute}).`,
+      );
+    }
+  }
+
+  const linkerFlags = () => [
+    ...(manifest.linkerFlags ?? []),
+    ...(manifest.linkerFlagsByPlatform?.[process.platform] ?? []),
+  ];
+
+  return {
+    name: manifest.name,
+    description: manifest.description ?? `Native extension from ${absolute}`,
+    nativeObjects: () => objects,
+    linkerFlags,
+    ...(manifest.builtins ? { builtins: () => manifest.builtins! } : {}),
+    ...(manifest.modules ? { modules: () => manifest.modules! } : {}),
+  };
+}

package/src/extensions/registry.ts

@@ -52,6 +52,17 @@ export interface Extension {
   readonly description?: string;
   /** C/asm sources compiled and linked alongside the generated module. */
   runtimeSources?(): readonly string[];
+  /**
+   * Pre-built object files or static archives linked alongside the generated
+   * module. This is the hook for native extensions written in C++ or Rust:
+   * compile them to `extern "C"` objects (or a static library) that use the
+   * `xt_value` ABI, then hand the artifacts to the driver through an
+   * `Extension` such as the one built by `src/extensions/native.ts`.
+   *
+   * The paths must already exist; the driver passes them straight to the
+   * linker and never recompiles them.
+   */
+  nativeObjects?(): readonly string[];
   /** Extra linker flags (e.g. `["-lm"]`, `["-framework", "CoreFoundation"]`). */
   linkerFlags?(): readonly string[];
   /** Global identifiers that resolve to runtime symbols when called. */
@@ -123,6 +134,16 @@ export class ExtensionRegistry {
     return sources;
   }
 
+  /** Flatten every registered extension's pre-built native objects. */
+  nativeObjects(): readonly string[] {
+    const objects: string[] = [];
+    for (const extension of this.extensions.values()) {
+      const extra = extension.nativeObjects?.();
+      if (extra) objects.push(...extra);
+    }
+    return objects;
+  }
+
   linkerFlags(): readonly string[] {
     const flags: string[] = [];
     for (const extension of this.extensions.values()) {

package/src/index.ts

@@ -32,4 +32,14 @@ export {
 export { ExtensionRegistry, createDefaultRegistry, coreExtension, type Extension, type ExtensionModule, type ModuleExport, type ModuleExports } from "./extensions/registry.js";
 export { nodeExtension } from "./extensions/node/index.js";
 export type { NodeModule } from "./extensions/node/module.js";
+export {
+  NativeExtensionError,
+  loadNativeManifest,
+  nativeExtensionFromManifest,
+  parseNativeManifest,
+  type NativeManifest,
+  type NativeBuiltin,
+  type NativeModule,
+  type NativeModuleExport,
+} from "./extensions/native.js";
 export * from "./driver/index.js";

package/src/lexer/scanner/numbers.ts

@@ -63,10 +63,14 @@ export const numberMethods: NumberMethods = {
     const cleaned = raw.replace(/_/g, "");
     if (isBigInt) {
       raw = raw.slice(0, -1).replace(/_/g, "");
-      const parsed = Number(raw);
-      if (parsed !== parsed) {
+      // Parse exactly with BigInt: a Number would round values past 2^53 and
+      // overflow huge literals to Infinity, silently corrupting the AST value.
+      let parsed: bigint;
+      try {
+        parsed = BigInt(raw);
+      } catch {
         this.error(DiagnosticCode.InvalidNumber, `Invalid BigInt literal '${raw}'`, start, this.pos);
-        return this.makeToken(TokenKind.BigIntLiteral, start, this.pos, raw, precededByLineBreak, 0);
+        return this.makeToken(TokenKind.BigIntLiteral, start, this.pos, raw, precededByLineBreak, 0n);
       }
       return this.makeToken(TokenKind.BigIntLiteral, start, this.pos, raw, precededByLineBreak, parsed);
     }

package/src/parser/expressions.ts

@@ -333,7 +333,7 @@ export const expressionMethods: ExpressionMethods = {
       }
       case TokenKind.BigIntLiteral:
         this.nextToken();
-        return { kind: SyntaxKind.BigIntLiteral, text: token.text, value: Number(token.value ?? 0), start: token.start, end: token.end };
+        return { kind: SyntaxKind.BigIntLiteral, text: token.text, value: (token.value as bigint | undefined) ?? 0n, start: token.start, end: token.end };
       case TokenKind.StringLiteral:
         this.nextToken();
         return this.stringLiteralFromToken(token);