xbintsc 0.3.35 → 0.3.49

package/src/cli/main.ts

@@ -22,7 +22,16 @@ import { build, compileEntry, COMPILER_VERSION, type EmitKind } from "../driver/
 import { findRuntimeDir, platformSlug } from "../driver/paths.js";
 import { runtimeLibDir } from "../driver/runtime-lib.js";
 import { resolveToolchain } from "../driver/toolchain-provider.js";
-import { realRunner } from "../driver/toolchain.js";
+import { realRunner, ToolchainError } from "../driver/toolchain.js";
+import {
+  DOC_CLI,
+  DOC_TROUBLESHOOTING,
+  documentationHints,
+  thrownFailure,
+  toolchainPointer,
+  UNEXPECTED_FAILURE_HINT,
+  type DocumentationPointer,
+} from "./hints.js";
 import { createDefaultRegistry, type ExtensionRegistry } from "../extensions/registry.js";
 import { bundledExtensions } from "../extensions/catalog.js";
 import { nativeExtensionFromManifest } from "../extensions/native.js";
@@ -46,6 +55,13 @@ const defaultIo: CliIo = {
   stderr: (text) => process.stderr.write(text),
 };
 
+/**
+ * `--no-hints` state for the current {@link run} call. A module-level flag (and
+ * not a parameter) keeps the hint layer out of the signatures of every helper
+ * that can fail.
+ */
+let hintsDisabled = false;
+
 interface ParsedArgs {
   readonly command?: string;
   readonly positionals: string[];
@@ -152,6 +168,46 @@ function printDiagnostics(diagnostics: readonly Diagnostic[], fileName: string,
     }
     io.stderr(formatDiagnostic(diagnostic, source) + "\n");
   }
+  for (const hint of documentationHints(diagnostics)) emitHint(io, hint);
+}
+
+/** Print one `hint:` line unless the caller asked for plain output. */
+function emitHint(io: CliIo, line: string): void {
+  if (hintsDisabled) return;
+  io.stderr(`${line}\n`);
+}
+
+/**
+ * Report a failure that threw instead of becoming a diagnostic, ending with the
+ * documentation pointer that explains it. Without this a toolchain failure is a
+ * raw stack trace, which tells a reader nothing about which document answers
+ * the question.
+ */
+function reportThrown(error: unknown, io: CliIo): number {
+  const message =
+    error instanceof ToolchainError
+      ? `xbintsc: ${error.message}`
+      : `xbintsc: ${error instanceof Error ? error.message : String(error)}`;
+  io.stderr(`${message.trimEnd()}\n`);
+  const text = error instanceof ToolchainError ? error.message : message;
+  const pointer = error instanceof ToolchainError ? toolchainPointer(text) : thrownPointer(text);
+  emitHint(io, `hint: ${pointer.text} — see ${pointer.path}`);
+  return 1;
+}
+
+/** The document that explains a failure the CLI reports as a plain message. */
+function thrownPointer(message: string): DocumentationPointer {
+  return thrownFailure(message)?.pointer ?? UNEXPECTED_FAILURE_HINT;
+}
+
+/** Report a project-config failure, which never becomes a diagnostic bag. */
+function reportConfigError(error: string, io: CliIo): number {
+  io.stderr(`xbintsc: ${error}\n`);
+  const pointer = /JSON/i.test(error)
+    ? `hint: fix the JSON; an unreadable or malformed config fails the build — see ${DOC_TROUBLESHOOTING}`
+    : `hint: check the config path and its fields — see ${DOC_CLI}`;
+  emitHint(io, pointer);
+  return 1;
 }
 
 const HELP = `xbintsc ${COMPILER_VERSION} - TypeScript binary compiler
@@ -181,6 +237,14 @@ Options:
       --app-id <id>     macOS bundle identifier (e.g. com.example.demo)
       --force           Ignore the incremental cache
       --verbose         Print progress information
+      --no-hints        Do not append the "hint: … see <doc>" line to failures
+
+xbintsc compiles a subset of TypeScript to a native binary. Types are erased
+(never checked), Node modules need --ext node, third-party npm imports are not
+supported, and clang 16+ is required to link.
+
+Docs: doc/ai/README.md (task guide) - doc/ai/language-support.md (subset) -
+doc/ai/troubleshooting.md (failures) - llms.txt (index of every document)
 `;
 
 function firstLine(text: string): string {
@@ -281,6 +345,19 @@ function listFlag(flags: Map<string, string | boolean>, name: string): string[]
 }
 
 export function run(argv: readonly string[], io: CliIo = defaultIo): number {
+  const disableHints = parseArgs(argv).flags.has("no-hints");
+  const previous = hintsDisabled;
+  hintsDisabled = disableHints;
+  try {
+    return dispatch(argv, io);
+  } catch (error) {
+    return reportThrown(error, io);
+  } finally {
+    hintsDisabled = previous;
+  }
+}
+
+function dispatch(argv: readonly string[], io: CliIo): number {
   const args = parseArgs(argv);
   const command = args.command;
 
@@ -301,13 +378,11 @@ export function run(argv: readonly string[], io: CliIo = defaultIo): number {
   if (command === "emit") {
     const cliEntry = args.positionals[0];
     const resolved = resolveProjectConfig(args.flags, cliEntry);
-    if (!resolved.ok) {
-      io.stderr(`xbintsc: ${resolved.error}\n`);
-      return 1;
-    }
+    if (!resolved.ok) return reportConfigError(resolved.error, io);
     const entry = cliEntry ?? resolved.config.entry;
     if (!entry) {
       io.stderr(`xbintsc: emit requires a source file (or 'entry' in ${CONFIG_FILE_NAME})\n`);
+      emitHint(io, `hint: pass the entry file, or set 'entry' in ${CONFIG_FILE_NAME} — see ${DOC_CLI}`);
       return 1;
     }
     const extNames = listFlag(args.flags, "ext") ?? resolved.config.extensions ?? [];
@@ -325,14 +400,12 @@ export function run(argv: readonly string[], io: CliIo = defaultIo): number {
   if (command === "build" || command === "run") {
     const cliEntry = args.positionals[0];
     const resolved = resolveProjectConfig(args.flags, cliEntry);
-    if (!resolved.ok) {
-      io.stderr(`xbintsc: ${resolved.error}\n`);
-      return 1;
-    }
+    if (!resolved.ok) return reportConfigError(resolved.error, io);
     const config = resolved.config;
     const entry = cliEntry ?? config.entry;
     if (!entry) {
       io.stderr(`xbintsc: ${command} requires a source file (or 'entry' in ${CONFIG_FILE_NAME})\n`);
+      emitHint(io, `hint: pass the entry file, or set 'entry' in ${CONFIG_FILE_NAME} — see ${DOC_CLI}`);
       return 1;
     }
     const emit = (args.flags.get("emit") as EmitKind | undefined) ?? "exe";

package/src/codegen/generator/module.ts

@@ -227,9 +227,19 @@ export const moduleMethods: ModuleMethods = {
 
   emitMain(): void {
     const moduleName = this.functionName(this.binding.moduleFunction);
+    // Module bindings and class constructors live in globals; register them as
+    // GC roots before the module body runs so a collection cannot free them.
+    const roots = [...this.classGlobals.values(), ...this.moduleGlobals.values()];
+    const rootLines = roots.map((name) => `  call void @xt_gc_add_root(i64* ${name})`);
     this.functions.push(
       [
         "define i32 @main(i32 %argc, i8** %argv) {",
+        "  %stackbase = alloca i64",
+        "  %stackbase.ptr = bitcast i64* %stackbase to i8*",
+        "  call void @xt_gc_set_stack_base(i8* %stackbase.ptr)",
+        "  call void @xt_gc_init()",
+        ...rootLines,
+        "  call void @xt_gc_arm()",
         "  call void @xt_set_program_args(i32 %argc, i8** %argv)",
         `  %result = call i64 @${moduleName}(i64 ${i64(XT_UNDEFINED)}, i64 ${i64(XT_UNDEFINED)}, i32 0, i64* null)`,
         "  call void @xt_drain_microtasks()",

package/src/codegen/generator/state.ts

@@ -103,6 +103,10 @@ export const RUNTIME_DECLARATIONS: readonly string[] = [
   "declare i64 @xt_await(i64)",
   "declare void @xt_drain_microtasks()",
   "declare void @xt_run_event_loop()",
+  "declare void @xt_gc_init()",
+  "declare void @xt_gc_set_stack_base(i8*)",
+  "declare void @xt_gc_arm()",
+  "declare void @xt_gc_add_root(i64*)",
   "declare i64 @xt_promise_resolve(i64)",
   "declare i64 @xt_promise_reject(i64)",
   "declare i64 @xt_promise_ctor(i32, i64*)",

package/src/codegen/generator/tables.ts

@@ -327,27 +327,73 @@ export const KIND_NAMES = new Map<number, string>([
   [SyntaxKind.SwitchStatement, "switch statement"],
 ]);
 
+/**
+ * The UTF-8 bytes of one code point, which must be an ASCII byte or a Unicode
+ * scalar (`0..0x10ffff`, not a surrogate). Encoded by hand rather than with
+ * `String.fromCharCode`/`String.fromCodePoint`, which a self-hosted build would
+ * run through the runtime and encode a second time.
+ */
+export function codePointBytes(code: number): number[] {
+  if (code < 0x80) return [code];
+  if (code < 0x800) return [0xc0 | (code >> 6), 0x80 | (code & 0x3f)];
+  if (code < 0x10000) return [0xe0 | (code >> 12), 0x80 | ((code >> 6) & 0x3f), 0x80 | (code & 0x3f)];
+  return [
+    0xf0 | (code >> 18),
+    0x80 | ((code >> 12) & 0x3f),
+    0x80 | ((code >> 6) & 0x3f),
+    0x80 | (code & 0x3f),
+  ];
+}
+
+/**
+ * The bytes a string literal stands for.
+ *
+ * A compiled string is a sequence of UTF-8 bytes (`"é".length` is 1 and
+ * `"😀".length` is 4 - see doc/ai/language-support.md). The text reaching this
+ * point is whatever a host's strings are, and both shapes have to produce the
+ * same bytes:
+ *
+ * - a UTF-16 host decodes a file into characters, so `é` is one code unit
+ *   (0xe9) and has to be encoded here;
+ * - the self-hosted runtime has no UTF-8 decoder, so `readFileSync` hands over
+ *   a file's bytes, and `é` is already `c3 a9`. Encoding that again is what
+ *   turned the three bytes of `—` into six and broke the self-hosting fixpoint.
+ *
+ * A code unit above 0xff can only be text, so its presence settles which shape
+ * this is: text is encoded (once), and text that is already a byte sequence is
+ * emitted as it is. A source file read on either host therefore lands on the
+ * same bytes, which is what the fixpoint needs.
+ */
 export function utf8Bytes(text: string): number[] {
+  if (!hasMultiByteUnit(text)) {
+    const bytes: number[] = [];
+    for (let index = 0; index < text.length; index++) bytes.push(text.charCodeAt(index) & 0xff);
+    return bytes;
+  }
   const bytes: number[] = [];
-  for (const character of text) {
-    const code = character.codePointAt(0)!;
-    if (code < 0x80) bytes.push(code);
-    else if (code < 0x800) {
-      bytes.push(0xc0 | (code >> 6), 0x80 | (code & 0x3f));
-    } else if (code < 0x10000) {
-      bytes.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 0x3f), 0x80 | (code & 0x3f));
-    } else {
-      bytes.push(
-        0xf0 | (code >> 18),
-        0x80 | ((code >> 12) & 0x3f),
-        0x80 | ((code >> 6) & 0x3f),
-        0x80 | (code & 0x3f),
-      );
+  const length = text.length;
+  for (let index = 0; index < length; index++) {
+    let code = text.charCodeAt(index);
+    if (code >= 0xd800 && code <= 0xdbff && index + 1 < length) {
+      const low = text.charCodeAt(index + 1);
+      if (low >= 0xdc00 && low <= 0xdfff) {
+        code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00);
+        index++;
+      }
     }
+    for (const byte of codePointBytes(code)) bytes.push(byte);
   }
   return bytes;
 }
 
+/** Whether any code unit is above one byte, which means text, not bytes. */
+function hasMultiByteUnit(text: string): boolean {
+  for (let index = 0; index < text.length; index++) {
+    if (text.charCodeAt(index) > 0xff) return true;
+  }
+  return false;
+}
+
 export function escapeBytes(bytes: readonly number[]): string {
   return bytes
     .map((byte) => {

package/src/diagnostics/source-text.ts

@@ -0,0 +1,78 @@
+/**
+ * Source text is a byte sequence.
+ *
+ * A compiled string is a sequence of UTF-8 bytes (`"e".length` is 1 for an
+ * accent and 4 for an emoji - see doc/ai/language-support.md), and the compiler
+ * is built by itself: there the self-hosted `readFileSync` hands the scanner a
+ * file's own bytes, one per code unit, and the runtime has no UTF-8 decoder to
+ * change that. A host whose strings are UTF-16 must be given the text in the
+ * same shape, or the same program emits different bytes depending on which
+ * generation of the compiler read it - which is how a non-ASCII literal ended
+ * up double-encoded and broke the self-hosting fixpoint.
+ *
+ * `sourceTextBytes` is that shape: a source file's bytes, one code unit each.
+ * The driver reads every source file through it, and codegen emits those bytes
+ * unchanged.
+ */
+
+/**
+ * The UTF-8 bytes of `text`, one code unit each. Text that is already a byte
+ * sequence is kept as it is, so this is idempotent and safe to apply to a read
+ * on either host.
+ */
+export function sourceTextBytes(text: string): string {
+  if (!hasCharacter(text)) return startsWithBom(text) ? text.slice(3) : text;
+  const encoded = encodeUtf8(text);
+  return startsWithBom(encoded) ? encoded.slice(3) : encoded;
+}
+
+/** Encode text as UTF-8, spelled one code unit per byte. */
+function encodeUtf8(text: string): string {
+  const chunks: string[] = [];
+  const units: number[] = [];
+  for (let index = 0; index < text.length; index++) {
+    let code = text.charCodeAt(index);
+    if (code >= 0xd800 && code <= 0xdbff && index + 1 < text.length) {
+      const low = text.charCodeAt(index + 1);
+      if (low >= 0xdc00 && low <= 0xdfff) {
+        code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00);
+        index++;
+      }
+    }
+    if (code < 0x80) units.push(code);
+    else if (code < 0x800) units.push(0xc0 | (code >> 6), 0x80 | (code & 0x3f));
+    else if (code < 0x10000) {
+      units.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 0x3f), 0x80 | (code & 0x3f));
+    } else {
+      units.push(
+        0xf0 | (code >> 18),
+        0x80 | ((code >> 12) & 0x3f),
+        0x80 | ((code >> 6) & 0x3f),
+        0x80 | (code & 0x3f),
+      );
+    }
+    if (units.length >= 512) flushUnits(units, chunks);
+  }
+  flushUnits(units, chunks);
+  return chunks.join("");
+}
+
+/** Whether any code unit is above one byte, which means text, not bytes. */
+function hasCharacter(text: string): boolean {
+  for (let index = 0; index < text.length; index++) {
+    if (text.charCodeAt(index) > 0xff) return true;
+  }
+  return false;
+}
+
+/** Whether a decoded read started with a UTF-8 byte order mark. */
+function startsWithBom(text: string): boolean {
+  return text.charCodeAt(0) === 0xef && text.charCodeAt(1) === 0xbb && text.charCodeAt(2) === 0xbf;
+}
+
+/** Append the pending code units as one chunk of the byte string. */
+function flushUnits(units: number[], chunks: string[]): void {
+  if (units.length === 0) return;
+  chunks.push(String.fromCharCode(...units));
+  units.length = 0;
+}

package/src/driver/bundler/graph.ts

@@ -9,6 +9,7 @@ import { SyntaxKind, type ExportDeclaration, type ImportDeclaration, type Statem
 import { bind } from "../../binder/binder.js";
 import { DiagnosticBag, DiagnosticCode } from "../../diagnostics/diagnostic.js";
 import { SourceFile } from "../../diagnostics/source.js";
+import { sourceTextBytes } from "../../diagnostics/source-text.js";
 import { Parser } from "../../parser/parser.js";
 import type { AssetLoader } from "../../extensions/registry.js";
 import { classifyDependency } from "./resolve.js";
@@ -53,7 +54,7 @@ export function loadGraph(
       return undefined;
     }
     visiting.add(path);
-    const raw = readFileSync(path, "utf8");
+    const raw = sourceTextBytes(readFileSync(path, "utf8"));
     /* An extension may rewrite an imported asset (e.g. `.html`) into a TS
      * module before it is parsed. The core only sees the transformed text. */
     const loader = assetLoaders[extname(path).toLowerCase()];

package/src/driver/compiler.ts

@@ -11,6 +11,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { basename, extname, join, resolve } from "node:path";
 import { DiagnosticBag, DiagnosticCode, type Diagnostic } from "../diagnostics/diagnostic.js";
 import { SourceFile } from "../diagnostics/source.js";
+import { sourceTextBytes } from "../diagnostics/source-text.js";
 import { Parser } from "../parser/parser.js";
 import { generate } from "../codegen/llvm.js";
 import { SyntaxKind } from "../ast/nodes.js";
@@ -106,7 +107,7 @@ function externalModuleSpecifiers(registry: ExtensionRegistry): Set<string> {
  */
 export function compileEntry(entryPath: string, extensions?: ExtensionRegistry): CompileStringResult {
   const absoluteEntry = resolve(entryPath);
-  const sourceText = readFileSync(absoluteEntry, "utf8");
+  const sourceText = sourceTextBytes(readFileSync(absoluteEntry, "utf8"));
   const file = new SourceFile(absoluteEntry, sourceText);
   const diagnostics = new DiagnosticBag();
   const parser = new Parser(file, diagnostics);
@@ -170,7 +171,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
   const optimize = options.optimize ?? "2";
 
   const absoluteEntry = resolve(entryPath);
-  const sourceText = readFileSync(absoluteEntry, "utf8");
+  const sourceText = sourceTextBytes(readFileSync(absoluteEntry, "utf8"));
 
   const baseName = basename(absoluteEntry, extname(absoluteEntry));
   const outputPath =

package/src/lexer/scanner/strings.ts

@@ -6,8 +6,21 @@
 import { DiagnosticCode } from "../../diagnostics/diagnostic.js";
 import { Token, TokenKind } from "../token.js";
 import { Char, isDigit, isLineBreak } from "../char.js";
+import { codePointBytes } from "../../codegen/generator/tables.js";
 import type { Scanner } from "../scanner.js";
 
+/**
+ * The bytes one escape sequence stands for. An escape names a code point, and a
+ * literal holds the UTF-8 bytes of that code point - the same bytes
+ * `String.fromCodePoint` produces at run time - spelled as one code unit per
+ * byte, so both hosts agree on what a literal's text is.
+ */
+function escapeText(code: number): string {
+  let text = "";
+  for (const byte of codePointBytes(code)) text += String.fromCharCode(byte);
+  return text;
+}
+
 export interface StringMethods {
   scanString(this: Scanner, start: number, precededByLineBreak: boolean, quote: number): Token;
   scanEscapeSequence(this: Scanner): string;
@@ -65,16 +78,16 @@ export const stringMethods: StringMethods = {
           const close = this.text.indexOf("}", this.pos);
           const hex = this.text.slice(this.pos + 1, close);
           this.pos = close + 1;
-          return String.fromCodePoint(parseInt(hex, 16) || 0);
+          return escapeText(parseInt(hex, 16) || 0);
         } else {
           const hex = this.text.slice(this.pos, this.pos + 4);
           this.pos += 4;
-          return String.fromCharCode(parseInt(hex, 16) || 0);
+          return escapeText(parseInt(hex, 16) || 0);
         }
       case Char.LowerX: {
         const hex = this.text.slice(this.pos + 1, this.pos + 3);
         this.pos += 3;
-        return String.fromCharCode(parseInt(hex, 16) || 0);
+        return escapeText(parseInt(hex, 16) || 0);
       }
       case Char.Zero:
         // \0 is only a null escape when not followed by a digit.