romdev-mcp 0.1.1

package/src/toolchains/_worker/pool.js

@@ -0,0 +1,253 @@
+// pool.js — child-process pool for WASM toolchain isolation.
+//
+// Spawns N child workers (wasm-worker.js). Acquire/release semantics:
+// when a caller wants to run a build, it grabs an idle worker, sends the
+// job, awaits the result. If the worker crashes (SIGSEGV, native abort,
+// OOM, anything that exits the process before answering), the pool
+// detects the unexpected exit, REJECTS the in-flight job with
+// {ok:false, stage:"crash"}, and replaces the worker so the next caller
+// gets a fresh one.
+//
+// This is the chokepoint that gives MCP the "one bad build can't kill
+// the server" property requested in Round 11 feedback.
+
+import { fork } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const WORKER_PATH = path.join(__dirname, "wasm-worker.js");
+
+const POOL_SIZE = Number(process.env.ROM_DEV_WASM_POOL_SIZE ?? 2);
+
+/**
+ * @typedef {Object} Worker
+ * @property {import("node:child_process").ChildProcess} child
+ * @property {"idle"|"busy"|"dead"} state
+ * @property {number} id
+ * @property {((res:any)=>void)|null} pendingResolve
+ * @property {((err:any)=>void)|null} pendingReject
+ * @property {number|null} pendingJobId
+ */
+
+/** @type {Worker[]} */
+const workers = [];
+/** @type {Array<(w:Worker)=>void>} */
+const waiters = [];
+let nextWorkerId = 1;
+let nextJobId = 1;
+
+function spawnWorker() {
+  const child = fork(WORKER_PATH, [], {
+    // Use IPC, inherit stderr so worker logs surface during dev.
+    stdio: ["ignore", "ignore", "inherit", "ipc"],
+    // Pass through env so EMSDK paths, etc. work.
+    env: process.env,
+    // Don't inherit the parent's `--test`, `--inspect`, etc. node-level
+    // flags. Workers run ordinary scripts; inheriting parent execArgv
+    // makes them run as test files which never exits + double-loads modules.
+    execArgv: [],
+  });
+  /** @type {Worker} */
+  const w = {
+    child,
+    state: "busy",  // become idle after the first "ready"
+    id: nextWorkerId++,
+    pendingResolve: null,
+    pendingReject: null,
+    pendingJobId: null,
+  };
+
+  w.child.on("message", (msg) => {
+    if (msg.type === "ready") {
+      // Transition to idle and hand the worker to the next waiter.
+      w.state = "idle";
+      const waiter = waiters.shift();
+      if (waiter) {
+        waiter(w);
+      } else {
+        // Nothing waiting — unref so this idle worker doesn't hold the
+        // parent's event loop open. Re-ref'd on next acquire.
+        w.child.unref();
+        w.child.channel?.unref?.();
+      }
+      return;
+    }
+    if (msg.type === "pong") {
+      // health check
+      return;
+    }
+    if (msg.type === "prewarmed") {
+      // ack — pool decided to prewarm a module
+      return;
+    }
+    if (msg.type === "result" && w.pendingResolve) {
+      const resolve = w.pendingResolve;
+      w.pendingResolve = null;
+      w.pendingReject = null;
+      w.pendingJobId = null;
+      resolve(msg.result);
+      return;
+    }
+    if (msg.type === "error" && w.pendingReject) {
+      const reject = w.pendingReject;
+      w.pendingResolve = null;
+      w.pendingReject = null;
+      w.pendingJobId = null;
+      reject(new Error(`[worker ${w.id}] ${msg.error}`));
+      return;
+    }
+  });
+
+  w.child.on("exit", (code, signal) => {
+    w.state = "dead";
+    // If we had a job in flight, fail it with crash metadata.
+    if (w.pendingReject) {
+      const reject = w.pendingReject;
+      w.pendingResolve = null;
+      w.pendingReject = null;
+      const err = new Error(
+        `worker ${w.id} exited unexpectedly (code=${code} signal=${signal}) ` +
+        `while running job ${w.pendingJobId}`
+      );
+      /** @type {any} */(err).crash = { exitCode: code, signal: signal ?? null };
+      w.pendingJobId = null;
+      reject(err);
+    }
+    // Remove from pool + spawn replacement so capacity is preserved.
+    const idx = workers.indexOf(w);
+    if (idx >= 0) workers.splice(idx, 1);
+    // Replenish (only if we still want a pool — process not shutting down).
+    if (!shuttingDown) {
+      const replacement = spawnWorker();
+      workers.push(replacement);
+    }
+  });
+
+  w.child.on("error", (e) => {
+    console.error(`[wasm-worker-pool] worker ${w.id} error:`, e);
+  });
+
+  return w;
+}
+
+let shuttingDown = false;
+let initialized = false;
+
+function ensureInit() {
+  if (initialized) return;
+  initialized = true;
+  for (let i = 0; i < POOL_SIZE; i++) {
+    workers.push(spawnWorker());
+  }
+}
+
+/** Acquire an idle worker, blocking until one is available. */
+function acquire() {
+  ensureInit();
+  return new Promise((resolve) => {
+    // Look for an already-idle worker.
+    const idle = workers.find((w) => w.state === "idle");
+    if (idle) {
+      idle.state = "busy";
+      // Re-ref since idle workers are unref'd to keep the event loop free.
+      idle.child.ref();
+      idle.child.channel?.ref?.();
+      resolve(idle);
+      return;
+    }
+    // Queue up — the 'ready' handler will deliver to us.
+    waiters.push((w) => {
+      w.state = "busy";
+      // Worker is already ref'd in spawn or was ref'd when previous job
+      // started; the 'ready' handler only unrefs when no waiter exists.
+      resolve(w);
+    });
+  });
+}
+
+/**
+ * Run a WASM toolchain job in an isolated worker.
+ *
+ * @param {{
+ *   gluePath: string,
+ *   argv: string[],
+ *   stdinText?: string,
+ *   inputFiles?: Array<{vfsPath:string, encoding:"utf8"|"base64", data:string}>,
+ *   hostDirMounts?: Array<{hostDir:string, vfsDir:string}>,
+ *   outputFiles?: Array<{vfsPath:string, encoding:"utf8"|"base64"}>,
+ * }} job
+ * @returns {Promise<{exitCode:number, log:string, outputs:Record<string,string>, crash?:{exitCode:number|null, signal:string|null}}>}
+ */
+export async function runInWorker(job) {
+  const w = await acquire();
+  const id = nextJobId++;
+  return new Promise((resolve, reject) => {
+    const settle = (result, err) => {
+      if (err) reject(err); else resolve(result);
+    };
+    w.pendingResolve = (result) => settle(result);
+    w.pendingReject = (err) => {
+      if (err && err.crash) {
+        // Convert crash to a normal result so callers don't have to catch.
+        settle({
+          exitCode: err.crash.exitCode ?? -1,
+          log: `[crash] worker exited unexpectedly — signal=${err.crash.signal} code=${err.crash.exitCode}\n`,
+          outputs: {},
+          crash: err.crash,
+        });
+      } else {
+        settle(null, err);
+      }
+    };
+    w.pendingJobId = id;
+    w.child.send({ type: "run", id, job });
+  });
+}
+
+/** Test-only: expose the current worker list for crash-isolation tests. */
+export function _internalWorkers() {
+  ensureInit();
+  return workers.slice();
+}
+
+/** Pre-warm a specific WASM module in every worker. Speeds first call. */
+export async function prewarm(gluePath) {
+  ensureInit();
+  // Don't await — fire-and-forget across all current workers.
+  for (const w of workers) {
+    if (w.state === "dead") continue;
+    w.child.send({ type: "prewarm", id: nextJobId++, gluePath });
+  }
+}
+
+/** Shut down all workers (called on process exit). */
+export async function shutdown() {
+  shuttingDown = true;
+  for (const w of workers) {
+    if (w.state === "dead") continue;
+    try {
+      if (w.child.connected) w.child.send({ type: "shutdown" });
+    } catch { /* IPC channel already closed */ }
+  }
+  // Give them a moment to exit cleanly.
+  await new Promise((r) => setTimeout(r, 100));
+  for (const w of workers) {
+    if (!w.child.killed) {
+      try { w.child.kill("SIGTERM"); } catch {}
+    }
+  }
+  workers.length = 0;
+}
+
+// Hook process exit so workers don't outlive us.
+process.on("exit", () => {
+  for (const w of workers) {
+    try { if (!w.child.killed) w.child.kill("SIGTERM"); } catch {}
+  }
+});
+// One-shot async-cleanup on SIGINT/SIGTERM. Use .catch to avoid emitting
+// "ERR_IPC_CHANNEL_CLOSED" stacks when workers are already dead.
+process.on("SIGINT", () => { shutdown().catch(() => {}).finally(() => process.exit(0)); });
+process.on("SIGTERM", () => { shutdown().catch(() => {}).finally(() => process.exit(0)); });

package/src/toolchains/_worker/run.js

@@ -0,0 +1,78 @@
+// run.js — caller-facing wrapper around the worker pool.
+//
+// Each toolchain helper (runCc65, runSdcc, runAsar, runDasm, ...) calls
+// `runIsolated()` with a structured spec instead of the old callback-style
+// `runEmscriptenTool(setup, collect)`. That spec gets handed to a child
+// worker that runs the WASM in isolation — so a WASM abort can't take
+// down the MCP server.
+//
+// The result shape mirrors the old in-process tool: { exitCode, log, ...outputs }.
+
+import { runInWorker } from "./pool.js";
+
+/**
+ * @typedef {Object} InputFile
+ * @property {string} vfsPath  e.g. "/work/main.c"
+ * @property {"utf8" | "base64"} encoding
+ * @property {string} data
+ */
+
+/**
+ * @typedef {Object} HostDirMount
+ * @property {string} hostDir  absolute path on the host
+ * @property {string} vfsDir   target mount path in MEMFS
+ */
+
+/**
+ * @typedef {Object} OutputFile
+ * @property {string} vfsPath
+ * @property {"utf8" | "base64"} encoding
+ */
+
+/**
+ * @typedef {Object} IsolatedJob
+ * @property {string} gluePath
+ * @property {string[]} argv
+ * @property {string} [stdinText]
+ * @property {InputFile[]} [inputFiles]
+ * @property {HostDirMount[]} [hostDirMounts]
+ * @property {OutputFile[]} [outputFiles]
+ */
+
+/**
+ * Run a WASM tool job in an isolated child worker.
+ *
+ * @param {IsolatedJob} job
+ * @returns {Promise<{
+ *   exitCode: number,
+ *   log: string,
+ *   outputs: Record<string, string>,
+ *   crash?: { exitCode: number | null, signal: string | null },
+ * }>}
+ */
+export async function runIsolated(job) {
+  return await runInWorker(job);
+}
+
+/** Convenience: get bytes (Uint8Array) for an output file from the result. */
+export function getOutputBytes(result, vfsPath) {
+  const data = result.outputs?.[vfsPath];
+  if (!data) return null;
+  return new Uint8Array(Buffer.from(data, "base64"));
+}
+
+/** Convenience: get text for an output file from the result. */
+export function getOutputText(result, vfsPath) {
+  return result.outputs?.[vfsPath] ?? "";
+}
+
+/** Convert text → InputFile spec. */
+export function textFile(vfsPath, content) {
+  return { vfsPath, encoding: "utf8", data: content };
+}
+
+/** Convert binary (Uint8Array / Buffer) → InputFile spec. */
+export function binaryFile(vfsPath, bytes) {
+  const buf = bytes instanceof Buffer ? bytes : Buffer.from(bytes);
+  return { vfsPath, encoding: "base64", data: buf.toString("base64") };
+}

package/src/toolchains/_worker/wasm-worker.js

@@ -0,0 +1,233 @@
+// wasm-worker.js — child process that runs ONE WASM toolchain call in isolation.
+//
+// Spawned as `child_process.fork(__filename)` by the pool. Listens for jobs
+// over IPC, runs them, returns results. If the WASM aborts / SIGSEGVs / OOMs,
+// the child exits and the parent sees the exit code — the MCP server stays
+// up regardless.
+//
+// Protocol (parent → child, via process.send):
+//   { type: "run", job }                      run a build job
+//   { type: "prewarm", gluePath }             instantiate a module to warm cache
+//   { type: "ping" }                          health check
+//   { type: "shutdown" }                      exit cleanly
+//
+// Protocol (child → parent):
+//   { type: "ready" }                         after each job, available again
+//   { type: "result", id, result }            job complete
+//   { type: "error", id, error }              job threw something catchable
+//   { type: "pong" }                          ping reply
+//
+// Crashes (uncaught WASM Abort, native SIGSEGV, OOM) → process exits;
+// parent's 'exit' handler reports {ok:false, stage:"crash"} to the caller.
+
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+
+// Keep a small per-worker cache of loaded module factories. The factory is
+// what the emcc glue exports; calling it returns an instantiated module
+// (which is what we throw away each call — the factory itself is reusable
+// because emscripten supports it via wasmBinary reuse).
+const factoryCache = new Map();   // gluePath → { factory, wasmBinary }
+
+async function loadFactory(gluePath) {
+  const cached = factoryCache.get(gluePath);
+  if (cached) return cached;
+  // Accept .js (CommonJS glue) or .mjs (ES-module glue). emcc's EXPORT_ES6=1
+  // output is .mjs-style (uses import.meta.url, dynamic require); some of
+  // our older toolchains use .js. The sibling .wasm is always alongside.
+  const wasmPath = gluePath.replace(/\.(m?js)$/, ".wasm");
+  const wasmBinary = await readFile(wasmPath);
+  const factory = (await import(gluePath)).default;
+  const entry = { factory, wasmBinary };
+  factoryCache.set(gluePath, entry);
+  return entry;
+}
+
+function ensureDir(FS, dir) {
+  const parts = dir.split("/").filter(Boolean);
+  let cur = "";
+  for (const p of parts) {
+    cur += "/" + p;
+    try { FS.mkdir(cur); } catch {}
+  }
+}
+
+async function preloadHostDir(FS, hostDir, mfsDir) {
+  ensureDir(FS, mfsDir);
+  const entries = await readdir(hostDir, { withFileTypes: true });
+  for (const e of entries) {
+    const hp = path.join(hostDir, e.name);
+    const mp = mfsDir + "/" + e.name;
+    if (e.isDirectory()) {
+      await preloadHostDir(FS, hp, mp);
+    } else if (e.isFile()) {
+      const data = await readFile(hp);
+      FS.writeFile(mp, new Uint8Array(data));
+    }
+  }
+}
+
+/**
+ * Execute one job inside this worker.
+ *
+ * @param {{
+ *   gluePath: string,
+ *   argv: string[],
+ *   stdinText?: string,
+ *   inputFiles?: Array<{vfsPath:string, encoding:"utf8"|"base64", data:string}>,
+ *   hostDirMounts?: Array<{hostDir:string, vfsDir:string}>,
+ *   outputFiles?: Array<{vfsPath:string, encoding:"utf8"|"base64"}>,
+ * }} job
+ * @returns {Promise<{exitCode:number, log:string, outputs:Record<string,string>}>}
+ */
+async function runJob(job) {
+  const { factory, wasmBinary } = await loadFactory(job.gluePath);
+
+  let log = "";
+  const print = (msg) => { log += msg + "\n"; };
+
+  let capturedExit = null;
+
+  // Build moduleArgs. Inject stdin via a small custom stdin function if the
+  // job provides one (SDCC sdcpp/sdcc use this).
+  const moduleArgs = {
+    wasmBinary,
+    noInitialRun: true,
+    print,
+    printErr: print,
+    quit: (status, toThrow) => {
+      capturedExit = status;
+      throw toThrow ?? new Error("exit " + status);
+    },
+    onExit: (status) => { capturedExit = status; },
+  };
+  if (job.stdinText != null) {
+    let idx = 0;
+    const buf = Buffer.from(job.stdinText, "utf8");
+    moduleArgs.stdin = () => {
+      if (idx >= buf.length) return null;
+      return buf[idx++];
+    };
+  }
+
+  const mod = await factory(moduleArgs);
+
+  // Populate MEMFS from the job spec.
+  for (const m of (job.hostDirMounts ?? [])) {
+    await preloadHostDir(mod.FS, m.hostDir, m.vfsDir);
+  }
+  for (const f of (job.inputFiles ?? [])) {
+    const parent = path.posix.dirname(f.vfsPath);
+    if (parent !== "/" && parent !== ".") ensureDir(mod.FS, parent);
+    const bytes = f.encoding === "base64"
+      ? new Uint8Array(Buffer.from(f.data, "base64"))
+      : f.data;
+    mod.FS.writeFile(f.vfsPath, bytes);
+  }
+  // Pre-create parent directories for declared output files so tools
+  // that write to e.g. /work/out.s don't ENOENT when /work doesn't exist
+  // (which happens when the job has no inputFiles, only stdin + outputs).
+  for (const f of (job.outputFiles ?? [])) {
+    const parent = path.posix.dirname(f.vfsPath);
+    if (parent !== "/" && parent !== ".") ensureDir(mod.FS, parent);
+  }
+
+  // Run callMain. Catch the "quit/exit threw" pattern emcc uses.
+  let exitCode = 0;
+  const originalExit = process.exit;
+  process.exit = (c) => {
+    capturedExit = c ?? 0;
+    throw new Error("intercepted exit " + capturedExit);
+  };
+  try {
+    mod.callMain(job.argv);
+  } catch (e) {
+    if (e && typeof e === "object" && "status" in e) {
+      exitCode = /** @type {any} */ (e).status;
+    } else if (capturedExit !== null) {
+      exitCode = capturedExit;
+    } else {
+      // Non-exit throw — bubble up so the worker reports it as an error.
+      // Won't crash the worker (caught here), but the build is recorded as failed.
+      log += `\n[worker] Abort in WASM: ${e?.message ?? e}\n`;
+      exitCode = exitCode || 1;
+    }
+  } finally {
+    process.exit = originalExit;
+  }
+  if (capturedExit !== null && exitCode === 0) exitCode = capturedExit;
+  if (process.exitCode != null && process.exitCode !== 0) {
+    if (exitCode === 0) exitCode = process.exitCode;
+    process.exitCode = undefined;
+  }
+
+  // Collect outputs.
+  /** @type {Record<string, string>} */
+  const outputs = {};
+  for (const f of (job.outputFiles ?? [])) {
+    try {
+      const bytes = mod.FS.readFile(f.vfsPath);
+      outputs[f.vfsPath] = f.encoding === "base64"
+        ? Buffer.from(bytes).toString("base64")
+        : new TextDecoder().decode(bytes);
+    } catch {
+      // file absent — caller decides what's required
+      outputs[f.vfsPath] = "";
+    }
+  }
+
+  return { exitCode, log, outputs };
+}
+
+// ── IPC dispatch ────────────────────────────────────────────────
+
+if (typeof process.send !== "function") {
+  console.error("wasm-worker.js must be launched via child_process.fork");
+  process.exit(2);
+}
+
+// Signal the parent we're ready to take jobs.
+process.send({ type: "ready" });
+
+process.on("message", async (msg) => {
+  try {
+    if (msg.type === "ping") {
+      process.send({ type: "pong" });
+      return;
+    }
+    if (msg.type === "shutdown") {
+      process.exit(0);
+    }
+    if (msg.type === "prewarm") {
+      await loadFactory(msg.gluePath);
+      process.send({ type: "prewarmed", id: msg.id, gluePath: msg.gluePath });
+      return;
+    }
+    if (msg.type === "run") {
+      const result = await runJob(msg.job);
+      process.send({ type: "result", id: msg.id, result });
+      // After every job, signal ready again so the pool can recycle us.
+      process.send({ type: "ready" });
+      return;
+    }
+    process.send({ type: "error", id: msg.id, error: `unknown message type: ${msg.type}` });
+  } catch (e) {
+    // This catches JS-level errors inside the worker (e.g. file-not-found,
+    // bad job spec). Real WASM aborts that bypass our handlers will crash
+    // the process — the parent sees that via the 'exit' event with a
+    // non-zero code, not via this path.
+    process.send({
+      type: "error",
+      id: msg?.id,
+      error: e?.stack ?? e?.message ?? String(e),
+    });
+    process.send({ type: "ready" });
+  }
+});
+
+// Unhandled rejections inside emcc's async code occasionally land here.
+// Log them, but the process keeps going so the parent's idle-pool logic
+// can decide what to do.
+process.on("unhandledRejection", (reason) => {
+  console.error("[wasm-worker] unhandledRejection:", reason);
+});