pi-supernova 0.8.2 → 0.9.0

package/src/runtime/guest-worker.js

@@ -1,14 +1,16 @@
+import {buildGuestApi} from "./guest-api.js";
+import {packageFinalReturn} from "../output/final.js";
 import { parentPort } from "node:worker_threads";
-import { AsyncLocalStorage } from "node:async_hooks";
+
 import * as nodeModule from "node:module";
-import { isString, isObject, isFunction, toPlain, looksLikePath } from "../shared/decode.js";
-import { truncateChars } from "../output/format.js";
-import { gatherReadArgs, normalizeRead, decodeReadValue, assertReadPaths } from "../contract/read.js";
-import { classifyEdit } from "../contract/edit.js";
-import { normalizeBash } from "../contract/bash.js";
+import { errorMessage, isString, isObject, isFunction, toPlain } from "../shared/decode.js";
+import { truncateChars,formatReturn,displayExceeds,formatBoundedValue } from "../output/format.js";
+
 import { guestImportMessage, isDeniedGuestImport } from "./guest-deny-imports.js";
 
 const { register, registerHooks } = nodeModule;
+// heapUsed/external belong to this worker; rss includes the entire Pi/OMP host.
+const memoryUsage = process.memoryUsage.bind(process);
 
 if (isFunction(registerHooks)) {
   registerHooks({
@@ -41,21 +43,17 @@ const PARAMS = ["console", "read", "edit", "write", "bash"];
 
 const BODY_LINE_OFFSET = 2;
 
-const QUERY_URI = /^(agent|artifact):\/\/.*\?/i;
-
-const PER_PATH_HINT = "; use Promise.allSettled(paths.map(path => read(path))) for per-path outcomes";
-
-/** Best-effort guest line:col from an error stack (V8 "<anonymous>:L:C", JSC "eval code").*/
+/** Only match the guest source, never an anonymous callback in the host bridge. */
 function guestLocation(err) {
   const stack = String(err?.stack);
-  const m = /(?:<anonymous>:|(?:eval code|anonymous)(?:@|:))(\d+):(\d+)/.exec(stack);
+  const m = /^\s+at (async )?[^\n]*\(supernova-guest\.js:(\d+):(\d+)\)$/m.exec(stack);
 
   if (!m) return null;
-  const line = Number(m[1]) - BODY_LINE_OFFSET;
+  const line = Number(m[2]) - BODY_LINE_OFFSET;
 
   if (line < 1) return null;
 
-  return { line, col: Number(m[2]) };
+  return { line, col: Number(m[3]), awaited: Boolean(m[1]) };
 }
 
 let activeRunId = 0;
@@ -70,15 +68,23 @@ function post(msg) {
   parentPort.postMessage(msg);
 }
 
-function callRpc(runId, method, args) {
+function reportMemory(runId) {
+  if (!runActive || runId !== activeRunId) return;
+  const { heapUsed, external } = memoryUsage();
+  // arrayBuffers is already included in external; do not double-charge it.
+  post({ op: "memory", runId, heapUsed, external });
+}
+
+function callRpc(runId, method, args, onItem) {
   if (!runActive || runId !== activeRunId) return Promise.reject(new Error("program is already complete"));
 
   return new Promise((resolve, reject) => {
     const id = ++rpcSeq;
-    pendingRpc.set(id, { resolve, reject });
+    pendingRpc.set(id, { resolve, reject, onItem });
 
     try {
-      post({ op: "rpc", id, runId, method, args });
+      reportMemory(runId);
+      post({ op: "rpc", id, runId, method, args, streamRead: Boolean(onItem) });
     } catch (err) {
       pendingRpc.delete(id);
       reject(new Error("nova." + method + " arguments are not transferable: " + err?.message));
@@ -86,202 +92,25 @@ function callRpc(runId, method, args) {
   });
 }
 
-function unwrapValue(res) {
-  if (res?.ok === false) throw new Error(String(res.value ?? res.error ?? "host tool failed"));
-
-  if ("value" in Object(res)) return res.value;
-
-  return res;
-}
-
-function unwrapRead(res, args) {
-  const value = unwrapValue(res);
-
-  if ((args.complete === true || args.json !== undefined) && res?.truncated) throw new Error("incomplete read: complete:true or json refuses truncated host output");
-
-  return value;
-}
-
-/** Keep details/truncated reachable but out of the returned literal unless they carry signal. */
-function leanEnvelope(res) {
-  if (!isObject(res)) return res;
-
-  if ("details" in res) Object.defineProperty(res, "details", { value: res.details, enumerable: false, writable: true });
-
-  for (const key of Object.keys(res)) if (res[key] === undefined) delete res[key];
-
-  if (res.truncated === false) delete res.truncated;
-
-  return res;
-}
-
-function swallow(promise) {
-  promise.catch(() => {});
-
-  return promise;
-}
-
-function throwReadPathError(target, detail) {
-  throw new Error(`read failed for ${target}: ${detail}${PER_PATH_HINT}`);
-}
-
-function isQueryUri(item) {
-  return QUERY_URI.test(item);
-}
-
-function firstItemErrorIndex(res) {
-  return res?.itemErrors?.findIndex(error => error != null) ?? -1;
-}
-
-function missingResolvedIndex(values, paths) {
-  return values.findIndex((value, index) => value?.status === "not_found" && looksLikePath(paths[index]));
-}
-
-async function rpcReadWave(rpc, wave) {
-  if (wave.length === 1) {
-    const res = leanEnvelope(await rpc("call", ["read", wave[0].args]));
-
-    return { values: [unwrapRead(res, wave[0].args)], errors: [] };
-  }
-  const args = { ...wave[0].args, path: wave.map(job => job.args.path), _independent: true };
-  const res = leanEnvelope(await rpc("call", ["read", args]));
-  unwrapRead(res, args);
-
-  return { values: res.items, errors: res.itemErrors ?? [] };
-}
-
-function settleReadWave(wave, values, errors) {
-  if (!Array.isArray(values) || values.length !== wave.length) throw new Error("invalid batch read response");
-
-  for (let i = 0; i < wave.length; i++) {
-    if (errors[i]) wave[i].reject(new Error(errors[i]));
-    else wave[i].resolve(values[i]);
+function encodeConsoleArg(a, limit) {
+  if (isString(a)) {
+    const raw = truncateChars(a,limit,"log");
+    const encoded = truncateChars(formatReturn(raw.text),limit,"log");
+    encoded.truncated ||= raw.truncated;
+    return encoded;
   }
-}
-
-function rejectReadWave(wave, error) {
-  for (const job of wave) job.reject(error);
-}
-
-function dispatchReadWaves(pending, pendingReadWaves, enqueueHost, rpc) {
-  for (let start = 0; start < pending.length; start += 64) {
-    const wave = pending.slice(start, start + 64);
-    const delivery = enqueueHost(() => rpcReadWave(rpc, wave))
-      .then(({ values, errors }) => settleReadWave(wave, values, errors))
-      .catch(error => rejectReadWave(wave, error));
-
-    pendingReadWaves.add(delivery);
-    void delivery.finally(() => pendingReadWaves.delete(delivery));
-  }
-}
-
-function enqueueCompatibleRead(readState, flushReads, args, decode) {
-  const key = JSON.stringify({ ...args, path: undefined });
-
-  if (readState.queued.length && readState.queued[0].key !== key) flushReads();
-
-  const promise = new Promise((resolve, reject) => {
-    readState.queued.push({ args, key, resolve, reject });
-
-    if (readState.queued.length === 1) queueMicrotask(flushReads);
-  }).then(decode);
-
-  return swallow(promise);
-}
-
-async function readManyPaths(readOne, invoke, batchRead, args, paths, decode) {
-  assertReadPaths(paths);
-  const readEach = async () => {
-    const values = await Promise.all(paths.map(item => readOne({ ...args, path: item })));
-    const missing = args.resolve ? missingResolvedIndex(values, paths) : -1;
-
-    if (missing >= 0) throwReadPathError(paths[missing], "not_found");
-
-    return values;
-  };
-
-  const guardedReadEach = () => swallow(readEach());
-
-  if (!batchRead || args.resolve || paths.some(isQueryUri)) return guardedReadEach();
-  const res = await invoke("read", args);
-  const failed = firstItemErrorIndex(res);
-
-  if (failed >= 0) throwReadPathError(paths[failed], res.itemErrors[failed]);
-  unwrapRead(res, args);
-
-  if (Array.isArray(res?.items)) return res.items.map(decode);
-
-  return guardedReadEach();
-}
-
-function attachCallManyMeta(wave) {
-  const results = Array.isArray(wave?.results) ? wave.results : Array.isArray(wave) ? wave : [];
-  Object.defineProperties(results, {
-    mode: { value: wave?.mode, enumerable: false },
-    reason: { value: wave?.reason, enumerable: false },
-    results: { value: results, enumerable: false },
-  });
-
-  return results;
-}
-
-async function runSpeculation(fn, token, checkpointScope, drainReads, enqueueHost, rpc) {
-  let began = false;
-
-  try {
-    await drainReads();
-    await enqueueHost(() => rpc("speculateBegin", []));
-    began = true;
-    const value = await checkpointScope.run(token, fn);
-    await drainReads();
-    await enqueueHost(() => rpc("speculateCommit", []));
-
-    return { ok: true, committed: true, value };
-  } catch (err) {
-    await drainReads();
-
-    if (began) await enqueueHost(() => rpc("speculateRollback", []));
-
-    throw err;
-  }
-}
-
-function formatBashFailure(command, res) {
-  let exitCode;
-
-  try {
-    exitCode = JSON.parse(res.details).exitCode;
-  } catch {}
-
-  const output = String(res.value).trimEnd();
-  const suffix = Number.isInteger(exitCode) ? " (exit " + exitCode + ")" : "";
-
-  return "command failed" + suffix + ": " + truncateChars(command, 240, "command").text + (output ? "\n" + output : "");
-}
-
-function markTruncatedOutput(res, text) {
-  if (res?.truncated && isString(text) && !text.includes("truncated")) return text + "\n…[output truncated]…";
-
-  return text;
-}
-
-function encodeConsoleArg(a) {
-  if (isString(a)) return a;
-
   try {
     const plain = toPlain(a);
+    if (displayExceeds(plain,limit)) return {text:formatBoundedValue(plain,limit,"log"),truncated:true};
     const encoded = JSON.stringify(plain);
-
-    return encoded === undefined ? String(plain) : encoded;
-  } catch {
-    return String(a);
-  }
+    return truncateChars(encoded === undefined ? String(plain) : encoded,limit,"log");
+  } catch { return truncateChars(formatReturn(String(a)),limit,"log"); }
 }
 
 function compileGuest(prepared, data) {
   const bindings = data === undefined ? PARAMS : [...PARAMS, "data"];
 
-  return { fn: new AsyncFunction(...bindings, prepared.body), hasReturn: prepared.hasReturn };
+  return { fn: new AsyncFunction(...bindings, prepared.body + "\n//# sourceURL=supernova-guest.js"), hasReturn: prepared.hasReturn };
 }
 
 function plainGuestValue(value) {
@@ -299,129 +128,14 @@ function handleRpcResult(msg) {
   else pending.reject(new Error(msg.error));
 }
 
-function buildGuestApi(available, batchRead, runId, _nativeArgv) {
-  const rpc = (method, args) => callRpc(runId, method, args);
-  const checkpointScope = new AsyncLocalStorage();
-  let checkpoint = null;
-
-  const assertScope = () => {
-    const token = checkpointScope.getStore();
-
-    if (token ? token !== checkpoint : checkpoint !== null) throw new Error("await the active edit checkpoint before issuing other commands; completed checkpoints cannot issue commands");
-  };
-
-  let operationTail = Promise.resolve();
-
-  const enqueueHost = operation => {
-    const next = operationTail.then(operation);
-    operationTail = next.then(() => {}, () => {});
-
-    return next;
-  };
-
-  const nova = {
-    call(name, args) {
-      assertScope(); flushReads();
-
-      return swallow(enqueueHost(() => rpc("call", [name, args]).then(leanEnvelope)));
-    },
-    callMany(calls) {
-      assertScope(); flushReads();
-
-      return swallow(enqueueHost(async () => attachCallManyMeta(await rpc("callMany", [calls]))));
-    },
-    async speculate(fn) {
-      if (checkpoint) throw new Error("edit checkpoints cannot overlap or nest; await the current checkpoint");
-      const token = {};
-      checkpoint = token;
-
-      try { return await runSpeculation(fn, token, checkpointScope, drainReads, enqueueHost, rpc); }
-      finally { checkpoint = null; }
-    },
-  };
-
-  // Coalesce already-started compatible reads without rewriting JS control flow.
-  const readState = { queued: [], waves: new Set() };
-
-  function flushReads() {
-    const pending = readState.queued;
-    readState.queued = [];
-    dispatchReadWaves(pending, readState.waves, enqueueHost, rpc);
-  }
-
-  async function drainReads() {
-    for (;;) {
-      flushReads();
-      const waves = [...readState.waves];
-
-      if (!waves.length) return;
-      await Promise.allSettled(waves);
-    }
-  }
-
-  const invoke = (name, args) => {
-    assertScope(); flushReads();
-
-    return swallow(nova.call(name, args));
-  };
-
-  const read = async (p, a, b) => {
-    assertScope();
-    const args = normalizeRead(gatherReadArgs(p, a, b));
-    const decode = value => decodeReadValue(args, value);
-    p = args.path;
-
-    if (Array.isArray(p)) {
-      const values = await readManyPaths(read, invoke, batchRead, args, p, decode);
-      for (const item of p) if (isString(item)) noteReadPath({ path: item }, values);
-
-      return values;
-    }
-
-    const readValue = !batchRead
-      ? decode(unwrapRead(await invoke("read", args), args))
-      : await enqueueCompatibleRead(readState, flushReads, args, decode);
-    noteReadPath(args, readValue);
-
-    return readValue;
-  };
-
-  const readFiles = new Set();
-
-  function noteReadPath(args, value) {
-    if (isString(args.path) && looksLikePath(args.path)) readFiles.add(args.path);
-    if (isObject(value) && isString(value.path) && looksLikePath(value.path)) readFiles.add(value.path);
-  }
-
-  const write = async (p, content) => {
-    const args = isObject(p) ? p : { path: p, content };
-
-    if (args.append !== true && args.replace !== true && isString(args.path) && readFiles.has(args.path)) {
-      throw new Error("file was already read this program; use edit(oldText, newText) or edit(view, ...). write({path,content,replace:true}) replaces anyway");
-    }
-
-    return unwrapValue(await invoke("write", args));
-  };
-
-  const edit = async (p, oldText, newText) => {
-    const classified = classifyEdit(p, oldText, newText);
-
-    if (classified.kind === "checkpoint") return nova.speculate(classified.fn);
-
-    return unwrapValue(await invoke(classified.command, classified.args));
-  };
-
-  const bash = async (command, opts) => {
-    const args = normalizeBash(command, opts);
-    command = args.command;
-    const res = await invoke("bash", args);
-
-    if (res?.ok === false) throw new Error(formatBashFailure(command, res));
-
-    return markTruncatedOutput(res, unwrapValue(res));
-  };
-
-  return { read, write, edit, bash, nova };
+function handleReadItem(msg) {
+  const pending = pendingRpc.get(msg.id);
+  if (!pending?.onItem || !runActive || msg.runId !== activeRunId) return;
+  let error;
+  try { pending.onItem(msg.index,msg.value); }
+  catch (err) { error = errorMessage(err); pending.reject(err); pendingRpc.delete(msg.id); }
+  reportMemory(activeRunId);
+  post({op:"rpc:ack",id:msg.id,index:msg.index,runId:activeRunId,error});
 }
 
 function makeConsole(runId, limits) {
@@ -439,7 +153,15 @@ function makeConsole(runId, limits) {
  return; }
 
     count++;
-    const line = args.map(encodeConsoleArg).join(" ");
+    let line = "", first = true;
+    for (const arg of args) {
+      const room = limits.maxLogLineChars-line.length-(first ? 0 : 1);
+      if (room <= 0) { markTruncated(); break; }
+      const encoded = encodeConsoleArg(arg,room);
+      if (encoded.truncated) markTruncated();
+      line += (first ? "" : " ")+encoded.text;
+      first = false;
+    }
     const clipped = truncateChars(line, limits.maxLogLineChars, "log");
 
     if (clipped.truncated) markTruncated();
@@ -451,7 +173,7 @@ function makeConsole(runId, limits) {
 
 function postFailure(runId, err, location) {
   runActive = false;
-  const message = err instanceof Error ? err.message : String(err);
+  const message = errorMessage(err);
   post({ op: "error", runId, message, location });
 }
 
@@ -495,7 +217,7 @@ function sealGuestRealm() {
 }
 
 async function handleRun(msg) {
-  const { runId, prepared, limits, available, batchRead = true } = msg;
+  const { runId, prepared, limits, batchRead = true } = msg;
   activeRunId = runId;
   runActive = true;
   let compiled;
@@ -510,8 +232,9 @@ async function handleRun(msg) {
     return;
   }
 
-  const api = buildGuestApi(available, batchRead, runId, msg.nativeArgv === true);
+  const api = buildGuestApi((method, args, onItem) => callRpc(runId, method, args, onItem), batchRead);
   const scopedConsole = makeConsole(runId, limits);
+  const memoryTimer = setInterval(() => reportMemory(runId), 50);
 
   try {
     const value = await compiled.fn(
@@ -520,17 +243,23 @@ async function handleRun(msg) {
 
     if (runId !== activeRunId) return;
     const plain = plainGuestValue(value);
+    // Reduce model output while it is still covered by worker memory/deadlines.
+    // Only a bounded display and retained images cross back to the Pi/OMP host.
+    const result = msg.formatResult ? {output:packageFinalReturn(plain,[],limits)} : {value:plain};
+    reportMemory(runId);
     runActive = false;
-    post({ op: "done", runId, value: plain, undefinedReturn: value === undefined && !compiled.hasReturn, hasReturn: compiled.hasReturn });
+    post({ op:"done",runId,...result,undefinedReturn:value === undefined && !compiled.hasReturn,hasReturn:compiled.hasReturn });
   } catch (err) {
     if (runId !== activeRunId) return;
     postFailure(runId, err, guestLocation(err));
-  }
+  } finally { clearInterval(memoryTimer); }
 }
 
 parentPort.on("message", (msg) => {
   if (!isObject(msg)) return;
 
+  if (msg.op === "rpc:item") { handleReadItem(msg); return; }
+
   if (msg.op === "rpc:result") {
     handleRpcResult(msg);
 

package/src/runtime/parallel.js

@@ -1,4 +1,4 @@
-import { isFunction, isObject } from "../shared/decode.js";
+import { isFunction, isObject, isString } from "../shared/decode.js";
 
 const READ_ONLY_TOOLS = new Set(["read", "grep", "glob", "find", "ls", "snap", "evidence", "surface", "asgrep_search", "asgrep_status", "ast_grep", "web_search"]);
 
@@ -47,18 +47,18 @@ export function isMutatingTool(name, config = {}, args = {}, definition) {
   return !READ_ONLY_TOOLS.has(name);
 }
 
-function takeScheduledWave(queue, first, maxParallelReads) {
-  const wave = [first];
-
-  if (first.name !== "read") return wave;
-
-  while (wave.length < maxParallelReads && queue[0]?.name === "read") {
-    const next = queue.shift();
-
-    if (!next.cancelled) wave.push(next);
-  }
+function compatibleScheduledJob(job, active) {
+  return [...active].every(running =>
+    (job.name === "read" && running.name === "read") ||
+    (isString(job.key) && isString(running.key) && job.key !== running.key));
+}
 
-  return wave;
+async function prepareScheduledJob(job) {
+  if (!job.resolveKey) return;
+  // Key lookup is advisory. Invalid paths still run in isolation so the real
+  // adapter reports the error through its normal trace/permission checks.
+  try { job.key = await job.resolveKey(); } catch {}
+  job.resolveKey = undefined;
 }
 
 async function runScheduledJob(job) {
@@ -74,12 +74,6 @@ async function runScheduledJob(job) {
   } catch (error) { job.reject(error); }
 }
 
-function runScheduledWave(wave) {
-  // Settle each promise separately: one failed read must not discard its
-  // siblings or release a write barrier while other reads are still active.
-  return Promise.all(wave.map(runScheduledJob));
-}
-
 function attachJobAbort(job, signal, reject) {
   job.abort = () => {
     if (job.started) return;
@@ -91,34 +85,47 @@ function attachJobAbort(job, signal, reject) {
   signal?.addEventListener("abort", job.abort, { once: true });
 }
 
-/** FIFO read waves with mutation barriers; callers keep independent promises. */
+/** FIFO admission: concurrent reads or disjoint native files, otherwise a barrier. */
 export function createNativeScheduler({ maxParallelReads = 8 } = {}) {
   if (!Number.isInteger(maxParallelReads) || maxParallelReads < 1) throw new Error("maxParallelReads must be a positive integer");
   const queue = [];
+  const active = new Set();
   const stats = { calls: 0, readWaves: 0, peakParallelReads: 0 };
   let draining = false;
 
+  function start(job) {
+    if (job.name === "read") {
+      if (!active.size) stats.readWaves++;
+      stats.peakParallelReads = Math.max(stats.peakParallelReads, active.size + 1);
+    }
+    active.add(job);
+    // Each promise settles independently. Barriers still wait for every active
+    // sibling, including when Promise.all in the guest has already rejected.
+    void runScheduledJob(job).finally(() => { active.delete(job); void drain(); });
+  }
+
   async function drain() {
+    if (draining) return;
+    draining = true;
     try {
-      while (queue.length) {
-        const first = queue.shift();
-
-        if (first.cancelled) continue;
-        const wave = takeScheduledWave(queue, first, maxParallelReads);
-
-        if (first.name === "read") {
-          stats.readWaves++;
-          stats.peakParallelReads = Math.max(stats.peakParallelReads, wave.length);
-        }
-
-        await runScheduledWave(wave);
+      while (queue.length && active.size < maxParallelReads) {
+        const job = queue[0];
+        if (job.cancelled) { queue.shift(); continue; }
+        // Resolve identities after shell/override barriers, which may change
+        // symlinks. Check synchronously: a finishing job must not lose its wakeup.
+        if (job.resolveKey && [...active].some(running => !isString(running.key))) break;
+        await prepareScheduledJob(job);
+        if (job.cancelled) continue;
+        if (!compatibleScheduledJob(job, active)) break;
+        queue.shift();
+        start(job);
       }
     } finally { draining = false; }
   }
 
   return {
     stats,
-    schedule(name, run, signal) {
+    schedule(name, run, signal, resolveKey) {
       if (!NATIVE_TOOLS.includes(name) || !isFunction(run)) {
         return Promise.reject(new Error("scheduler requires a native tool and an executor"));
       }
@@ -127,16 +134,11 @@ export function createNativeScheduler({ maxParallelReads = 8 } = {}) {
       stats.calls++;
 
       return new Promise((resolve, reject) => {
-        const job = { name, run, signal, resolve, reject, started: false, cancelled: false };
+        const job = { name, run, signal, resolve, reject, resolveKey, started: false, cancelled: false };
         attachJobAbort(job, signal, reject);
         queue.push(job);
 
-        if (!draining) {
-          draining = true;
-          // Pi submits sibling tools in the same turn without model-side code.
-          // Collect those submissions before selecting the first read wave.
-          queueMicrotask(() => { void drain(); });
-        }
+        void drain();
       });
     },
   };