pi-supernova 0.3.2 → 0.4.0

package/src/output/format.js

@@ -4,7 +4,9 @@ export function truncateChars(text, maxChars, label = "value") {
   const normalized = isString(text) ? text : String(text ?? "");
   const numericLimit = Number(maxChars);
   const limit = Number.isFinite(numericLimit) ? Math.max(0, Math.floor(numericLimit)) : numericLimit === Infinity ? normalized.length : 0;
+
   if (normalized.length <= limit) return { text: normalized, truncated: false };
+
   if (limit <= 100) {
     return {
       text: normalized.slice(0, headEnd(normalized, limit)),
@@ -12,20 +14,25 @@ export function truncateChars(text, maxChars, label = "value") {
       originalChars: normalized.length,
     };
   }
+
   let head = headEnd(normalized, Math.floor(limit * 0.7));
   let tail = 0;
   let marker = "";
+
   for (;;) {
     const omitted = normalized.length - head - tail;
     marker = "\n…[" + label + " truncated " + omitted + " chars]…\n";
+
     if (marker.length > limit) return { text: normalized.slice(0, headEnd(normalized, limit)), truncated: true, originalChars: normalized.length };
     const budget = limit - marker.length;
     const nextHead = headEnd(normalized, Math.min(head, budget));
     const nextTail = normalized.length - tailStartIndex(normalized, Math.max(0, budget - nextHead));
+
     if (nextHead === head && nextTail === tail) break;
     head = nextHead;
     tail = nextTail;
   }
+
   return { text: normalized.slice(0, head) + marker + normalized.slice(normalized.length - tail), truncated: true, originalChars: normalized.length };
 }
 
@@ -33,6 +40,7 @@ export function truncateChars(text, maxChars, label = "value") {
 // boundary, so a cut must never split a surrogate pair.
 function headEnd(text, end) {
   const code = text.charCodeAt(end - 1);
+
   return code >= 0xd800 && code <= 0xdbff ? end - 1 : end;
 }
 
@@ -40,29 +48,71 @@ function tailStartIndex(text, tail) {
   if (tail <= 0) return text.length;
   const start = text.length - tail;
   const code = text.charCodeAt(start);
+
   return code >= 0xdc00 && code <= 0xdfff ? start + 1 : start;
 }
 
 // Unicode mode matches lone surrogate code points, not valid UTF-16 pairs.
 const UNPAIRED_SURROGATE = /[\uD800-\uDFFF]/u;
 
+// Without the Unicode flag this class matches any surrogate code unit, including
+// the halves of a valid pair, so it is a strictly wider test. Source text almost
+// never contains a surrogate at all, and the wide scan decides those strings on
+// its own; only text that really has one pays for the precise Unicode pass.
+const SURROGATE_CODE_UNIT = /[\uD800-\uDFFF]/;
+
+function hasUnpairedSurrogate(text) {
+  return SURROGATE_CODE_UNIT.test(text) && UNPAIRED_SURROGATE.test(text);
+}
+
 function hasWellFormedStrings(values) {
-  for (const value of values) if (!isString(value) || UNPAIRED_SURROGATE.test(value)) return false;
+  for (const value of values) if (!isString(value) || hasUnpairedSurrogate(value)) return false;
+
   return true;
 }
 
 /** Lossless framing for source arrays, not string escaping or source compression. */
 export function formatReturn(value) {
   if (isString(value)) return value;
+
   if (Array.isArray(value) && value.length && hasWellFormedStrings(value) && value.some(text => text.includes("\n"))) {
     const raw = "strings[" + value.length + "]\n" + value.map((text, i) => "[" + i + "] " + text.length + " UTF-16 units\n" + text + "\n").join("");
     const escapedSize = value.reduce((sum, text) => sum + JSON.stringify(text).length, value.length + 1);
+
     if (raw.length < escapedSize) return raw;
   }
-  return formatValue(value);
+
+  const escaped = formatValue(value);
+  const strings = [];
+
+  const visit = input => {
+    if (isString(input) && input.includes("\n") && !hasUnpairedSurrogate(input) && JSON.stringify(input).length - input.length > 64) {
+      const index = strings.push(input) - 1;
+
+      return { [RAW_TEXT]: "raw[" + index + "]" };
+    }
+
+    if (Array.isArray(input)) return input.map(visit);
+
+    if (isObject(input)) return Object.fromEntries(Object.entries(input).map(([key, child]) => [key, visit(child)]));
+
+    return input;
+  };
+
+  const referencedValue = visit(value);
+
+  if (!strings.length) return escaped;
+  // Keep every key, value, duplicate string and byte. References are unquoted
+  // expressions, so literal "raw[0]" values and header-like source cannot collide.
+  const framed = formatValue(referencedValue) + "\nraw strings[" + strings.length + "]\n" + strings.map((text, i) => "raw[" + i + "] " + text.length + " UTF-16 units\n" + text + "\n").join("");
+
+  return framed.length < escaped.length ? framed : escaped;
 }
 
+const RAW_TEXT = Symbol("raw text reference");
+
 const IDENT_KEY = /^[A-Za-z_$][\w$]*$/;
+
 const FORMAT_WIDTH = 120;
 
 function formatKey(key) {
@@ -71,26 +121,67 @@ function formatKey(key) {
 
 function formatPrimitive(value) {
   if (value === undefined) return "undefined";
+
   if (Number.isNaN(value) || value === Infinity || value === -Infinity) return String(value);
+
   return JSON.stringify(value) ?? String(value);
 }
 
-function formatFlatList(value) {
-  if (value.length === 0) return "[]";
-  let out = "[";
-  for (let i = 0; i < value.length; i++) out += (i ? "," : "") + formatFlat(value[i] === undefined ? null : value[i]);
-  return out + "]";
+/**
+ * Single-line rendering that gives up the moment it would exceed `limit` characters.
+ *
+ * formatValue consults this at every level, so the previous build-the-whole-string
+ * form re-walked each subtree once per ancestor: a container that is too wide paid
+ * the full cost of its children and then paid for them again while descending.
+ * Abandoning at the budget keeps the pass linear. Returning null is exactly
+ * equivalent to "the flat form is longer than limit", because an abandoned walk
+ * always has at least `used` characters still to come.
+ */
+function formatFlatWithin(value, limit) {
+  const parts = [];
+  let used = 0;
+
+  const push = (chunk) => {
+    if (used + chunk.length > limit) return false;
+    used += chunk.length;
+    parts.push(chunk);
+
+    return true;
+  };
+
+  return walkFlat(value, push) ? parts.join("") : null;
 }
 
-function formatFlat(value) {
-  if ((!isObject(value) && !Array.isArray(value))) return formatPrimitive(value);
-  if (Array.isArray(value)) return formatFlatList(value);
-  let out = "";
-  for (const key of Object.keys(value)) {
-    if (value[key] === undefined) continue;
-    out += (out ? "," : "{") + formatKey(key) + ":" + formatFlat(value[key]);
+function walkFlat(value, push) {
+  if (value?.[RAW_TEXT] !== undefined) return push(value[RAW_TEXT]);
+
+  if (!isObject(value) && !Array.isArray(value)) return push(formatPrimitive(value));
+
+  if (Array.isArray(value)) {
+    if (value.length === 0) return push("[]");
+
+    if (!push("[")) return false;
+
+    for (let i = 0; i < value.length; i++) {
+      if (i && !push(",")) return false;
+
+      if (!walkFlat(value[i] === undefined ? null : value[i], push)) return false;
+    }
+
+    return push("]");
   }
-  return out ? out + "}" : "{}";
+
+  const keys = Object.keys(value).filter((key) => value[key] !== undefined);
+
+  if (keys.length === 0) return push("{}");
+
+  for (let i = 0; i < keys.length; i++) {
+    if (!push((i ? "," : "{") + formatKey(keys[i]) + ":")) return false;
+
+    if (!walkFlat(value[keys[i]], push)) return false;
+  }
+
+  return push("}");
 }
 
 /**
@@ -100,14 +191,23 @@ function formatFlat(value) {
  * than JSON.stringify(value, null, 2) on typical shaped returns (gpt-tokenizer).
  */
 export function formatValue(value, indent = "", width = FORMAT_WIDTH) {
-  const flat = formatFlat(value);
-  if ((!isObject(value) && !Array.isArray(value)) || flat.length + indent.length <= width) return flat;
+  if (value?.[RAW_TEXT] !== undefined) return value[RAW_TEXT];
+
+  if (!isObject(value) && !Array.isArray(value)) return formatPrimitive(value);
+  const flat = formatFlatWithin(value, width - indent.length);
+
+  if (flat !== null) return flat;
   const pad = indent + " ";
+
   if (Array.isArray(value)) {
     if (value.length === 0) return "[]";
+
     return "[\n" + value.map((item) => pad + formatValue(item === undefined ? null : item, pad, width)).join(",\n") + "\n" + indent + "]";
   }
+
   const keys = Object.keys(value).filter((key) => value[key] !== undefined);
+
   if (keys.length === 0) return "{}";
+
   return "{\n" + keys.map((key) => pad + formatKey(key) + ":" + formatValue(value[key], pad, width)).join(",\n") + "\n" + indent + "}";
 }

package/src/runtime/guest-worker.js

@@ -2,27 +2,37 @@ import { parentPort } from "node:worker_threads";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { isString, isObject, isFunction, toPlain } from "../shared/decode.js";
 import { truncateChars } from "../output/format.js";
+import { sessionJsonArgs, validateJsonRead } from "../fs/json-read.js";
 
 // Guest programs run here, off the host thread. The host can terminate() this
 // worker mid-loop, so a runaway "while (true) {}" or process.exit() in guest
 // code cannot take the harness down with it.
 
 const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor;
+
 const PARAMS = ["console", "read", "edit", "write", "bash"];
+
 const BODY_LINE_OFFSET = 2;
+
 /** Best-effort guest line:col from an error stack (V8 "<anonymous>:L:C", JSC "eval code").*/
 function guestLocation(err) {
   const stack = String(err?.stack);
   const m = /(?:<anonymous>:|(?:eval code|anonymous)(?:@|:))(\d+):(\d+)/.exec(stack);
+
   if (!m) return null;
   const line = Number(m[1]) - BODY_LINE_OFFSET;
+
   if (line < 1) return null;
+
   return { line, col: Number(m[2]) };
 }
 
 let activeRunId = 0;
+
 let runActive = false;
+
 let rpcSeq = 0;
+
 const pendingRpc = new Map();
 
 function post(msg) {
@@ -31,9 +41,11 @@ function post(msg) {
 
 function callRpc(runId, method, args) {
   if (!runActive || runId !== activeRunId) return Promise.reject(new Error("program is already complete"));
+
   return new Promise((resolve, reject) => {
     const id = ++rpcSeq;
     pendingRpc.set(id, { resolve, reject });
+
     try {
       post({ op: "rpc", id, runId, method, args });
     } catch (err) {
@@ -45,18 +57,23 @@ function callRpc(runId, method, args) {
 
 function unwrapValue(res) {
   if (res?.ok === false) throw new Error(String(res.value ?? res.error ?? "host tool failed"));
+
   if ("value" in Object(res)) return res.value;
+
   return res;
 }
 
 function unwrapRead(res, args) {
   const value = unwrapValue(res);
-  if (args.complete === true && res?.truncated) throw new Error("incomplete read: complete:true refuses truncated host output");
+
+  if ((args.complete === true || args.json !== undefined) && res?.truncated) throw new Error("incomplete read: complete:true or json refuses truncated host output");
+
   return value;
 }
 
 function unwrapJsonValue(res) {
   const value = unwrapValue(res);
+
   try {
     return JSON.parse(value);
   } catch {
@@ -67,9 +84,13 @@ function unwrapJsonValue(res) {
 /** Keep details/truncated reachable but out of the returned literal unless they carry signal. */
 function leanEnvelope(res) {
   if (!isObject(res)) return res;
+
   if ("details" in res) Object.defineProperty(res, "details", { value: res.details, enumerable: false, writable: true });
+
   for (const key of Object.keys(res)) if (res[key] === undefined) delete res[key];
+
   if (res.truncated === false) delete res.truncated;
+
   return res;
 }
 
@@ -78,10 +99,13 @@ function buildGuestApi(available, batchRead, runId, nativeArgv) {
   const availableSet = new Set(available);
   const checkpointScope = new AsyncLocalStorage();
   let checkpoint = null;
+
   const assertScope = () => {
     const token = checkpointScope.getStore();
+
     if (token ? token !== checkpoint : checkpoint !== null) throw new Error("await the active edit checkpoint before issuing other commands; completed checkpoints cannot issue commands");
   };
+
   const nova = {
     search: (query, limit) => rpc("search", [query, limit]),
     describe: (name) => rpc("describe", [name]),
@@ -94,6 +118,7 @@ function buildGuestApi(available, batchRead, runId, nativeArgv) {
         reason: { value: wave?.reason, enumerable: false },
         results: { value: results, enumerable: false },
       });
+
       return results;
     },
     async speculate(fn) {
@@ -101,6 +126,7 @@ function buildGuestApi(available, batchRead, runId, nativeArgv) {
       const token = {};
       checkpoint = token;
       let began = false;
+
       try {
         flushReads();
         await rpc("speculateBegin", []);
@@ -108,10 +134,13 @@ function buildGuestApi(available, batchRead, runId, nativeArgv) {
         const value = await checkpointScope.run(token, fn);
         flushReads();
         await rpc("speculateCommit", []);
+
         return { ok: true, committed: true, value };
       } catch (err) {
         flushReads();
+
         if (began) await rpc("speculateRollback", []);
+
         return { ok: false, committed: false, error: err instanceof Error ? err.message : String(err) };
       } finally { checkpoint = null; }
     },
@@ -123,101 +152,171 @@ function buildGuestApi(available, batchRead, runId, nativeArgv) {
 
   // Coalesce already-started compatible reads without rewriting JS control flow.
   let queuedReads = [];
+
   function flushReads() {
     const pending = queuedReads;
     queuedReads = [];
+
     for (let start = 0; start < pending.length; start += 64) {
       const wave = pending.slice(start, start + 64);
       const args = { ...wave[0].args, path: wave.map(job => job.args.path), _independent: true };
+
       const run = wave.length === 1
         ? nova.call("read", wave[0].args).then(res => ({ values: [unwrapRead(res, wave[0].args)], errors: [] }))
-        : nova.call("read", args).then(res => { unwrapRead(res, args); return { values: res.items, errors: res.itemErrors ?? [] }; });
+        : nova.call("read", args).then(res => { unwrapRead(res, args);
+
+ return { values: res.items, errors: res.itemErrors ?? [] }; });
+
       void run.then(({ values, errors }) => {
         if (!Array.isArray(values) || values.length !== wave.length) throw new Error("invalid batch read response");
+
         for (let i = 0; i < wave.length; i++) {
           const value = values[i];
+
           if (errors[i]) wave[i].reject(new Error(errors[i]));
           else wave[i].resolve(value);
         }
       }).catch(error => { for (const job of wave) job.reject(error); });
     }
   }
-  const invoke = (name, args) => { assertScope(); flushReads(); return nova.call(name, args); };
+
+  const invoke = (name, args) => { assertScope(); flushReads();
+
+ return nova.call(name, args); };
+
   const readArgs = (p, a, b) => isObject(p) && !Array.isArray(p)
     ? { ...p, path: p.path ?? p.query }
     : isObject(a) && !Array.isArray(a) ? { path: p, ...a } : { path: p, offset: a, limit: b };
+
   const read = async (p, a, b) => {
     assertScope();
-    const args = readArgs(p, a, b);
+    const args = sessionJsonArgs(readArgs(p, a, b));
+    validateJsonRead(args);
+    const decode = value => args.resolve || args.json !== undefined ? JSON.parse(value) : value;
+
     if (args.complete === true && (args.outline || args.evidence || args.about)) throw new Error("complete:true requires a raw file read, not an outline or evidence view");
     const evidencePath = isObject(p) && !Array.isArray(p) ? p.path : args.about ? p : undefined;
     p = args.path;
+
     if (args.evidence) return unwrapJsonValue(await invoke("evidence", { ...args, path: evidencePath, query: args.about ?? args.query ?? p }));
+
     if (args.outline) return unwrapJsonValue(await invoke("surface", args));
+
     if (Array.isArray(p)) {
       if (p.length > 64) throw new Error("read accepts at most 64 paths per batch");
-      if (p.some(item => !isString(item) || !item.trim())) throw new Error("read paths must be non-empty strings");
+
+      for (const item of p) if (!isString(item) || !item.trim()) throw new Error("read paths must be non-empty strings");
       const readEach = () => Promise.all(p.map(item => read({ ...args, path: item })));
-      if (!batchRead || args.resolve) return readEach();
+
+      if (!batchRead || args.resolve || p.some(item => /^(agent|artifact):\/\/.*\?/i.test(item))) return readEach();
       const res = await invoke("read", args);
       const failed = res?.itemErrors?.findIndex(error => error != null) ?? -1;
+
       if (failed >= 0) throw new Error(`read failed for ${p[failed]}: ${res.itemErrors[failed]}; use Promise.allSettled(paths.map(path => read(path))) for per-path outcomes`);
       unwrapRead(res, args);
-      if (Array.isArray(res?.items)) return res.items;
+
+      if (Array.isArray(res?.items)) return res.items.map(decode);
+
       // Captured host executor without batch support: fan out.
       return readEach();
     }
+
     if (!batchRead) {
       const res = await invoke("read", args);
       unwrapRead(res, args);
-      return args.resolve ? unwrapJsonValue(res) : unwrapRead(res, args);
+
+      return decode(unwrapRead(res, args));
     }
+
     const key = JSON.stringify({ ...args, path: undefined });
+
     if (queuedReads.length && queuedReads[0].key !== key) flushReads();
+
     return new Promise((resolve, reject) => {
       queuedReads.push({ args, key, resolve, reject });
+
       if (queuedReads.length === 1) queueMicrotask(flushReads);
-    }).then(value => args.resolve ? JSON.parse(value) : value);
+    }).then(decode);
   };
+
   const write = async (p, content) => unwrapValue(await invoke("write", isObject(p) ? p : { path: p, content }));
+
   const edit = async (p, oldText, newText) => {
     if (isFunction(p)) return nova.speculate(p);
+    const usage = 'invalid edit signature; use edit(path,oldText,newText), edit({path,edits:[{oldText,newText}]}), or edit({path,patch:"@@ -1 +1 @@\n-old\n+new\n"})';
+
+    if (isObject(p) && (Array.isArray(p) || oldText !== undefined || newText !== undefined)) throw new Error(usage);
+
+    if (!isObject(p) && isObject(oldText) && !Array.isArray(oldText)) throw new Error(usage);
     const args = isObject(p) ? p : Array.isArray(oldText) ? { path: p, edits: oldText } : { path: p, oldText, newText };
+
+    if (!isString(args.path) || !args.path.trim()) throw new Error(usage);
+    const modes = Number(args.patch !== undefined) + Number(args.edits !== undefined) + Number(args.oldText !== undefined || args.newText !== undefined);
+
+    if (modes !== 1 || (Array.isArray(oldText) && newText !== undefined)) throw new Error(usage);
+
+    if (args.patch !== undefined) {
+      if (!isString(args.patch) || !args.patch.trim()) throw new Error(usage);
+    } else {
+      const edits = args.edits === undefined ? [args] : args.edits;
+
+      if (!Array.isArray(edits) || !edits.length) throw new Error(usage);
+
+      for (const e of edits) if (!isString(e?.oldText) || !e.oldText.length || !isString(e?.newText)) throw new Error(usage + "; replacements require non-empty oldText and string newText");
+    }
+
     return unwrapValue(await invoke(args.patch === undefined ? "edit" : "apply_patch", args));
   };
+
   const patch = async (p, diff) => unwrapValue(await nova.call("apply_patch", { path: p, patch: diff }));
+
   const bash = async (command, opts) => {
     const args = isObject(command) ? { ...command } : { command, ...opts };
+
     if (args.args !== undefined) {
       if (!isString(args.command) || !Array.isArray(args.args) || args.args.some(arg => !isString(arg))) throw new Error("bash argv requires a command string and an array of string args");
+
       if (nativeArgv) args._directArgv = true;
       else {
         delete args._directArgv;
         args.command = [args.command, ...args.args].map(quoteShellArg).join(" ");
       }
     }
+
     command = args.command;
+
     if (args.timeout !== undefined && args.timeoutMs === undefined) args.timeoutMs = args.timeout * 1000;
     const res = await invoke("bash", args);
+
     if (res?.ok === false) {
       let exitCode;
+
       try {
         exitCode = JSON.parse(res.details).exitCode;
       } catch {}
+
       const output = String(res.value).trimEnd();
       const suffix = Number.isInteger(exitCode) ? " (exit " + exitCode + ")" : "";
       throw new Error("command failed" + suffix + ": " + command + (output ? "\n" + output : ""));
     }
+
     let text = unwrapValue(res);
+
     if (res?.truncated && isString(text) && !text.includes("truncated")) text += "\n…[output truncated]…";
+
     return text;
   };
+
   const quoteShellArg = (value) => "'" + String(value).replaceAll("'", "'\\''") + "'";
+
   const exec = async (cmd, args, opts) => {
     const command = String(cmd ?? "").trim();
+
     if (!command) throw new Error("exec requires command");
+
     // exec("git status") is a shell line; exec("git", ["status"]) is argv.
     if (!Array.isArray(args) || args.length === 0) return bash(command, opts);
+
     return bash([command, ...args].map(quoteShellArg).join(" "), opts);
   };
 
@@ -227,16 +326,23 @@ function buildGuestApi(available, batchRead, runId, nativeArgv) {
 function makeConsole(runId, limits) {
   let count = 0;
   let truncated = false;
+
   const markTruncated = () => {
     if (!truncated) post({ op: "logTruncated", runId });
     truncated = true;
   };
+
   const emit = (...args) => {
-    if (count >= limits.maxLogLines) { markTruncated(); return; }
+    if (count >= limits.maxLogLines) { markTruncated();
+
+ return; }
+
     count++;
+
     const line = args
       .map((a) => {
         if (isString(a)) return a;
+
         try {
           return JSON.stringify(toPlain(a));
         } catch {
@@ -244,10 +350,13 @@ function makeConsole(runId, limits) {
         }
       })
       .join(" ");
+
     const clipped = truncateChars(line, limits.maxLogLineChars, "log");
+
     if (clipped.truncated) markTruncated();
     post({ op: "log", runId, line: clipped.text, truncated: clipped.truncated });
   };
+
   return { log: emit, warn: emit, error: emit, info: emit, debug: emit };
 }
 
@@ -262,25 +371,34 @@ async function handleRun(msg) {
   activeRunId = runId;
   runActive = true;
   let compiled;
+
   try {
-    compiled = { fn: new AsyncFunction(...PARAMS, prepared.body), hasReturn: prepared.hasReturn };
+    // Existing programs may declare their own data variable; bind it only when supplied.
+    const bindings = msg.data === undefined ? PARAMS : [...PARAMS, "data"];
+    compiled = { fn: new AsyncFunction(...bindings, prepared.body), hasReturn: prepared.hasReturn };
   } catch (err) {
-    postFailure(runId, err);
+    postFailure(runId, new Error("JavaScript syntax error: " + err.message + "; no commands ran. When passing data, do not redeclare its binding."));
+
     return;
   }
+
   const api = buildGuestApi(available, batchRead, runId, msg.nativeArgv === true);
   const scopedConsole = makeConsole(runId, limits);
+
   try {
     const value = await compiled.fn(
-      scopedConsole, api.read, api.edit, api.write, api.bash,
+      scopedConsole, api.read, api.edit, api.write, api.bash, msg.data,
     );
+
     if (runId !== activeRunId) return;
     let plain;
+
     try {
       plain = toPlain(value);
     } catch (err) {
       plain = "[unserializable: " + (err?.message || err) + "]";
     }
+
     runActive = false;
     post({ op: "done", runId, value: plain, undefinedReturn: value === undefined && !compiled.hasReturn, hasReturn: compiled.hasReturn });
   } catch (err) {
@@ -291,14 +409,19 @@ async function handleRun(msg) {
 
 parentPort.on("message", (msg) => {
   if (!isObject(msg)) return;
+
   if (msg.op === "rpc:result") {
     const pending = pendingRpc.get(msg.id);
+
     if (!pending) return;
     pendingRpc.delete(msg.id);
+
     if (msg.ok) pending.resolve(msg.value);
     else pending.reject(new Error(msg.error));
+
     return;
   }
+
   if (msg.op === "run") void handleRun(msg);
 });