pi-supernova 0.3.2 → 0.4.0

package/index.js

@@ -1,4 +1,6 @@
 import { createRequire } from "node:module";
+import { runProgramBatch } from "./src/runtime/program-batch.js";
+import { REFERENCE } from "./src/runtime/reference.js";
 import { isString, isFunction } from "./src/shared/decode.js";
 import { loadConfig } from "./src/config/config.js";
 import { createHostBridge } from "./src/bridge/host-bridge.js";
@@ -10,13 +12,17 @@ export { renderSupernovaCall, renderSupernovaResult };
 
 // Sync only, never top-level await. Dynamic import of host/deps hung OMP plugin load.
 const require = createRequire(import.meta.url);
+
 let Type;
+
 try {
   Type = require("typebox").Type;
 } catch {
   Type = {
     Object: (props, opts) => ({ type: "object", properties: props || {}, additionalProperties: false, ...opts }),
     String: (opts) => ({ type: "string", ...opts }),
+    Unknown: (opts) => ({ ...opts }),
+    Array: (items, opts) => ({ type: "array", items, ...opts }),
     Integer: (opts) => ({ type: "integer", ...opts }),
     Optional: (s) => ({ ...s }),
   };
@@ -38,30 +44,38 @@ export function progressEmitter(onUpdate) {
   let pending = null;
   let timer = null;
   let lastSent = -Infinity;
+
   const send = () => {
     timer = null;
+
     if (pending === null) return;
     // Snapshot only at emission, not on every tool event. Completed records must
     // not mutate a previously emitted frame while Pi is still consuming it.
     const trace = pending.map(record => ({ ...record }));
     pending = null;
     lastSent = performance.now();
+
     try {
       onUpdate({ content: [{ type: "text", text: "" }], details: { trace, running: true } });
     } catch {}
   };
+
   const emit = (trace) => {
     pending = trace;
+
     if (timer !== null) return;
     const wait = PROGRESS_FRAME_MS - (performance.now() - lastSent);
+
     if (wait <= 0) send();
     else timer = setTimeout(send, wait);
   };
+
   emit.flush = () => {
     if (timer !== null) clearTimeout(timer);
     pending = null;
     timer = null;
   };
+
   return emit;
 }
 
@@ -70,36 +84,42 @@ function sessionStats({ programs, returnedChars }) {
 }
 
 function logsBlock(outcome, tail = "") {
-  return outcome.logs?.length ? `\n--- logs\n${outcome.logs.join("\n")}${tail}` : "";
+  if (!outcome.logs?.length && !outcome.logTruncated) return "";
+
+  return `\n--- logs${outcome.logTruncated ? " [logs truncated]" : ""}\n${outcome.logs?.join("\n") ?? ""}${tail}`;
+}
+
+function mutationText(outcome) {
+  const m = outcome.mutations;
+
+  if (!m) return "";
+  const external = m.external ? "; external calls attempted=" + m.external + ", their side effects cannot be rolled back" : "";
+  const uncertain = m.pendingCommits || m.recoveryFailed ? "; filesystem outcome uncertain: inspect disk and any recovery backups before retrying" : "";
+
+  return "\nmutations: committed=" + m.committed + " rolledBack=" + m.rolledBack + " (file versions)" + external + uncertain;
+}
+
+// Corrective hint, emitted only when a turn actually split. Independent work
+// belongs in one program: a split cannot use the single prewarmed worker and pays
+// one extra spawn per sibling. Costs nothing until it fires, so it needs no room in
+// the tool definition.
+function splitTurnHint(outcome) {
+  return outcome.overlappedTurn ? ` (${outcome.overlappedTurn} supernova calls ran at once; independent work belongs in one program)` : "";
 }
 
 function errorText(outcome, call) {
-  return `error #${call} ${outcome.wallMs}ms: ${outcome.error}${logsBlock(outcome)}`;
+  return `error #${call} ${outcome.wallMs}ms${outcome.returnTruncated ? " [output truncated]" : ""}${mutationText(outcome)}${splitTurnHint(outcome)}
+error: ${outcome.error}${logsBlock(outcome)}`;
 }
 
 function successText(outcome, call) {
   const truncated = outcome.returnTruncated ? " [return truncated]" : "";
   const hint = outcome.undefinedReturn ? " (no return statement; add `return` to get a value)" : "";
-  return `ok #${call} ${outcome.wallMs}ms${truncated}${logsBlock(outcome, "\n--- result")}\n${outcome.resultText}${hint}`;
+
+  return `ok #${call} ${outcome.wallMs}ms${truncated}${outcome.mutations?.committed || outcome.mutations?.rolledBack || outcome.mutations?.external ? mutationText(outcome) : ""}${splitTurnHint(outcome)}${logsBlock(outcome, "\n--- result")}\n${outcome.resultText}${hint}`;
 }
-const TOOL_DESCRIPTION = `Run one JavaScript program with four familiar commands: read, write, edit, bash. Use an async body or arrow. Return a small value; strings stay raw.
-
-Native commands (async):
-read(path|paths, offset?, limit?) → file text or text[]; read(directory) → directory entries
-read("symbol or question") → locate and open source in one call, without an index; selected file text stays raw
-read({query, resolve:true}) → {status,path,line,lines,text,complete,nextOffset?} for a direct resolve→edit handoff
-read(path, {about: question}) → relevant file bodies, or source selection inside a directory
-read({query, evidence:true}) → ranked evidence; read({path, outline:true}) → structural declarations
-write(path, text) → write a file; write({path,content,append:true}) appends a chunk without a bounded read
-edit(path, oldText, newText) → post-edit lines, checks, and references
-edit(async () => {...}) → filesystem checkpoint: commit on success, rollback on throw; no shell commands, nesting, or concurrent outside commands
-bash(command, {cwd?, timeoutMs?}) → bounded output; throws on non-zero exit
-bash({command, args:[...]}) → literal argv without shell expansion of arguments
-
-Only found selects and opens a file. Uncertain reads return ambiguous, not_found, or incomplete with no selected path. Use resolve:true for structured status checks; narrow the directory with path+about when uncertain.
-For read-modify-write, use read({path,complete:true}); it rejects partial output. Prefer edit for large files. Array reads reject failures; use Promise.allSettled for per-path outcomes.
-Object arguments also work: read({path, offset?, limit?, about?, outline?, evidence?, resolve?, complete?}), edit({path, edits:[{oldText,newText}]}), edit({path,patch}), write({path,content}), bash({command,timeoutMs?}).
-Independent read starts batch automatically. Mutations preserve submission order. Plain reads remain self-contained; oversized reads provide continuation offsets. Return only what the model needs. console.log is captured.`;
+
+const TOOL_DESCRIPTION = REFERENCE;
 
 export default function piSupernova(pi) {
   registerCodeMode(pi);
@@ -107,12 +127,20 @@ export default function piSupernova(pi) {
 
 // Shared entry used by both host adapters and direct engine integration.
 export function registerCodeMode(pi) {
-  // Local cache residency cannot establish what remains in the model's context.
-  const config = { ...loadConfig(), seenWindow: 0 };
+  // Citation elision is experimental and disabled by default. A context event is
+  // not the final provider payload: hidden details or later transforms can invalidate
+  // a citation. A positive seenWindow explicitly opts in despite those limitations.
+  const config = loadConfig();
   let cwd = process.cwd();
   let programSeq = 0;
   let stopped = false;
   let warmTimer;
+  // Program runs currently executing. Only one pristine worker is ever prewarmed,
+  // so concurrent invocations cannot share it and each sibling pays a fresh spawn.
+  // Counting them lets a result say so without adding standing guidance to the
+  // tool definition, which is resent on every request.
+  let inFlight = 0;
+
   function cancelWarmTimer() {
     if (warmTimer !== undefined) clearImmediate(warmTimer);
     warmTimer = undefined;
@@ -148,28 +176,34 @@ export function registerCodeMode(pi) {
     name: "supernova",
     label: "Supernova",
     description: TOOL_DESCRIPTION,
-    promptSnippet: "Use read, write, edit, and bash in one program",
-    promptGuidelines: [
-      "Use read, write, edit, and bash inside supernova. Start with read(question), or read(directory, {about: question}) for scoped source selection. A source question already opens the selected file; do not issue a redundant read. Use read({query,resolve:true}) and check status before editing its path. Explicit about/outline/evidence reads remain available when needed. Return a compact value.",
-    ],
+    promptSnippet: "JavaScript with read, write, edit, and bash",
     parameters: Type.Object({
-      code: Type.String({ maxLength: config.maxCodeChars ?? 48000, description: `JavaScript program: async body or arrow function. Maximum ${config.maxCodeChars ?? 48000} UTF-16 code units; split large writes into write({path,content,append:true}) chunks.` }),
+      code: Type.Optional(Type.String({ maxLength: config.maxCodeChars ?? 48000 })),
+      file: Type.Optional(Type.String({ minLength: 1 })),
+      data: Type.Optional(Type.Unknown({ description: "Literal JSON input available as data in the program; put Markdown, scripts or argv here instead of nesting JavaScript quoting. JSON-encoded size is limited to the code character budget." })),
       timeoutMs: Type.Optional(Type.Integer({ minimum: 1000, description: "Hard timeout in ms." })),
-    }, { required: ["code"] }),
+      programs: Type.Optional(Type.Array(Type.Object({
+        code: Type.Optional(Type.String({ maxLength: config.maxCodeChars ?? 48000 })),
+        file: Type.Optional(Type.String({ minLength: 1 })),
+        data: Type.Optional(Type.Unknown()),
+      }, {additionalProperties:false}), {minItems:1,maxItems:32,description:"Instead of top-level code/file/data. JSON-encoded array shares the code character cap."})),
+    }),
     // One self-owned result frame is shared by Pi and OMP; renderCall stays empty
     // so separate call/result slots cannot duplicate the lifecycle card.
     renderShell: "self",
     mergeCallAndResult: true,
     renderCall: renderSupernovaCall,
     renderResult: renderSupernovaResult,
-    async execute(_id, params, signal, onUpdate, ctx) {
+    execute: async function execute(_id, params, signal, onUpdate, ctx, budget) {
+      if (params?.programs !== undefined) return runProgramBatch(_id,params,signal,onUpdate,ctx,config,execute);
       cancelWarmTimer();
       const runCwd = ctx?.cwd || cwd;
       const runController = new AbortController();
       const abortRun = () => runController.abort(signal?.reason);
+
       if (signal?.aborted) abortRun();
       else signal?.addEventListener("abort", abortRun, { once: true });
-      const runBridge = bridge.fork({ getCwd: () => runCwd });
+      const runBridge = bridge.fork({ getCwd: () => runCwd, budget });
       runBridge.bindCallContext(ctx, runController.signal);
       runBridge.resetCallBudget();
 
@@ -180,61 +214,103 @@ export function registerCodeMode(pi) {
       emitProgress([]);
       const started = performance.now();
       let outcome;
+      const overlappedTurn = inFlight > 0 ? inFlight + 1 : 0;
+      inFlight += 1;
+
       try {
         refreshCatalog(runBridge);
         runBridge.beginSpeculation();
         outcome = await runGuestProgram({
           code: params?.code,
+          file: params?.file,
+          cwd: runCwd,
+          data: params?.data,
           nova: makeNovaApi(runBridge, abortRun),
-          config: { ...config, timeoutMs: Number.isInteger(params?.timeoutMs) ? params.timeoutMs : config.timeoutMs },
+          config: { ...config, maxLogLines: Math.max(0,config.maxLogLines-(budget?.logLines ?? 0)), timeoutMs: Number.isInteger(params?.timeoutMs) ? params.timeoutMs : config.timeoutMs },
           signal: runController.signal,
           onTimeout: abortRun,
         });
         runBridge.close();
+
         if (outcome.ok) {
           if (runBridge.getOverlayDepth() !== 1) throw new Error("program ended with an unfinished edit checkpoint; await it before returning");
           await runBridge.commitSpeculation();
         }
-        else runBridge.rollbackSpeculation();
+        else while (runBridge.getOverlayDepth()) runBridge.rollbackSpeculation();
       } catch (error) {
         abortRun();
         runBridge.close();
-        runBridge.rollbackSpeculation();
+
+        while (runBridge.getOverlayDepth()) runBridge.rollbackSpeculation();
         outcome = { ok: false, error: error instanceof Error ? error.message : String(error), logs: outcome?.logs ?? [], wallMs: Math.round(performance.now() - started) };
       } finally {
+        inFlight -= 1;
         runBridge.setCallListener(null);
         emitProgress.flush();
         signal?.removeEventListener("abort", abortRun);
         // Prepare one pristine worker during the model's next decision. Never
         // recycle a worker that has executed arbitrary guest JavaScript.
         cancelWarmTimer();
+
         if (!stopped && !runController.signal.aborted) {
           // Deliver the result before paying for another Worker constructor.
           warmTimer = setImmediate(() => {
             warmTimer = undefined;
+
             if (!stopped && !runController.signal.aborted) warmGuestWorker(config).catch(() => {});
           });
           warmTimer.unref?.();
         }
       }
+
+      if (budget) budget.logLines += outcome.logs?.length ?? 0;
+      // inFlight has dropped by now, so a non-zero value means a sibling is still
+      // running: report the overlap from either side so the hint does not depend on
+      // which invocation happened to start first.
+      outcome.overlappedTurn = overlappedTurn || (inFlight > 0 ? inFlight + 1 : 0);
+      outcome.mutations = runBridge.getMutations();
       const trace = runBridge.getTrace();
-      const text = outcome.ok ? successText(outcome, call) : errorText(outcome, call);
+      const format = outcome.ok ? successText : errorText;
+      let text = format(outcome, call);
+
+      if (text.length > config.maxReturnChars) {
+        outcome.returnTruncated = true;
+        text = format(outcome, call);
+      }
+
       const bounded = truncateChars(text, config.maxReturnChars, "output").text;
       const visible = runBridge.ledger.dedupe(bounded, call);
-      if (!outcome.ok) throw new Error(visible);
+
       const response = result(visible, {
         ok: outcome.ok, error: outcome.error, wallMs: outcome.wallMs,
         returnTruncated: outcome.returnTruncated, logTruncated: outcome.logTruncated,
-        logs: outcome.logs, result: outcome.result, trace,
+        logs: outcome.logs, result: outcome.result, trace, mutations: outcome.mutations,
       });
+
       if (outcome.images?.length) response.content.push(...outcome.images);
+
+      if (!outcome.ok) {
+        const error = new Error(visible);
+        Object.defineProperty(error,"supernovaResult",{value:response});
+        throw error;
+      }
+
       return response;
     },
   });
 
-  pi.on("session_shutdown", () => { stopped = true; cancelWarmTimer(); return stopWarmGuestWorker(); });
+  // This is a pre-conversion observation, not a final-payload retention proof.
+  // With the shipping seenWindow:0 default, observe is a no-op.
+  pi.on("context", event => {
+    try { bridge.ledger.observe(event?.messages); } catch {}
+  });
+
+  pi.on("session_shutdown", () => { stopped = true; cancelWarmTimer();
+
+ return stopWarmGuestWorker(); });
   pi.on("session_start", (_event, ctx) => {
     stopped = false;
+
     if (ctx && isString(ctx.cwd) && ctx.cwd) cwd = ctx.cwd;
     // A new session is a new model context: nothing has been seen yet.
     bridge.bindCallContext(ctx);
@@ -250,11 +326,13 @@ export function registerCodeMode(pi) {
       bridge.bindCallContext(ctx);
       refreshCatalog();
       const commands = ["read", "edit", "write", "bash"].filter(bridge.isCallable);
+
       const lines = [
         `Supernova CodeMode: ${commands.join(", ")}`,
         `timeoutMs=${config.timeoutMs} maxCallResultChars=${config.maxCallResultChars} maxReturnChars=${config.maxReturnChars} maxBridgeCalls=${config.maxBridgeCalls} maxHeapMb=${config.maxHeapMb}`,
         sessionStats(bridge.ledger.stats),
       ];
+
       ctx.ui.notify(lines.join("\n"), "info");
     },
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-supernova",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "One CodeMode invocation for Pi and OMP, with four guest commands, automatic read batching and source context.",
   "type": "module",
   "author": "AdityaVG13",
@@ -33,6 +33,7 @@
     "test": "node --test tests/*/*.test.mjs",
     "test:hosts": "node tests/hosts/verify.mjs",
     "measure": "node tests/efficiency/measure.mjs",
+    "test:tokens": "node tests/efficiency/tokens.mjs",
     "prepublishOnly": "npm test && node ../../scripts/preflight.mjs"
   },
   "pi": {
@@ -58,6 +59,7 @@
     }
   },
   "devDependencies": {
+    "js-tiktoken": "1.0.21",
     "typebox": "^1.0.0"
   },
   "publishConfig": {

package/src/bridge/catalog.js

@@ -4,7 +4,7 @@ import { isString, isObject } from "../shared/decode.js";
 const NATIVE_TOOL_DEFINITIONS = [
   {
     name: "read",
-    description: "Read files or directories. Source questions locate and open source directly; resolve returns structured source/status without guessing.",
+    description: "Read files, images or directories. JSON selectors project full documents within output budgets. Source questions locate and open source directly; resolve returns structured source/status without guessing.",
     parameters: { type: "object", properties: {
       path: { anyOf: [{ type: "string" }, { type: "array", items: { type: "string" } }], description: "Workspace-relative file or directory, source question, or array of paths" },
       target: { anyOf: [{ type: "string" }, { type: "array" }], description: "File path/query or array of paths" },
@@ -14,6 +14,7 @@ const NATIVE_TOOL_DEFINITIONS = [
       query: { type: "string", description: "Source question; optional path scopes the search directory" },
       resolve: { type: "boolean", description: "Return structured source/status for a direct resolve-to-edit handoff" },
       complete: { type: "boolean", description: "Fail unless the entire requested file fits without clipping" },
+      json: { anyOf: [{ type: "boolean" }, { type: "string" }, { type: "array", items: { type: "string" } }], description: "Parse the complete JSON input (up to 16 MiB), then select .field, .items[0:3], or quoted keys. A selector array returns an array of values; true selects the root. Oversized selections fail, never clip." },
     } },
   },
   {
@@ -85,28 +86,36 @@ export function mergeNativeToolDefinitions(tools, capturedNames = []) {
     merged.push(fallback && !captured.has(tool.name)
       ? { ...tool, ...fallback, sourceInfo: { path: "<native:" + tool.name + ">" } }
       : tool);
+
     if (tool?.name) seen.add(tool.name);
   }
+
   for (const fallback of NATIVE_TOOL_DEFINITIONS) {
     if (!seen.has(fallback.name)) {
       merged.push({ ...fallback, sourceInfo: { path: "<native:" + fallback.name + ">" } });
     }
   }
+
   return merged;
 }
 
 function sourcePathOf(tool) {
   if (tool.sourceInfo && isString(tool.sourceInfo.path)) return tool.sourceInfo.path;
+
   if (isString(tool.extensionPath)) return tool.extensionPath;
+
   if (isString(tool.sourcePath)) return tool.sourcePath;
+
   return undefined;
 }
 
 function normalizeTool(tool) {
   if (!tool || !isObject(tool)) return null;
   const name = isString(tool.name) ? tool.name : "";
+
   if (!name) return null;
   const description = isString(tool.description) ? tool.description : "";
+
   return {
     name,
     nameLower: name.toLowerCase(),
@@ -121,12 +130,16 @@ function normalizeTool(tool) {
 export function buildCatalog(tools, excludeNames = []) {
   const exclude = new Set(excludeNames);
   const rows = [];
+
   for (const tool of tools || []) {
     const row = normalizeTool(tool);
+
     if (!row || exclude.has(row.name)) continue;
     rows.push(row);
   }
+
   rows.sort((a, b) => a.name.localeCompare(b.name));
+
   return rows;
 }
 
@@ -142,74 +155,94 @@ function scoreRow(row, tokens) {
   const name = row.nameLower || row.name.toLowerCase();
   const desc = row.descLower || row.description.toLowerCase();
   let score = 0;
+
   for (const token of tokens) {
     if (name === token) score += 10;
     else if (name.includes(token)) score += 5;
     else if (desc.includes(token)) score += 2;
   }
+
   return score;
 }
 
 export function searchCatalog(catalog, query, limit = 12) {
   const tokens = tokenize(query);
   const scored = [];
+
   for (const row of catalog) {
     const score = scoreRow(row, tokens);
+
     if (score <= 0 && tokens.length > 0) continue;
     scored.push({ name: row.name, description: row.description.slice(0, 160), score });
   }
+
   scored.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
+
   return scored.slice(0, Math.max(1, limit)).map(({ score: _s, ...hit }) => hit);
 }
 
 /** Optimal string alignment distance: insert/delete/substitute/adjacent-transpose cost 1. */
 function editDistance(a, b) {
   const rows = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array.from({ length: b.length }, () => 0)]);
+
   for (let j = 1; j <= b.length; j++) rows[0][j] = j;
+
   for (let i = 1; i <= a.length; i++) {
     for (let j = 1; j <= b.length; j++) {
       const cost = a[i - 1] === b[j - 1] ? 0 : 1;
       let best = Math.min(rows[i - 1][j] + 1, rows[i][j - 1] + 1, rows[i - 1][j - 1] + cost);
+
       if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) best = Math.min(best, rows[i - 2][j - 2] + 1);
       rows[i][j] = best;
     }
   }
+
   return rows[a.length][b.length];
 }
 
 /** Closest tool names for a mistyped name: substring hits first, then a length-scaled edit distance. */
 function suggestNames(name, candidates, limit = 3) {
   const needle = String(name || "").toLowerCase();
+
   if (!needle) return [];
   const maxDistance = Math.max(1, Math.floor(needle.length / 3));
   const scored = [];
+
   for (const candidate of candidates) {
     const lower = candidate.toLowerCase();
+
     if (lower === needle) continue;
     const distance = lower.includes(needle) || needle.includes(lower) ? 1 : editDistance(needle, lower);
+
     if (distance <= maxDistance) scored.push({ candidate, distance });
   }
+
   scored.sort(
     (a, b) =>
       a.distance - b.distance ||
       Math.abs(a.candidate.length - needle.length) - Math.abs(b.candidate.length - needle.length) ||
       a.candidate.localeCompare(b.candidate),
   );
+
   return scored.slice(0, limit).map((s) => s.candidate);
 }
 
 export function unknownToolMessage(name, candidates) {
   const close = suggestNames(name, candidates);
   const hint = close.length ? ` Did you mean ${close.map((c) => JSON.stringify(c)).join(", ")}?` : "";
+
   return `unknown tool "${name}".${hint} Check the command name and configured tool exclusions.`;
 }
 
 export function describeTool(catalog, name) {
   const row = catalog.find((t) => t.name === name);
+
   if (!row) {
     return { ok: false, error: unknownToolMessage(name, catalog.map((t) => t.name)) };
   }
+
   if (!isObject(row.parameters)) return { ok: false, name, error: "tool schema unavailable: " + (row.schemaError ?? name) };
+
   if (!row._described) {
     row._described = {
       ok: true,
@@ -219,5 +252,6 @@ export function describeTool(catalog, name) {
       sourcePath: row.sourcePath,
     };
   }
+
   return row._described;
 }