pi-supernova 0.8.2 → 0.9.0

package/src/output/bottleneck.js

@@ -1,8 +1,9 @@
+import {READ_VALUE} from "../shared/result.js";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { randomUUID } from "node:crypto";
-import { isString, isObject, mapChangedChildren, assertModelImageMime } from "../shared/decode.js";
-import { truncateChars, formatReturn, formatBoundedStringArray } from "./format.js";
+import { isString, isObject } from "../shared/decode.js";
+import { truncateChars } from "./format.js";
 
 function json(value) {
   try { return JSON.stringify(value) ?? "null"; } catch { return JSON.stringify(String(value)); }
@@ -221,8 +222,16 @@ function attachTruncation(result, truncated, batch, text, config, maxChars, capp
 }
 
 export function packageHostResult(raw, config) {
-  const maxChars = config.maxCallResultChars ?? 65536;
   const details = detailsOf(raw);
+  if (raw && Object.hasOwn(raw, READ_VALUE)) {
+    const batch = batchFromDetails(details);
+    const result = {ok: !hostResultFailed(raw), value: raw[READ_VALUE], typed: true, cloneItems: details?.jsonMany === true, truncated: details?.outputTruncated === true};
+    attachDetails(result, details, batch);
+    if (batch) { result.items = batch; result.itemErrors = details.itemErrors ?? []; }
+    if (details?.streamed) result.streamed = true;
+    return result;
+  }
+  const maxChars = config.maxCallResultChars ?? 65536;
   const batch = batchFromDetails(details);
   const text = batch ? "" : extractRawString(raw);
   const capped = truncateChars(text, maxChars, "host-result");
@@ -237,67 +246,4 @@ export function packageHostResult(raw, config) {
   return result;
 }
 
-function collectImage(input, acc) {
-  if (!(input?.type === "image" && isString(input.data) && isString(input.mimeType) && input.mimeType.startsWith("image/"))) return null;
-  assertModelImageMime(input.mimeType);
-  const size = Buffer.byteLength(input.data, "base64");
-
-  acc.imageCount += 1;
-  acc.imageBytes += size;
-  if (acc.imageCount > 16 || acc.imageBytes > 20 * 1024 * 1024) {
-    acc.imageOverflow = true;
-    return "[image over budget]";
-  }
-  acc.images.push({ type: "image", data: input.data, mimeType: input.mimeType });
-
-  return `[image ${acc.images.length}: ${input.mimeType}]`;
-}
-
-function collectImages(input, acc) {
-  const replaced = collectImage(input, acc);
-
-  if (replaced !== null) return replaced;
-
-  return mapChangedChildren(input, collectImages, acc);
-}
-
-function serializeReturn(value, formatted, maxReturn, imageOverflow) {
-  if (formatted.length <= maxReturn) return { text: formatted, truncated: imageOverflow };
-
-  if (Array.isArray(value) && value.length && value.every(isString)) return { text: formatBoundedStringArray(value, maxReturn), truncated: true };
-
-  return { ...truncateChars(formatted, maxReturn, "return"), truncated: true };
-}
-
-function clipLogLine(line, maxLogLineChars) {
-  const result = truncateChars(line, maxLogLineChars, "log");
-
-  return { text: result.text, truncated: result.truncated };
-}
-
-function clipLogs(logs, config) {
-  const maxLines = config.maxLogLines ?? 100;
-  let logTruncated = logs.length > maxLines;
-  const clipped = logs.slice(0, maxLines).map(line => {
-    const result = clipLogLine(line, config.maxLogLineChars ?? 4096);
-    logTruncated ||= result.truncated;
-
-    return result.text;
-  });
-
-  return { logs: clipped, logTruncated };
-}
-
-export function packageFinalReturn(value, logs, config) {
-  const acc = { images: [], imageCount: 0, imageBytes: 0, imageOverflow: false };
-  value = collectImages(value, acc);
-  if (acc.imageOverflow) {
-    throw new Error(`image attachment budget exceeded: ${acc.imageCount} images / ${acc.imageBytes} bytes; limit is 16 images / 20971520 bytes (20 MiB). No images returned; return fewer or smaller images per program`);
-  }
-  const maxReturn = config.maxReturnChars ?? 32000;
-  const serialized = serializeReturn(value, formatReturn(value), maxReturn, acc.imageOverflow);
-  const clipped = clipLogs(logs, config);
-
-  return { returnValue: serialized.truncated ? serialized.text : value, returnText: serialized.text,
-    returnTruncated: serialized.truncated, logs: clipped.logs, logTruncated: clipped.logTruncated, images: acc.images };
-}
+export {packageFinalReturn} from "./final.js";

package/src/output/final.js

@@ -0,0 +1,114 @@
+import {isString,mapChangedChildren,assertModelImageMime,decodeImageData} from "../shared/decode.js";
+import {assertPng} from "../shared/png.js";
+import {truncateChars,formatReturn,formatBoundedStringArray,formatBoundedValue,isStringArray} from "./format.js";
+
+function collectImage(input, acc) {
+  if (!(input?.type === "image" && isString(input.data) && isString(input.mimeType) && input.mimeType.startsWith("image/"))) return null;
+  assertModelImageMime(input.mimeType);
+  const size = Buffer.byteLength(input.data, "base64");
+
+  acc.imageCount += 1;
+  acc.imageBytes += size;
+  if (acc.imageCount > 16 || acc.imageBytes > 20 * 1024 * 1024) {
+    acc.imageOverflow = true;
+    return "[image over budget]";
+  }
+  const bytes = decodeImageData(input.data);
+  if (input.mimeType === "image/png") assertPng(bytes);
+  acc.images.push({ type: "image", data: input.data, mimeType: input.mimeType });
+
+  return `[image ${acc.images.length}: ${input.mimeType}]`;
+}
+
+function displayWeight(value,key) {
+  // Count even omitted undefined fields: typed result metadata crosses RPC too.
+  const field = isString(key) ? key.length+1 : 0;
+  return field + (isString(value) ? value.length : 1) + (Array.isArray(value) ? value.length : 0);
+}
+
+function collectImages(input, acc, key) {
+  const replaced = collectImage(input, acc);
+  acc.displayUnits += displayWeight(replaced ?? input,key);
+  if (replaced !== null) return replaced;
+
+  return mapChangedChildren(input, collectImages, acc);
+}
+
+function serializeReturn(value, formatted, maxReturn, imageOverflow) {
+  if (formatted.length <= maxReturn) return { text: formatted, truncated: imageOverflow };
+
+  if (isStringArray(value)) return { text: formatBoundedStringArray(value, maxReturn), truncated: true };
+
+  return { ...truncateChars(formatted, maxReturn, "return"), truncated: true };
+}
+
+function clipLogLine(line, maxLogLineChars) {
+  const result = truncateChars(line, maxLogLineChars, "log");
+
+  return { text: result.text, truncated: result.truncated };
+}
+
+function clipLogs(logs, config) {
+  const maxLines = config.maxLogLines ?? 100;
+  let logTruncated = logs.length > maxLines;
+  const clipped = logs.slice(0, maxLines).map(line => {
+    const result = clipLogLine(line, config.maxLogLineChars ?? 4096);
+    logTruncated ||= result.truncated;
+
+    return result.text;
+  });
+
+  return { logs: clipped, logTruncated };
+}
+
+function isSourceView(value) {
+  return value?.status === "found" && isString(value.path) && isString(value.text) && Array.isArray(value.lines);
+}
+
+function sourcePreview(value, maxReturn) {
+  if (!isSourceView(value)) return null;
+  if (value.text.length < maxReturn / 2) return null;
+  if (value.text.length <= maxReturn && formatReturn(value).length <= maxReturn-256) return null;
+  const base = {...value,text:"",complete:false,nextOffset:value.lines[0]+1};
+  let room = maxReturn - formatReturn(base).length - 512;
+  while (room > 0) {
+    const end = value.text.lastIndexOf("\n",room-1) + 1;
+    if (!end || end >= value.text.length) return null;
+    const text = value.text.slice(0,end);
+    const lines = [value.lines[0],value.lines[0]+text.match(/\n/g).length-1];
+    const preview = {...base,text,lines,nextOffset:lines[1]+1};
+    if (formatReturn(preview).length <= maxReturn-256) return preview;
+    room = Math.floor(room / 2);
+  }
+  return null;
+}
+
+function presentValue(value, maxReturn, oversized) {
+  if (isString(value)) {
+    const bounded = truncateChars(value,maxReturn,"return");
+    const result = serializeReturn(value,formatReturn(bounded.text),maxReturn,false);
+    result.truncated ||= bounded.truncated;
+    return result;
+  }
+  if (isStringArray(value) && oversized) {
+    return {text:formatBoundedStringArray(value,maxReturn),truncated:true};
+  }
+  if (oversized) return {text:formatBoundedValue(value,maxReturn),truncated:true};
+  return serializeReturn(value,formatReturn(value),maxReturn,false);
+}
+
+export function packageFinalReturn(value, logs, config) {
+  const acc = { images: [], imageCount: 0, imageBytes: 0, imageOverflow: false, displayUnits:0 };
+  value = collectImages(value, acc);
+  if (acc.imageOverflow) {
+    throw new Error(`image attachment budget exceeded: ${acc.imageCount} images / ${acc.imageBytes} bytes; limit is 16 images / 20971520 bytes (20 MiB). No images returned; return fewer or smaller images per program`);
+  }
+  const maxReturn = config.maxReturnChars ?? 32000;
+  const preview = sourcePreview(value,maxReturn);
+  if (preview) value = preview;
+  const serialized = presentValue(value,maxReturn,!preview && acc.displayUnits>maxReturn);
+  const clipped = clipLogs(logs, config);
+
+  return { returnValue: serialized.truncated ? serialized.text : value, returnText: serialized.text,
+    returnTruncated: serialized.truncated || Boolean(preview), logs: clipped.logs, logTruncated: clipped.logTruncated, images: acc.images };
+}

package/src/output/format.js

@@ -1,4 +1,5 @@
 import { isString, isObject, mapChangedChildren } from "../shared/decode.js";
+import {maxJsonStringPrefix} from "../fs/json-size.js";
 
 function normalizeText(text) {
   return isString(text) ? text : String(text ?? "");
@@ -76,24 +77,40 @@ function hasUnpairedSurrogate(text) {
   return SURROGATE_CODE_UNIT.test(text) && UNPAIRED_SURROGATE.test(text);
 }
 
+export function isStringArray(value) {
+  if (!Array.isArray(value) || !value.length) return false;
+  for (const item of value) if (!isString(item)) return false;
+  return true;
+}
+
 function hasWellFormedStrings(values) {
   for (const value of values) if (!isString(value) || hasUnpairedSurrogate(value)) return false;
 
   return true;
 }
 
-/** Keep every array item in an oversized return by giving each a fair truncated share. */
+function boundedStringText(value, limit) {
+  const text = truncateChars(value,limit,"return").text;
+  return hasUnpairedSurrogate(text) ? truncateChars(JSON.stringify(text),limit,"return").text : text;
+}
+
+/** Give displayed array items a fair share; disclose any omitted items. */
 export function formatBoundedStringArray(values, budget) {
   const n = values.length;
   const header = "strings[" + n + "]\n";
   let remaining = Math.max(0, budget - header.length);
   let out = header;
+  if (header.length >= budget) return truncateChars(header,budget,"return").text;
+  const omitted = count => "…[return truncated; " + count + " string items omitted]…\n";
 
   for (let i = 0; i < n; i++) {
     const itemHeader = "[" + i + "] " + values[i].length + " UTF-16 units\n";
-    const per = Math.max(32, Math.floor(remaining / (n - i)) - itemHeader.length - 1);
-    const bounded = truncateChars(values[i], per, "return");
-    const chunk = itemHeader + bounded.text + "\n";
+    const footer = i + 1 < n ? omitted(n - i - 1) : "";
+    if (remaining < itemHeader.length + Math.min(32,values[i].length) + 1 + footer.length) {
+      return out + truncateChars(omitted(n - i),remaining,"return").text;
+    }
+    const per = Math.min(remaining - itemHeader.length - 1 - footer.length, Math.max(32, Math.floor(remaining / (n - i)) - itemHeader.length - 1));
+    const chunk = itemHeader + boundedStringText(values[i],per) + "\n";
     remaining = Math.max(0, remaining - chunk.length);
     out += chunk;
   }
@@ -149,7 +166,7 @@ function framedReturn(value) {
 
 /** Lossless framing for source arrays, not string escaping or source compression. */
 export function formatReturn(value) {
-  if (isString(value)) return value;
+  if (isString(value)) return hasUnpairedSurrogate(value) ? JSON.stringify(value) : value;
   const rawArray = formatRawStringArray(value);
 
   if (rawArray !== null) return rawArray;
@@ -287,3 +304,75 @@ export function formatValue(value, indent = "", width = FORMAT_WIDTH, seen = new
     seen.delete(value);
   }
 }
+
+
+function displayKeys(value) {
+  return Array.isArray(value) ? value.keys() : Object.keys(value);
+}
+
+/** A cheap lower bound stops before expanding large/shared result subtrees. */
+export function displayExceeds(value, budget) {
+  const seen = new Set();
+  const spend = units => (budget -= units) < 0;
+  function container(item) {
+    if (seen.has(item)) return false;
+    seen.add(item);
+    const array = Array.isArray(item);
+    if (array && spend(item.length)) return true;
+    for (const key of displayKeys(item)) {
+      if (!array && spend(key.length+1)) return true;
+      if (visit(item[key])) return true;
+    }
+    seen.delete(item);
+    return false;
+  }
+  function visit(item) {
+    if (spend(isString(item) ? item.length : 1)) return true;
+    return isObject(item) || Array.isArray(item) ? container(item) : false;
+  }
+  return visit(value);
+}
+
+/** Display-only prefix; never constructs an oversized JSON/string rendering. */
+export function formatBoundedValue(value, budget, label = "return") {
+  const footer = "\n…["+label+" truncated]…";
+  const parts = [];
+  const seen = new Set();
+  let remaining = Math.max(0,budget-footer.length), stopped = false;
+  const push = text => {
+    if (text.length > remaining) { stopped = true; return false; }
+    remaining -= text.length; parts.push(text); return true;
+  };
+  function quoted(text) {
+    const end = maxJsonStringPrefix(text,remaining);
+    push(JSON.stringify(text.slice(0,end)));
+    if (end < text.length) stopped = true;
+  }
+  function visitChild(item,key,array) {
+    if (!array) { quoted(key); push(":"); }
+    visit(array && item[key] === undefined ? null : item[key]);
+  }
+  function container(item) {
+    if (seen.has(item)) { push('"[Circular]"'); return; }
+    seen.add(item);
+    const array = Array.isArray(item);
+    push(array ? "[" : "{");
+    let first = true;
+    for (const key of displayKeys(item)) {
+      if (stopped) break;
+      if (!first) push(",");
+      first = false;
+      visitChild(item,key,array);
+    }
+    if (!stopped) push(array ? "]" : "}");
+    seen.delete(item);
+  }
+  function visit(item) {
+    if (stopped) return;
+    if (isString(item)) quoted(item);
+    else if (isObject(item) || Array.isArray(item)) container(item);
+    else push(formatPrimitive(item));
+  }
+  visit(value);
+  return truncateChars(parts.join("")+footer,budget,"return").text;
+}

package/src/output/outcome.js

@@ -0,0 +1,91 @@
+import {isString} from '../shared/decode.js';
+import {truncateChars,formatBoundedStringArray,isStringArray} from './format.js';
+
+function result(text, details) {
+  return { content: [{ type: "text", text }], details };
+}
+
+function logsBlock(outcome, tail = "") {
+  if (!outcome.logs?.length && !outcome.logTruncated) return "";
+
+  return `\n--- logs${outcome.logTruncated ? " [logs truncated]" : ""}\n${outcome.logs?.join("\n") ?? ""}${tail}`;
+}
+
+function mutationText(outcome) {
+  const m = outcome.mutations;
+
+  if (!m) return "";
+  const external = m.external ? "; external calls attempted=" + m.external + ", their side effects cannot be rolled back" : "";
+  const uncertain = m.pendingCommits || m.recoveryFailed ? "; filesystem outcome uncertain: inspect disk and any recovery backups before retrying" : "";
+
+  return "\nmutations: committed=" + m.committed + " rolledBack=" + m.rolledBack + " (file versions)" + external + uncertain;
+}
+
+function mutationReceipts(trace) {
+  if (!Array.isArray(trace)) return "";
+
+  return trace
+    .filter(row => row?.ok && (row.name === "write" || row.name === "edit") && isString(row.resultText) && row.resultText)
+    .map(row => row.resultText)
+    .join("\n");
+}
+
+// Corrective hint, emitted only when a turn actually split. Independent work
+// belongs in one program: a split cannot use the single prewarmed worker and pays
+// one extra spawn per sibling. Costs nothing until it fires, so it needs no room in
+// the tool definition.
+function splitTurnHint(outcome) {
+  return outcome.overlappedTurn ? ` (${outcome.overlappedTurn} supernova calls ran at once; independent work belongs in one program)` : "";
+}
+
+function errorText(outcome, call) {
+  return `error #${call} ${outcome.wallMs}ms${outcome.returnTruncated ? " [output truncated]" : ""}${mutationText(outcome)}${splitTurnHint(outcome)}
+error: ${outcome.error}${logsBlock(outcome)}`;
+}
+
+function successText(outcome, call) {
+  const truncated = outcome.returnTruncated ? " [return truncated]" : "";
+  const hint = outcome.undefinedReturn ? " (no return statement; add `return` to get a value)" : "";
+  const m = outcome.mutations;
+  const showMutations = m && (m.committed || m.rolledBack || m.external || m.pendingCommits || m.recoveryFailed) ? mutationText(outcome) : "";
+
+  return `ok #${call} ${outcome.wallMs}ms${truncated}${showMutations}${splitTurnHint(outcome)}${logsBlock(outcome, "\n--- result")}\n${outcome.resultText}${hint}`;
+}
+
+function fitOutput(outcome, call, limit, format) {
+  let text = format(outcome, call);
+
+  if (text.length <= limit) return text;
+  outcome.returnTruncated = true;
+  const wrapper = format({ ...outcome, resultText: "", logs: [] }, call);
+  const room = Math.max(256, limit - wrapper.length);
+
+  if (isStringArray(outcome.result)) {
+    outcome.resultText = formatBoundedStringArray(outcome.result, room);
+  } else if (isString(outcome.resultText) && outcome.resultText.length > room) {
+    outcome.resultText = truncateChars(outcome.resultText, room, "output").text;
+  }
+
+  text = format(outcome, call);
+
+  return text.length <= limit ? text : truncateChars(text, limit, "output").text;
+}
+
+function attachReceipts(outcome, trace) {
+  if (outcome.ok && outcome.result === undefined) {
+    const receipts = mutationReceipts(trace);
+
+    if (receipts) {
+      outcome.resultText = receipts;
+      outcome.undefinedReturn = false;
+    }
+  }
+}
+
+function throwIfFailed(outcome, visible, response) {
+  if (outcome.ok) return response;
+  const error = new Error(visible);
+  Object.defineProperty(error,"supernovaResult",{value:response});
+  throw error;
+}
+export { result, errorText, successText, fitOutput, attachReceipts, throwIfFailed };

package/src/runtime/batch-input.js

@@ -0,0 +1,68 @@
+import {isObject,isString} from '../shared/decode.js';
+
+function assertProgramEntry(p, defaults = {}) {
+  const source = p?.code === undefined && p?.file === undefined ? defaults : p;
+  if (!objectData(p) || Object.keys(p).some(key => !["code", "file", "data"].includes(key)) || !validProgramSource(source)) {
+    throw new Error("each program requires code OR file (own or shared), with optional data; no nested batches or per-entry timeouts; no programs ran");
+  }
+}
+
+const objectData = value => isObject(value) && !Array.isArray(value);
+
+function applyBatchDefaults(parsed, mergeData) {
+  if (mergeData && !objectData(parsed.data)) throw new Error("mergeData requires top-level object data; no programs ran");
+  const source = parsed.code !== undefined ? {code:parsed.code} : parsed.file !== undefined ? {file:parsed.file} : {};
+
+  return parsed.programs.map(program => {
+    assertProgramEntry(program, source);
+    const entry = program.code === undefined && program.file === undefined ? {...source,...program} : program;
+
+    if (mergeData) {
+      if (program.data !== undefined && !objectData(program.data)) throw new Error("mergeData requires object data in every explicit entry; no programs ran");
+      // Shallow own-property overlay, including literal __proto__ keys. The
+      // runtime snapshots data again per guest; no mutable heap is shared.
+      entry.data = {...parsed.data,...program.data};
+    } else if (program.data === undefined && Object.hasOwn(parsed,"data")) entry.data = parsed.data;
+
+    return entry;
+  });
+}
+
+function parseBatchPayload(params, config) {
+  assertBatchOptions(params);
+  const defaults = Object.fromEntries(["code", "file", "data"].filter(key => params[key] !== undefined).map(key => [key, params[key]]));
+  if (defaults.code !== undefined || defaults.file !== undefined) assertProgramEntry(defaults);
+  for (const p of params.programs) assertProgramEntry(p, defaults);
+  // Validate before serialization and again after snapshotting: toJSON may change an entry.
+  return applyBatchDefaults(snapshotBatch(params, defaults, config), params.mergeData === true);
+}
+
+function batchTimeoutMs(params, config) {
+  const requestedTimeout = params.timeoutMs === undefined ? config.timeoutMs : Number(params.timeoutMs);
+
+  if (!Number.isFinite(requestedTimeout) || requestedTimeout <= 0) throw new Error("program batch timeoutMs must be a positive finite number");
+
+  return requestedTimeout;
+}
+
+function validProgramSource(source) {
+  const value = source.code ?? source.file;
+  return (source.code === undefined) !== (source.file === undefined) && isString(value) && value.trim();
+}
+
+function assertBatchOptions(params) {
+  if (!Array.isArray(params.programs) || !params.programs.length || params.programs.length > 32) throw new Error("programs requires 1..32 entries; no programs ran");
+  if (params.mergeData !== undefined && ![true, false].includes(params.mergeData)) throw new Error("mergeData must be boolean; no programs ran");
+}
+
+function snapshotBatch(params, defaults, config) {
+  const hasDefaults = Object.keys(defaults).length > 0;
+  let encoded;
+  try { encoded = JSON.stringify(hasDefaults ? {programs:params.programs,...defaults} : params.programs); }
+  catch { throw new Error("programs and defaults must be JSON-serializable; no programs ran"); }
+  if (encoded.length > (config.maxCodeChars ?? 48000)) throw new Error("programs JSON exceeds the code character budget (including shared code/file/data); no programs ran");
+  const parsed = hasDefaults ? JSON.parse(encoded) : {programs:JSON.parse(encoded)};
+  if (Object.hasOwn(defaults,"data") && !Object.hasOwn(parsed,"data")) throw new Error("data must be JSON-serializable; no programs ran");
+  return parsed;
+}
+export { parseBatchPayload, batchTimeoutMs };