pi-supernova 0.8.1 → 0.9.0

package/src/runtime/program-batch.js

@@ -1,4 +1,5 @@
-import { isObject, isString } from "../shared/decode.js";
+import {parseBatchPayload,batchTimeoutMs} from "./batch-input.js";
+import { isString } from "../shared/decode.js";
 import { truncateChars } from "../output/format.js";
 
 const textOf = result => (Array.isArray(result?.content) ? result.content : []).filter(block => block?.type === "text").map(block => block.text).join("\n");
@@ -28,66 +29,6 @@ export function programBatchText(results, total, stopped = "", failed = 0) {
     }).join("");
 }
 
-function assertProgramEntry(p, defaults = {}) {
-  const source = p?.code === undefined && p?.file === undefined ? defaults : p;
-
-  if (!isObject(p) || Array.isArray(p) || Object.keys(p).some(key => !["code","file","data"].includes(key)) ||
-      ((source.code === undefined) === (source.file === undefined)) || !isString(source.code ?? source.file) || !(source.code ?? source.file).trim()) {
-    throw new Error("each program requires code OR file (own or shared), with optional data; no nested batches or per-entry timeouts; no programs ran");
-  }
-}
-
-const objectData = value => isObject(value) && !Array.isArray(value);
-
-function applyBatchDefaults(parsed, mergeData) {
-  if (mergeData && !objectData(parsed.data)) throw new Error("mergeData requires top-level object data; no programs ran");
-  const source = parsed.code !== undefined ? {code:parsed.code} : parsed.file !== undefined ? {file:parsed.file} : {};
-
-  return parsed.programs.map(program => {
-    assertProgramEntry(program, source);
-    const entry = program.code === undefined && program.file === undefined ? {...source,...program} : program;
-
-    if (mergeData) {
-      if (program.data !== undefined && !objectData(program.data)) throw new Error("mergeData requires object data in every explicit entry; no programs ran");
-      // Shallow own-property overlay, including literal __proto__ keys. The
-      // runtime snapshots data again per guest; no mutable heap is shared.
-      entry.data = {...parsed.data,...program.data};
-    } else if (program.data === undefined && Object.hasOwn(parsed,"data")) entry.data = parsed.data;
-
-    return entry;
-  });
-}
-
-function parseBatchPayload(params, config) {
-  if (!Array.isArray(params.programs) || !params.programs.length || params.programs.length > 32) throw new Error("programs requires 1..32 entries; no programs ran");
-  if (params.mergeData !== undefined && params.mergeData !== true && params.mergeData !== false) throw new Error("mergeData must be boolean; no programs ran");
-  const defaults = Object.fromEntries(["code","file","data"].filter(key => params[key] !== undefined).map(key => [key,params[key]]));
-
-  if (defaults.code !== undefined || defaults.file !== undefined) assertProgramEntry(defaults);
-  for (const p of params.programs) assertProgramEntry(p, defaults);
-  const hasDefaults = Object.keys(defaults).length > 0;
-  let encoded;
-
-  try { encoded = JSON.stringify(hasDefaults ? {programs:params.programs,...defaults} : params.programs); } catch { throw new Error("programs and defaults must be JSON-serializable; no programs ran"); }
-
-  if (encoded.length > (config.maxCodeChars ?? 48000)) throw new Error("programs JSON exceeds the code character budget (including shared code/file/data); no programs ran");
-  const parsed = hasDefaults ? JSON.parse(encoded) : {programs:JSON.parse(encoded)};
-
-  if (Object.hasOwn(defaults,"data") && !Object.hasOwn(parsed,"data")) throw new Error("data must be JSON-serializable; no programs ran");
-
-  // Validate and expand every entry before executing any. Defaults count once
-  // against admission, not once for each independent guest receiving a copy.
-  return applyBatchDefaults(parsed, params.mergeData === true);
-}
-
-function batchTimeoutMs(params, config) {
-  const requestedTimeout = params.timeoutMs === undefined ? config.timeoutMs : Number(params.timeoutMs);
-
-  if (!Number.isFinite(requestedTimeout) || requestedTimeout <= 0) throw new Error("program batch timeoutMs must be a positive finite number");
-
-  return requestedTimeout;
-}
-
 const MAX_PARALLEL_PROGRAMS = 8;
 
 class ProgramBatch {
@@ -136,7 +77,7 @@ class ProgramBatch {
   }
 
   collectImages(result, i) {
-    for (const block of Array.isArray(result?.content) ? result.content : []) if (block?.type === "image" && isString(block.data)) {
+    for (const block of imageBlocks(result)) {
       this.imageBytes += Buffer.byteLength(block.data,"base64");
 
       if (this.images.length >= 16 || this.imageBytes > 20*1024*1024) { this.imageDropped = true; continue; }
@@ -156,16 +97,26 @@ class ProgramBatch {
     this.collectImages(result, i);
   }
 
+  parallelBudgetStop(settled) {
+    const results = settled.filter(Boolean);
+    const {images, bytes} = imageTotals(settled);
+    let kind;
+    if (images > 16 || bytes > 20 * 1024 * 1024) kind = "image";
+    else if (results.some(result => result.details?.logTruncated) || results.reduce((n, result) => n + (result.details?.logs?.length ?? 0), 0) > (this.config.maxLogLines ?? 100)) kind = "log";
+    return kind ? "batch " + kind + " budget exceeded; completed commits remain" : "";
+  }
+
   async runParallel() {
     const limit = Math.min(this.programs.length, MAX_PARALLEL_PROGRAMS);
     const settled = Array.from({ length: this.programs.length });
     let next = 0;
 
     await Promise.all(Array.from({length: limit}, async () => {
-      while (next < this.programs.length && !this.combined.aborted && performance.now() < this.deadline) {
+      while (next < this.programs.length && !this.stopped && !this.combined.aborted && performance.now() < this.deadline) {
         const i = next++;
         settled[i] = await this.runOne(this.programs[i], i);
         this.live[i] = [];
+        this.stopped ||= this.parallelBudgetStop(settled);
       }
     }));
 
@@ -174,7 +125,7 @@ class ProgramBatch {
       this.takeSettled(settled[i], i);
     }
 
-    if (settled.includes(undefined) || this.combined.aborted || performance.now() >= this.deadline) this.stopped = "batch deadline or cancellation; earlier commits remain" + this.deadlineNote();
+    if ((!this.stopped && settled.includes(undefined)) || this.combined.aborted || performance.now() >= this.deadline) this.stopped = "batch deadline or cancellation; earlier commits remain" + this.deadlineNote();
   }
 
   sequentialStop(result, i) {
@@ -183,9 +134,8 @@ class ProgramBatch {
     if (this.imageDropped) stopped = "batch image budget exceeded; remaining programs did not run";
 
     if (result.details?.ok === false) stopped = "program " + (i+1) + " failed; remaining programs did not run; earlier commits remain";
-    const text = programBatchText(this.results, this.programs.length, stopped);
-
-    if (text.length + this.imageTextChars() > this.config.maxReturnChars || result.details?.returnTruncated) stopped ||= "batch output budget exceeded; remaining programs did not run; earlier commits remain";
+    // Display clipping is not an execution failure. Finish every requested entry
+    // unless a real execution/resource limit stops it; boundedText caps delivery.
 
     if (result.details?.logTruncated) stopped ||= "batch log budget exceeded; remaining programs did not run; earlier commits remain";
 
@@ -222,6 +172,8 @@ class ProgramBatch {
     const failed = this.results.filter(result => result.details?.ok === false).length;
     const bounded = this.boundedText(failed);
     const content = [{type:"text",text:bounded.text}];
+    const logs = this.results.flatMap(result => result.details?.logs ?? []);
+    const logLimit = this.config.maxLogLines ?? 100;
     this.images.forEach((image,i) => content.push({type:"text",text:this.imageLabels[i]},image));
 
     // Return a typed stop report instead of throwing away earlier results/images.
@@ -230,7 +182,7 @@ class ProgramBatch {
       programs:this.results,attempted:this.results.length,total:this.programs.length,stopped:this.stopped,parallel:this.parallel,
       result:bounded.truncated ? bounded.text : this.results.map(result=>result.details?.result),
       returnTruncated:bounded.truncated || this.results.some(result=>result.details?.returnTruncated),
-      logTruncated:this.results.some(result=>result.details?.logTruncated),logs:this.results.flatMap(result=>result.details?.logs ?? []),trace:this.trace,mutations:mutationTotals(this.results)}};
+      logTruncated:logs.length > logLimit || this.results.some(result=>result.details?.logTruncated),logs:logs.slice(0,logLimit),trace:this.trace,mutations:mutationTotals(this.results)}};
   }
 
   async run() {
@@ -249,3 +201,18 @@ export async function runProgramBatch(id, params, signal, onUpdate, ctx, config,
 
   return new ProgramBatch(id, params, signal, onUpdate, ctx, config, execute, programs, batchTimeoutMs(params, config)).run();
 }
+
+function imageBlocks(result) {
+  return (Array.isArray(result?.content) ? result.content : []).filter(block => block?.type === "image" && isString(block.data));
+}
+
+function imageTotals(settled) {
+  let images = 0, bytes = 0;
+  for (const result of settled) {
+    for (const block of imageBlocks(result)) {
+      images++;
+      bytes += Buffer.byteLength(block.data, "base64");
+    }
+  }
+  return {images, bytes};
+}

package/src/runtime/program-file.js

@@ -1,3 +1,4 @@
+import { readLimitedBytes } from "../fs/file-io.js";
 import * as fs from "node:fs/promises";
 import { resolveWorkspacePath } from "../fs/workspace.js";
 
@@ -21,22 +22,13 @@ export async function readProgramFile(file, cwd, maxChars, signal) {
     const maxBytes = chars * 3;
     const tooLarge = () => new Error("code exceeds " + chars + " characters; split the program");
 
-    if (stat.size > maxBytes) throw tooLarge();
-    const chunks = [];
-    let bytes = 0;
-
-    for await (const chunk of handle.createReadStream({ end: maxBytes, autoClose: false, signal })) {
-      bytes += chunk.length;
-
-      if (bytes > maxBytes) throw tooLarge();
-      chunks.push(chunk);
-    }
+    const bytes = await readLimitedBytes(handle, stat, maxBytes, file, signal, tooLarge);
 
     signal?.throwIfAborted();
     // Do not silently replace invalid bytes in executable source. Preserve BOMs.
     let code;
 
-    try { code = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(Buffer.concat(chunks)); }
+    try { code = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes); }
     catch { throw new Error("program file " + file + " is not valid UTF-8 (encoded data could not be decoded); save it as UTF-8 text"); }
 
     if (code.length > chars) throw tooLarge();

package/src/runtime/program.js

@@ -0,0 +1,141 @@
+import {parse} from 'acorn';
+import {isObject,isString} from '../shared/decode.js';
+import {guestImportMessage,isDeniedGuestImport} from './guest-deny-imports.js';
+
+const PARSE_OPTIONS = { ecmaVersion: "latest", sourceType: "module", allowReturnOutsideFunction: true, allowAwaitOutsideFunction: true };
+
+const FUNCTION_TYPES = new Set(["FunctionDeclaration", "FunctionExpression", "ArrowFunctionExpression"]);
+
+function hasReturn(node) {
+  if (!isObject(node)) return false;
+
+  if (node.type === "ReturnStatement") return true;
+
+  if (FUNCTION_TYPES.has(node.type)) return false;
+
+  return Object.values(node).some(value => Array.isArray(value) ? value.some(hasReturn) : hasReturn(value));
+}
+
+function parseExpressionFunction(code) {
+  try {
+    const program = parse(code, PARSE_OPTIONS);
+    const statements = program.body.filter(node => node.type !== "EmptyStatement");
+    const statement = statements.length === 1 ? statements[0] : undefined;
+    const candidate = statement?.type === "ExpressionStatement" ? statement.expression : statement;
+
+    if (candidate && FUNCTION_TYPES.has(candidate.type)) {
+      return { program, expression: candidate, expressionSource: code.slice(statement.start, statement.end).replace(/;\s*$/, "") };
+    }
+
+    return { program };
+  } catch (bodyError) {
+    const expressionSource = code.trimEnd().replace(/;+\s*$/, "");
+
+    try {
+      const wrapped = parse("(" + expressionSource + "\n)", PARSE_OPTIONS);
+      const expression = wrapped.body[0]?.expression;
+
+      if (!expression || !FUNCTION_TYPES.has(expression.type)) throw bodyError;
+
+      return { expression, expressionSource };
+    } catch { throw bodyError; }
+  }
+}
+
+function deniedSpecifier(node) {
+  if (node?.type === "Literal" && isString(node.value)) return node.value;
+  if (node?.type === "TemplateLiteral" && node.expressions.length === 0) return node.quasis[0]?.value?.cooked;
+}
+
+function assertGuestImports(node) {
+  if (Array.isArray(node)) { node.forEach(assertGuestImports); return; }
+  if (!isObject(node)) return;
+  if (["ImportDeclaration", "ImportExpression"].includes(node.type)) rejectGuestImport(node.source);
+  if (node.type === "CallExpression" && node.callee?.type === "Identifier" && node.callee.name === "require") rejectGuestImport(node.arguments?.[0]);
+  for (const key of Object.keys(node)) {
+    if (!["start", "end", "loc", "range"].includes(key)) assertGuestImports(node[key]);
+  }
+}
+
+function prepareProgram(code) {
+  const parsed = parseExpressionFunction(code);
+  assertGuestImports(parsed.program ?? parsed.expression);
+  const body = parsed.expression ? "return await (" + parsed.expressionSource + "\n)();" : code;
+  const returns = parsed.expression
+    ? parsed.expression.type === "ArrowFunctionExpression" && parsed.expression.body.type !== "BlockStatement" || hasReturn(parsed.expression.body)
+    : hasReturn(parsed.program);
+
+  return { body, hasReturn: returns };
+}
+
+function admitData(data, cap) {
+  if (data === undefined) return { data };
+
+  try {
+    const encoded = JSON.stringify(data);
+
+    if (encoded === undefined) return { error: "data must be JSON-serializable" };
+    if (encoded.length > cap) return { error: "data exceeds " + cap + " characters (serialized JSON: " + encoded.length + " UTF-16 characters); no commands ran. Split literal inputs across invocations; large text can use write({path,content,append:true}) chunks without omitting content" };
+
+    return { data: JSON.parse(encoded) };
+  } catch { return { error: "data must be JSON-serializable" }; }
+}
+
+function admitCode({ code, file, cap }) {
+  if ((code === undefined) === (file === undefined)) return { error: "supply exactly one of code or file; no commands ran" };
+  if (file === undefined && (!isString(code) || !code.trim())) return { error: "code must be a non-empty string" };
+  if (file === undefined && code.length > cap) return { error: "code exceeds " + cap + " characters; split large writes into write({path,content,append:true}) chunks" };
+}
+
+function admitTimeout(config) {
+  const requestedTimeout = Number(config.timeoutMs === undefined ? 60000 : config.timeoutMs);
+
+  if (!Number.isFinite(requestedTimeout) || requestedTimeout <= 0) return { error: "timeoutMs must be a positive finite number" };
+
+  return { timeoutMs: Math.max(1, Math.min(2_147_483_647, Math.floor(requestedTimeout))) };
+}
+
+function admitGuest({ code, file, data, config }) {
+  const cap = config.maxCodeChars ?? 48000;
+  const codeError = admitCode({ code, file, cap });
+
+  if (codeError) return codeError;
+  const admitted = admitData(data, cap);
+
+  if (admitted.error) return admitted;
+  const timeout = admitTimeout(config);
+
+  if (timeout.error) return timeout;
+
+  return { data: admitted.data, timeoutMs: timeout.timeoutMs };
+}
+export { admitGuest, prepareProgram };
+
+function rejectGuestImport(source) {
+  const spec = deniedSpecifier(source);
+  const reason = spec && isDeniedGuestImport(spec) ? guestImportMessage(spec) : "guest cannot import modules; use read, edit, write, or bash";
+  throw new Error(reason + "; no commands ran");
+}
+
+function containingAwait(node, offset) {
+  if (Array.isArray(node)) return node.map(child => containingAwait(child, offset)).find(Boolean);
+  if (!isObject(node) || offset < node.start || offset >= node.end) return null;
+  for (const child of Object.values(node)) {
+    const found = containingAwait(child, offset);
+    if (found) return found;
+  }
+  return node.type === "AwaitExpression" ? node : null;
+}
+
+// V8 points at 'await'; JSC points at the called function's parenthesis. Report
+// the enclosing await expression consistently, using source syntax, not offsets.
+export function normalizeGuestLocation(source, location) {
+  if (!location?.awaited) return location;
+  const {line, col} = location;
+  const prefix = source.split("\n").slice(0, line - 1).join("\n");
+  const offset = prefix.length + Number(line > 1) + col - 1;
+  try {
+    const node = containingAwait(parse(source, {...PARSE_OPTIONS, locations:true}), offset);
+    return node ? {line:node.loc.start.line, col:node.loc.start.column + 1} : location;
+  } catch { return location; }
+}

package/src/runtime/reference.js

@@ -1,15 +1,16 @@
 // Standing tool description: sent on every request. No result or history compression.
-export const REFERENCE = `JS body/async arrow with read/write/edit/bash; no fs/import/require. file runs workspace scripts. data holds literal text/scripts/argv (≤48000 serialized JSON chars).
-read(path|paths,offset=1,limit?) → raw text/text[]; directories → entries; images: PNG/JPEG/GIF/WebP (≤16 attachments/20 MiB total).
-Path-only: ≤160 lines AND 8192 characters (UTF-16). Use read(path,{offset:1,limit:80}), about, or complete:true (whole file ≤31744 chars). Large JSONL: bounded parser via bash.
-read({path,json:selector}) → parsed JSON; selectors ".field", ".a[0:3]", ".a.length", quoted keys, true; 16 MiB input, no jq. Oversized → {status:"too_large",keys} (arrays: length); narrow selectors.
+export const REFERENCE = `JS body/async arrow: read/write/edit/bash; no fs/import/require. file: workspace scripts; data: literals (≤48000 JSON chars).
+read(path|paths,offset=1,limit?) → raw text/text[]; directories → entries; images: PNG/JPEG/GIF/WebP (≤16 images/20 MiB).
+Text ≤64 MiB internally; complete:true requires the whole file. Display alone is capped; return a summary or read(path,{offset:1,limit:80}). Larger files/JSONL: bounded bash parser.
+read({path,json:selector}) → parsed JSON: ".field", ".a[0:3]", ".a.length", quoted keys, true; input ≤16 MiB, selections ≤64 MiB storage, no jq. Values retain their types.
 read("symbol or question") = read({query,resolve:true}) → view; check status; view.text is a span, not the file.
 read(path,{about}) → windows; read({query,evidence:true}) → ranked evidence; read({path,outline:true}) → declarations.
 write(path,text) replaces unread workspace files; write({path,content,append:true}) appends without reading. After read: edit or replace:true.
-edit(path,oldText,newText) | edit({path,edits:[{oldText,newText}]}) exact unique read text; numbered windows (including misses), checks/references.
-edit(view,text) replaces the span; edit(view,old,new) matches uniquely within it. edit(async()=>{...}) checkpoint merges on success, rolls back/rethrows on failure; catch to recover.
-bash(command,{cwd?,timeoutMs?}) | bash({command,args}) literal argv; bounded output, nonzero throws; inherits program timeout unless overridden.
-Edits stage until success; bash commits first. Array errors abort; Promise.allSettled for optional reads.
-programs:[{code?,file?,data?}] inherits top-level code OR file and data. Entry source overrides; data replaces unless mergeData:true (shallow objects, entry keys win).
-Fresh guests/separate commits; sequential failure stops, prior commits stay. parallel:true for disjoint entries. Batch known work; separate calls only for new decisions.
+edit(path,oldText,newText) | edit({path,edits:[{oldText,newText}]}) unique exact read text; returns numbered windows/checks/references.
+edit(view,text) replaces span; edit(view,old,new) uniquely matches within it. edit(async()=>{...}) checkpoint: merge on success, rollback/rethrow on failure.
+bash(command,{cwd?,timeoutMs?}) | bash({command,args}) literal argv for scripts; bounded output, nonzero throws. Outer timeoutMs caps ALL waits/commands; bash inherits unless overridden.
+Edits commit on success/before bash. Promise.allSettled keeps partial results.
+programs:[{code?,file?,data?}] inherits source/data; entries override. mergeData:true shallow object merge (entry keys win).
+Promise.all: ≤8 reads or disjoint-file mutations; same-file serial; bash/checkpoint barriers.
+Fresh guests/separate commits. Sequential failure stops; prior commits stay. parallel:true for disjoint entries. ONE call for known work; split for new decisions.
 `;