pi-supernova 0.0.8 → 0.0.11

package/host-bridge.js

@@ -8,9 +8,11 @@ import { unknownToolMessage } from "./catalog.js";
 import { extractStructuralSurface } from "./surface.js";
 import { buildEditDiff, buildMultiEditDiff, buildPatchDiff, buildWriteDiff } from "./diff.js";
 import { executeSnap } from "./snap.js";
+import { selectEvidence } from "./evidence.js";
+import { WorkspaceIndex, globToRegExp } from "./repo-index.js";
 import { CausalVfs } from "./vfs.js";
 import { applyPatchToText } from "./patch.js";
-import { resolveWorkspacePath, runCommand } from "./workspace.js";
+import { resolveWorkspacePath, runCommand, clearPathCache } from "./workspace.js";
 
 function textResult(text, details) {
   return {
@@ -113,7 +115,61 @@ async function formatLsEntry(dirPath, entry) {
   return `${entry.name}${isDir ? "/" : ""} (${typeLabel}${sizeSuffix})`;
 }
 
-function createNativeAdapters(getCwd, vfs, config) {
+function rgGrepArgs(pattern, params, searchPath) {
+  const args = ["--line-number", "--no-heading", "--color", "never"];
+  if (params?.caseSensitive !== true) args.push("--ignore-case");
+  if (params?.glob) args.push("--glob", String(params.glob));
+  args.push("--", pattern, searchPath);
+  return args;
+}
+
+/** rg --files, then find(1) when rg is unavailable; both accept an optional glob/name pattern. */
+async function listWithTools(searchDir, pattern, cwd, signal) {
+  const args = ["--files"];
+  if (pattern) args.push("-g", pattern);
+  const res = await runCommand(["rg", ...args, searchDir], { cwd, timeoutMs: 30_000, signal }).catch(() => null);
+  if (res && (res.exitCode === 0 || res.exitCode === 1)) return textResult(res.stdout, { via: "rg" });
+  const findArgs = [searchDir];
+  if (pattern) findArgs.push("-name", pattern);
+  const findRes = await runCommand(["find", ...findArgs], { cwd, timeoutMs: 30_000, signal });
+  return textResult(findRes.stdout, { via: "find" });
+}
+
+/** rg-compatible grep served from the index; null when the pattern or tree needs real rg. */
+async function grepIndexed(index, pattern, params, searchPath, cwd) {
+  let regex;
+  try {
+    regex = new RegExp(pattern, params?.caseSensitive === true ? "" : "i");
+  } catch {
+    return null;
+  }
+  let files = await index.files(searchPath);
+  if (!index.canScan(files)) return null;
+  if (params?.glob) {
+    const matcher = globToRegExp(String(params.glob));
+    files = files.filter((f) => matcher.test(path.relative(cwd, f).split(path.sep).join("/")));
+  }
+  const rows = index.grep(files, regex, cwd);
+  return rows.length ? rows.join("\n") + "\n" : "";
+}
+
+/** rg --files [-g pattern] served from the index; null when the tree is too large. */
+async function listIndexed(index, root, cwd, pattern) {
+  const files = await index.files(root);
+  if (!index.canScan(files)) return null;
+  const rel = files.map((f) => path.relative(cwd, f).split(path.sep).join("/"));
+  if (!pattern) return rel.length ? rel.join("\n") + "\n" : "";
+  let matcher;
+  try {
+    matcher = globToRegExp(pattern);
+  } catch {
+    return null;
+  }
+  const hits = rel.filter((f) => matcher.test(f));
+  return hits.length ? hits.join("\n") + "\n" : "";
+}
+
+function createNativeAdapters(getCwd, vfs, config, index) {
   async function readAdapter(params, signal) {
       const cwd = getCwd();
       const targetParam = params?.path ?? params?.target;
@@ -143,8 +199,9 @@ function createNativeAdapters(getCwd, vfs, config) {
           const snapRes = await executeSnap({
             query: targetParam,
             searchDir: cwd,
-            vfs,
-            runCommand: (argv, opts) => runCommand(argv, { cwd, signal, ...opts }),
+            index,
+            overlayText: (p) => vfs.getOverlay(p),
+            pendingPaths: vfs.getOverlayPaths(),
           });
           return textResult(JSON.stringify(snapRes, null, 2), { ...snapRes, isSnap: true });
         } catch {}
@@ -243,19 +300,25 @@ function createNativeAdapters(getCwd, vfs, config) {
       const res = await executeSnap({
         query: params.query,
         searchDir: snapTarget,
+        root: cwd,
         includeHidden,
-        vfs: {
-          read: async (candidate) => {
-            // Jail each snap candidate (symlink files must not escape the workspace).
-            const jailed = await resolveWorkspacePath(cwd, candidate, "snap", false);
-            return vfs.read(jailed);
-          },
-        },
-        runCommand: (argv, opts) => runCommand(argv, { cwd: snapTarget, signal, ...opts }),
+        index,
+        overlayText: (p) => vfs.getOverlay(p),
         pendingPaths: vfs.getOverlayPaths(),
       });
       return textResult(JSON.stringify(res, null, 2), res);
     },
+    async evidence(params, signal) {
+      const cwd = getCwd();
+      if (!isString(params?.query) || !params.query.trim()) throw new Error("evidence requires query");
+      if (signal?.aborted) throw new Error("aborted");
+      const searchDir = params?.path ? await resolveWorkspacePath(cwd, params.path, "evidence", true) : cwd;
+      const options = {};
+      if (Number.isInteger(params?.k) && params.k > 0) options.k = params.k;
+      if (Number.isInteger(params?.maxChars) && params.maxChars > 0) options.maxChars = params.maxChars;
+      const res = await selectEvidence({ query: params.query, root: cwd, searchDir, index, overlayText: (p) => vfs.getOverlay(p), options });
+      return textResult(JSON.stringify(res), { route: res.route, count: res.spans.length });
+    },
     async surface(params, signal) {
       const cwd = getCwd();
       const target = await resolveWorkspacePath(cwd, params?.path, "surface", false);
@@ -286,6 +349,7 @@ function createNativeAdapters(getCwd, vfs, config) {
         });
       } finally {
         vfs.invalidateCache();
+        index.invalidate();
       }
       const { stdout, stderr } = res;
       const text = stdout && stderr ? stdout + (stdout.endsWith("\n") ? "" : "\n") + stderr : stdout || stderr;
@@ -300,11 +364,9 @@ function createNativeAdapters(getCwd, vfs, config) {
       const pattern = String(params?.pattern || "");
       if (!pattern) throw new Error("grep requires pattern");
       const searchPath = params?.path ? await resolveWorkspacePath(cwd, params.path, "grep", true) : cwd;
-      const args = ["--line-number", "--no-heading", "--color", "never"];
-      if (params?.caseSensitive !== true) args.push("--ignore-case");
-      if (params?.glob) args.push("--glob", String(params.glob));
-      args.push("--", pattern, searchPath);
-      const res = await runCommand(["rg", ...args], { cwd, timeoutMs: 30_000, signal });
+      const indexed = await grepIndexed(index, pattern, params, searchPath, cwd);
+      if (indexed !== null) return textResult(indexed, { exitCode: indexed ? 0 : 1, via: "index" });
+      const res = await runCommand(["rg", ...rgGrepArgs(pattern, params, searchPath)], { cwd, timeoutMs: 30_000, signal });
       if (res.exitCode !== 0 && res.exitCode !== 1) {
         throw new Error(res.stderr.trim() || `rg exited ${res.exitCode}`);
       }
@@ -314,6 +376,8 @@ function createNativeAdapters(getCwd, vfs, config) {
       const cwd = getCwd();
       const pattern = String(params?.pattern || "");
       if (!pattern) throw new Error("glob requires pattern");
+      const indexed = await listIndexed(index, cwd, cwd, pattern);
+      if (indexed !== null) return textResult(indexed, { via: "index" });
       const rg = await runCommand(["rg", "--files", "-g", pattern], { cwd, timeoutMs: 30_000, signal }).catch(
         () => null,
       );
@@ -333,16 +397,10 @@ function createNativeAdapters(getCwd, vfs, config) {
       const searchDir = params?.path ? await resolveWorkspacePath(cwd, params.path, "find", true) : cwd;
       const pattern = params?.pattern || params?.glob;
       if (signal?.aborted) throw new Error("aborted");
-      const args = ["--files"];
-      if (pattern) args.push("-g", String(pattern));
-      const res = await runCommand(["rg", ...args, searchDir], { cwd, timeoutMs: 30_000, signal }).catch(() => null);
-      if (res && (res.exitCode === 0 || res.exitCode === 1)) {
-        return textResult(res.stdout, { via: "rg" });
-      }
-      const findArgs = [searchDir];
-      if (pattern) findArgs.push("-name", String(pattern));
-      const findRes = await runCommand(["find", ...findArgs], { cwd, timeoutMs: 30_000, signal });
-      return textResult(findRes.stdout, { via: "find" });
+      const globPattern = pattern ? String(pattern) : null;
+      const indexed = await listIndexed(index, searchDir, cwd, globPattern);
+      if (indexed !== null) return textResult(indexed, { via: "index" });
+      return listWithTools(searchDir, globPattern, cwd, signal);
     },
     async ls(params, signal) {
       const cwd = getCwd();
@@ -359,9 +417,10 @@ function createNativeAdapters(getCwd, vfs, config) {
 }
 
 export function createHostBridge({ pi, config, getCwd }) {
-  const vfs = new CausalVfs();
+  const index = new WorkspaceIndex((argv, opts) => runCommand(argv, opts));
+  const vfs = new CausalVfs(() => index.invalidate());
   const executors = new Map();
-  const natives = createNativeAdapters(getCwd, vfs, config);
+  const natives = createNativeAdapters(getCwd, vfs, config, index);
   let callCount = 0;
   let activeCtx = null;
   let activeSignal = undefined;
@@ -395,6 +454,7 @@ export function createHostBridge({ pi, config, getCwd }) {
     trace = [];
     // Files may change between programs (editor, git); never serve a stale run.
     vfs.invalidateCache();
+    clearPathCache();
   }
 
   function getTrace() {
@@ -514,6 +574,7 @@ export function createHostBridge({ pi, config, getCwd }) {
     } catch (error) {
       record.ok = false;
       record.ms = Date.now() - record.time;
+      record.error = error instanceof Error ? error.message : String(error);
       notifyCall(record);
       throw error;
     }

package/index.js

@@ -31,14 +31,44 @@ function result(text, details) {
   return { content: [{ type: "text", text }], details };
 }
 
-/** Live trace updates for the card; a throwing host callback must never break the run. */
+const PROGRESS_FRAME_MS = 40;
+
+/**
+ * Live trace updates for the card. The first update is immediate (seeds the result slot);
+ * later ones are coalesced to one host re-render per frame so a tight loop of nova.calls
+ * is not throttled by the TUI. A throwing host callback must never break the run.
+ */
 function progressEmitter(onUpdate) {
-  if (!isFunction(onUpdate)) return () => {};
-  return (trace) => {
+  if (!isFunction(onUpdate)) return Object.assign(() => {}, { flush() {} });
+  let pending = null;
+  let timer = null;
+  const send = (trace) => {
     try {
       onUpdate({ content: [{ type: "text", text: "" }], details: { trace, running: true } });
     } catch {}
   };
+  const flush = () => {
+    timer = null;
+    if (pending === null) return;
+    const trace = pending;
+    pending = null;
+    send(trace);
+  };
+  const emit = (trace) => {
+    if (timer === null && pending === null) {
+      send(trace);
+      timer = setTimeout(flush, PROGRESS_FRAME_MS);
+      return;
+    }
+    pending = trace;
+    if (timer === null) timer = setTimeout(flush, PROGRESS_FRAME_MS);
+  };
+  emit.flush = () => {
+    if (timer !== null) clearTimeout(timer);
+    pending = null;
+    timer = null;
+  };
+  return emit;
 }
 
 function logsBlock(outcome, tail = "") {
@@ -73,6 +103,7 @@ Globals (async):
 read(path|paths, offset?, limit?) → text | text[]
 write(path, text) · edit(path, oldText, newText) · patch(path, unifiedDiff)
 bash(cmd, {cwd?, timeoutMs?}) → output, throws on non-zero exit · exec(cmd, argv?) quotes argv
+evidence(query, {k?}) → {spans: [{path, lines, name, text}]} top-K spans that answer a question — use before read
 snap(query, root?) → {path, line, signature, context} · surface(path) → {items: [{name, kind, line}]}
 nova.call(name, args) → {ok, value} for any host tool · nova.callMany([{name, args}]) parallel when read-only
 nova.search(query) → [{name, description}] · nova.describe(name) → parameters · nova.has(name) sync
@@ -148,7 +179,7 @@ export default function piSupernova(pi) {
     description: TOOL_DESCRIPTION,
     promptSnippet: "Compose host tools in one JavaScript program",
     promptGuidelines: [
-      "Use supernova for multi-step tool work: loops, filtering, parallel reads, read→edit chains. Return a compact shaped value; keep raw tool output inside the program.",
+      "Use supernova for multi-step tool work: loops, filtering, parallel reads, read→edit chains. To understand code, call evidence(question) and read only the returned spans; read whole files only to edit them. Return a compact shaped value; keep raw tool output inside the program.",
     ],
     parameters: Type.Object({
       code: Type.String({ description: "JavaScript program: async body or arrow function." }),
@@ -180,6 +211,7 @@ export default function piSupernova(pi) {
         outcome = await runProgram(params, runController.signal, abortRun);
       } finally {
         bridge.setCallListener(null);
+        emitProgress.flush();
         signal?.removeEventListener("abort", abortRun);
       }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-supernova",
-  "version": "0.0.8",
+  "version": "0.0.11",
   "description": "Dual-host CodeMode for Pi/OMP: progressive tool discovery, result bottleneck, and Amdahl Auto parallel.",
   "type": "module",
   "author": "AdityaVG13",
@@ -30,6 +30,8 @@
     "parallel.js",
     "runtime.js",
     "guest-worker.js",
+    "repo-index.js",
+    "evidence.js",
     "format.js",
     "patch.js",
     "vfs.js",

package/render.js

@@ -178,7 +178,7 @@ function displayOperation(tool, target, diff, ok, item) {
 	const rawName = cleanInlineText(tool);
 	if (!rawName) return null;
 	const normalized = rawName === "apply_patch" ? "patch" : rawName;
-	return { tool: normalized, target, diff, ok, ms: item?.ms, exitCode: item?.exitCode, time: item?.time };
+	return { tool: normalized, target, diff, ok, ms: item?.ms, exitCode: item?.exitCode, time: item?.time, error: item?.error };
 }
 
 function isTheme(value) {
@@ -271,6 +271,11 @@ export function normalizeResultRenderArgs(result, options, themeOrCtx, ctxOrArgs
 	throw new Error("supernova renderResult: theme missing (expected Pi or OMP signature)");
 }
 
+function batchTarget(paths) {
+	const names = paths.map((p) => String(p).replace(/\\/g, "/").split("/").pop());
+	return `${paths.length} files: ${names.join(", ")}`;
+}
+
 const OPERATION_TARGETS = [
 	[(item) => item?.name === "snap", (item, args) => {
 		const query = args.query ? `"${args.query}"` : "";
@@ -278,6 +283,7 @@ const OPERATION_TARGETS = [
 		return `${query} → ${args.path}`;
 	}],
 	[(item) => item?.name === "search", (item, args) => (args.query ? `"${args.query}"` : "")],
+	[(item, args) => Array.isArray(args.path), (item, args) => batchTarget(args.path)],
 	[(item, args) => args.path, (item, args) => String(args.path)],
 	[(item) => item?.diff?.path, (item) => String(item.diff.path)],
 	[(item, args) => args.target && isString(args.target), (item, args) => args.target],
@@ -405,8 +411,11 @@ function formatOpRow(theme, op, width, isPartial, isError) {
 		prefix += theme.fg("toolDiffAdded", added) + theme.fg("dim", "/") + theme.fg("toolDiffRemoved", removed) + " ";
 		used += added.length + 1 + removed.length + 1;
 	}
-	const target = formatTarget(op, Math.max(1, width - used));
-	return prefix + (target ? theme.fg("muted", target) : theme.fg("dim", "done"));
+	const budget = Math.max(1, width - used);
+	const target = formatTarget(op, budget);
+	if (target) return prefix + theme.fg("muted", target);
+	if (op.ok === false && op.error) return prefix + theme.fg("error", clampLine(cleanInlineText(op.error), budget));
+	return prefix.trimEnd();
 }
 
 function operationsFor(payload, context, args) {
@@ -486,7 +495,8 @@ class UnifiedResultCard {
 					}),
 					sections: view.lines.length > 0 ? [{ lines: view.lines }] : [],
 					state: model.isError ? "error" : model.isPartial ? "pending" : "success",
-					borderColor: model.isError ? "error" : "borderMuted",
+					// borderMuted is invisible on OMP's card background; dim matches the duration column.
+					borderColor: model.isError ? "error" : "dim",
 					width: frameWidth,
 				};
 			});

package/repo-index.js

@@ -0,0 +1,179 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { extractStructuralSurface } from "./surface.js";
+
+// In-process workspace index: the gitignore-aware file list comes from one
+// \`rg --files\` spawn and is then reused; file text, lowercase text, and the
+// structural surface are cached per path and validated by mtime. snap/grep/glob
+// read from here instead of spawning, so a warm call is sub-millisecond.
+
+const LIST_TTL_MS = 10_000;
+const MAX_INDEXED_FILES = 4000;
+const MAX_FILE_BYTES = 512 * 1024;
+const BINARY_EXT = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".tgz", ".tar", ".bz2", ".xz", ".7z",
+  ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".mov", ".wav", ".ogg", ".webm", ".wasm", ".class",
+  ".jar", ".so", ".dylib", ".dll", ".exe", ".bin", ".o", ".a", ".node", ".lock", ".sqlite", ".sqlite3", ".db",
+]);
+const REGEX_SPECIAL = /[.+^${}()|\\]/g;
+const IDENT_TOKEN = /[A-Za-z_$][\w$]*/g;
+const EMPTY = Object.freeze([]);
+const DEF_PATTERN = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)/;
+
+export function isTextCandidate(filePath) {
+  return !BINARY_EXT.has(path.extname(filePath).toLowerCase());
+}
+
+/** Translate one glob token at index i → [regexSource, nextIndex]. */
+function globToken(glob, i) {
+  const ch = glob[i];
+  if (ch === "*" && glob[i + 1] === "*") {
+    const slashAfter = glob[i + 2] === "/";
+    return [slashAfter ? "(?:.*/)?" : ".*", i + (slashAfter ? 3 : 2)];
+  }
+  if (ch === "*") return ["[^/]*", i + 1];
+  if (ch === "?") return ["[^/]", i + 1];
+  if (ch === "{" || ch === "[") return globGroup(glob, i, ch);
+  return [ch.replace(REGEX_SPECIAL, "\\$&"), i + 1];
+}
+
+/** {a,b} alternation or [..] class starting at i. */
+function globGroup(glob, i, open) {
+  const close = open === "{" ? "}" : "]";
+  const end = glob.indexOf(close, i);
+  if (end < 0) throw new SyntaxError("unclosed " + open + " in glob");
+  const inner = glob.slice(i + 1, end);
+  const source = open === "{" ? "(?:" + inner.split(",").map(globBody).join("|") + ")" : "[" + inner + "]";
+  return [source, end + 1];
+}
+
+function globBody(glob) {
+  let source = "";
+  let i = 0;
+  while (i < glob.length) {
+    const [piece, next] = globToken(glob, i);
+    source += piece;
+    i = next;
+  }
+  return source;
+}
+
+/** gitignore-style glob (rg -g) → RegExp over a "/"-separated relative path. No slash ⇒ basename match anywhere. */
+export function globToRegExp(glob) {
+  const body = globBody(glob);
+  return new RegExp(glob.includes("/") ? "^" + body + "$" : "(?:^|/)" + body + "$");
+}
+
+export class WorkspaceIndex {
+  constructor(runCommand) {
+    this.runCommand = runCommand;
+    this.lists = new Map();
+    this.entries = new Map();
+  }
+
+  invalidate() {
+    this.lists.clear();
+  }
+
+  /** Absolute, sorted file list for a root; gitignore-aware via rg; cached for LIST_TTL_MS. */
+  async files(root, includeHidden = false) {
+    const key = root + "\0" + (includeHidden ? "h" : "");
+    const cached = this.lists.get(key);
+    if (cached && Date.now() - cached.at < LIST_TTL_MS) return cached.files;
+    const args = ["rg", "--files"];
+    if (includeHidden) args.push("--hidden");
+    args.push("-g", "!.git/**", "-g", "!**/.git/**", root);
+    let files = [];
+    try {
+      const res = await this.runCommand(args, { cwd: root, timeoutMs: 15_000 });
+      files = res.stdout.split("\n").map((f) => f.trim()).filter(Boolean).map((f) => path.resolve(root, f)).sort();
+    } catch {
+      files = [];
+    }
+    this.lists.set(key, { files, at: Date.now() });
+    return files;
+  }
+
+  /** Cached {text, lower, ext, surface?} for a file, re-read when mtime/size changed. Null for unreadable, binary, or huge files. */
+  entry(filePath) {
+    if (!isTextCandidate(filePath)) return null;
+    let stat;
+    try {
+      stat = fs.statSync(filePath);
+    } catch {
+      this.entries.delete(filePath);
+      return null;
+    }
+    if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null;
+    const cached = this.entries.get(filePath);
+    if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) return cached;
+    let text;
+    try {
+      text = fs.readFileSync(filePath, "utf8");
+    } catch {
+      return null;
+    }
+    if (text.includes("\0")) return null;
+    const created = { text, lower: text.toLowerCase(), mtimeMs: stat.mtimeMs, size: stat.size, ext: path.extname(filePath), surface: undefined, lines: undefined };
+    this.entries.set(filePath, created);
+    return created;
+  }
+
+  static fromText(filePath, text) {
+    return { text, lower: text.toLowerCase(), ext: path.extname(filePath), surface: undefined, lines: undefined };
+  }
+
+  /** Per-line raw text, lowercase text, declared identifier (or ""), and identifier tokens — computed once per entry. */
+  static linesOf(entry) {
+    if (entry.lines) return entry.lines;
+    const raw = entry.text.split("\n");
+    const lower = new Array(raw.length);
+    const defNames = new Array(raw.length);
+    const idents = new Array(raw.length);
+    for (let i = 0; i < raw.length; i++) {
+      const trimmed = raw[i].trim();
+      lower[i] = trimmed.toLowerCase();
+      defNames[i] = DEF_PATTERN.exec(trimmed)?.[2].toLowerCase() ?? "";
+      idents[i] = trimmed.match(IDENT_TOKEN) || EMPTY;
+    }
+    entry.lines = { raw, lower, defNames, idents };
+    return entry.lines;
+  }
+
+  static surfaceOf(entry) {
+    if (!entry.surface) entry.surface = extractStructuralSurface(entry.text, entry.ext);
+    return entry.surface;
+  }
+
+  /** True when the list is small enough to scan in-process instead of spawning rg. */
+  canScan(files) {
+    return files.length <= MAX_INDEXED_FILES;
+  }
+
+  /** Files whose lowercase text contains any (or every) needle; needles are lowercase. */
+  filesContaining(files, needles, anyOf) {
+    const hits = [];
+    for (const filePath of files) {
+      const e = this.entry(filePath);
+      if (!e) continue;
+      const found = anyOf ? needles.some((n) => e.lower.includes(n)) : needles.every((n) => e.lower.includes(n));
+      if (found) hits.push(filePath);
+    }
+    return hits;
+  }
+
+  /** rg-style "path:line:text" rows over the indexed files, paths relative to root. */
+  grep(files, regex, root) {
+    const out = [];
+    for (const filePath of files) {
+      const e = this.entry(filePath);
+      if (!e || !regex.test(e.text)) continue;
+      const lines = e.text.split("\n");
+      const rel = path.relative(root, filePath) || filePath;
+      for (let i = 0; i < lines.length; i++) {
+        if (regex.test(lines[i])) out.push(rel + ":" + (i + 1) + ":" + lines[i]);
+      }
+    }
+    return out;
+  }
+}