pi-supernova 0.3.1 → 0.4.0

package/src/context/ledger.js

@@ -1,20 +1,79 @@
+import { isObject, isString } from "../shared/decode.js";
+
 const MIN_RUN = 6;
+
 const MIN_SUBSTANTIVE = 4;
+
 const MAX_CANDIDATES = 8;
-const DEFAULT_WINDOW = 40;
+
+const DEFAULT_WINDOW = 0;
+
 const MAX_STORED_LINES = 200_000;
 
+const MAX_OBSERVE_DEPTH = 64;
+
+// A line this long is a minified blob or an inlined image, not a run of source. It
+// cannot be part of a collapsible run anyway (MIN_RUN needs six lines), and skipping
+// it keeps base64 payloads out of the retention set entirely.
+const MAX_OBSERVED_LINE = 4096;
+
+// A collapse marker is a citation, not content. Never index or re-collapse one, so
+// citations cannot nest into a chain that points somewhere the reader must follow.
+const COLLAPSE_MARKER = /^⋯ \d+ lines same as #\d+/;
+
 function hashLine(line) {
   let hash = 0x811c9dc5;
+
   for (let i = 0; i < line.length; i++) hash = Math.imul(hash ^ line.charCodeAt(i), 0x01000193);
+
   return hash >>> 0;
 }
+
 function substantive(line) { return line.trim().length >= 8; }
+
 function newHistory() {
-  return { results: new Map(), occurrences: new Map(), storedLines: 0, latestCall: 0,
+  return { results: new Map(), occurrences: new Map(), storedLines: 0, latestCall: 0, retained: null,
     stats: { programs: 0, returnedChars: 0, collapsedChars: 0, collapsedRuns: 0 } };
 }
 
+// Experimental candidate collection, not proof of provider-visible retention:
+// AgentMessage metadata and later context transformations can hide these strings.
+// Keep this optimization opt-in until exact outgoing citation targets are validated.
+function collectRetained(value, retained, depth = 0) {
+  if (isString(value)) {
+    // Every line, not only substantive ones. A run may span a blank or short line,
+    // so isRetained must be exact per line or the run truncates there. An oversized
+    // line is never stored: it cannot sit inside a six-line run, and this is what
+    // keeps base64 image payloads out of the retention set.
+    if (value.length > MAX_OBSERVED_LINE && !value.includes("\n")) return;
+
+    for (const line of value.split("\n")) {
+      if (retained.size >= MAX_STORED_LINES) return;
+
+      if (line.length <= MAX_OBSERVED_LINE) retained.add(line);
+    }
+
+    return;
+  }
+
+  if (depth >= MAX_OBSERVE_DEPTH) return;
+
+  if (Array.isArray(value)) {
+    for (const item of value) collectRetained(item, retained, depth + 1);
+
+    return;
+  }
+
+  // pi's AgentMessage payload is plain objects and arrays, so isObject is the whole
+  // traversal contract; anything else is not message text.
+  if (!isObject(value)) return;
+
+  for (const key of Object.keys(value)) collectRetained(value[key], retained, depth + 1);
+}
+
+/** Content that may be collapsed out of a later result. Citations are excluded. */
+function collapsible(line) { return substantive(line) && !COLLAPSE_MARKER.test(line); }
+
 export class SeenLedger {
   constructor({ window = DEFAULT_WINDOW, history } = {}) {
     this.window = window;
@@ -26,6 +85,31 @@ export class SeenLedger {
   get occurrences() { return this.history.occurrences; }
   get storedLines() { return this.history.storedLines; }
   get stats() { return this.history.stats; }
+  get retainedLines() { return this.history.retained; }
+
+  /**
+   * Record candidate lines from the host's pre-request hook. This observation
+   * includes metadata and precedes later transformations, so it is not proof
+   * that a citation target will be present in the final provider payload.
+   * Until this runs, nothing is collapsible, so the first result of a session is
+   * always sent in full.
+   */
+  observe(payload) {
+    if (this.window === 0) { this.history.retained = null;
+
+ return; }
+
+    const retained = new Set();
+    collectRetained(payload, retained);
+    this.history.retained = retained;
+  }
+
+  /** Candidate membership only; not a final-payload retention guarantee. */
+  isRetained(line) {
+    const retained = this.history.retained;
+
+    return retained !== null && retained.has(line);
+  }
   fork() { return new SeenLedger({ window: this.window, history: this.history }); }
 
   reset() {
@@ -39,54 +123,74 @@ export class SeenLedger {
     this.pinned.clear();
     this.stats.programs++;
     this.history.latestCall = Math.max(this.history.latestCall, call);
+
     for (const old of this.results.keys()) if (old <= this.history.latestCall - this.window) this.forget(old);
   }
 
   forget(call) {
     const entry = this.results.get(call);
+
     if (!entry) return;
+
     for (const hash of new Set(entry.hashes)) {
       const kept = (this.occurrences.get(hash) ?? []).filter(item => item.call !== call);
+
       if (kept.length) this.occurrences.set(hash, kept);
       else this.occurrences.delete(hash);
     }
+
     this.history.storedLines -= entry.lines.length;
     this.results.delete(call);
   }
 
   recordOrigin(path, firstLine, lines, pin = false) {
     if (this.window === 0) return;
+
     for (let i = 0; i < lines.length; i++) {
       const text = lines[i];
+
       if (!substantive(text)) continue;
+
       if (this.origins.size >= MAX_STORED_LINES && !this.origins.has(text)) this.origins.delete(this.origins.keys().next().value);
       this.origins.set(text, { path, line: firstLine + i });
+
       if (pin) this.pinned.add(text);
     }
   }
 
   runLength(lines, hashes, candidate, index) {
     const earlier = this.results.get(candidate.call);
+
     if (!earlier) return 0;
     let count = 0;
+
     while (index + count < lines.length && candidate.index + count < earlier.lines.length
       && hashes[index + count] === earlier.hashes[candidate.index + count]
       && lines[index + count] === earlier.lines[candidate.index + count]
-      && !this.pinned.has(lines[index + count])) count++;
+      && !this.pinned.has(lines[index + count])
+      // Experimental line membership does not establish a visible, ordered target.
+      && this.isRetained(lines[index + count])) count++;
+
     return count;
   }
 
   longestRun(lines, hashes, index, call) {
     const candidates = this.occurrences.get(hashes[index]);
+
     if (!candidates) return null;
     let best = null;
+
     for (const candidate of candidates.filter(item => item.call < call).slice(-MAX_CANDIDATES)) {
       const length = this.runLength(lines, hashes, candidate, index);
+
       if (length >= MIN_RUN && (!best || length > best.length)) best = { ...candidate, length };
     }
+
     if (!best) return null;
     let count = 0;
-    for (let i = index; i < index + best.length; i++) if (substantive(lines[i])) count++;
+
+    for (let i = index; i < index + best.length; i++) if (collapsible(lines[i])) count++;
+
     return count >= MIN_SUBSTANTIVE ? best : null;
   }
 
@@ -94,56 +198,74 @@ export class SeenLedger {
     const earlier = this.results.get(run.call);
     const origin = offset => this.origins.get(lines[index + offset]) ?? earlier?.origins[run.index + offset];
     const first = origin(0);
+
     if (!first) return "";
     let expected = first.line;
+
     for (let i = 0; i < run.length; i++) {
       const item = origin(i);
+
       if (item && (item.path !== first.path || item.line !== expected)) return "";
       expected++;
     }
+
     return first.path + ":" + first.line + "–" + (expected - 1);
   }
 
   dedupe(text, call) {
     if (this.window === 0) {
       this.stats.returnedChars += text.length;
+
       return text;
     }
+
     const lines = text.split("\n");
     const hashes = Uint32Array.from(lines, hashLine);
     const out = [];
     let collapsedChars = 0;
+
     for (let i = 0; i < lines.length;) {
-      const run = substantive(lines[i]) ? this.longestRun(lines, hashes, i, call) : null;
+      const run = collapsible(lines[i]) ? this.longestRun(lines, hashes, i, call) : null;
+
       if (!run) { out.push(lines[i++]); continue; }
+
       const cite = this.citation(lines, i, run);
       out.push("⋯ " + run.length + " lines same as #" + run.call + (cite ? " · " + cite : "") + " ⋯");
+
       for (let k = 0; k < run.length; k++) collapsedChars += lines[i + k].length + 1;
       this.stats.collapsedRuns++;
       i += run.length;
     }
+
     const sent = out.join("\n");
     this.stats.returnedChars += sent.length;
     this.stats.collapsedChars += collapsedChars;
     this.remember(call, out);
+
     return sent;
   }
 
   remember(call, lines) {
     if (this.window === 0 || call <= this.history.latestCall - this.window || lines.length > MAX_STORED_LINES) return;
+
     if (this.results.has(call)) this.forget(call);
+
     for (const old of [...this.results.keys()].sort((a, b) => a - b)) {
       if (this.storedLines + lines.length <= MAX_STORED_LINES) break;
       this.forget(old);
     }
+
     const hashes = Uint32Array.from(lines, hashLine);
     const origins = lines.map(line => this.origins.get(line));
+
     for (let i = 0; i < lines.length; i++) {
-      if (!substantive(lines[i])) continue;
+      if (!collapsible(lines[i])) continue;
       let list = this.occurrences.get(hashes[i]);
+
       if (!list) this.occurrences.set(hashes[i], (list = []));
       list.push({ call, index: i });
     }
+
     this.results.set(call, { hashes, lines, origins });
     this.history.storedLines += lines.length;
   }

package/src/context/outline.js

@@ -14,12 +14,16 @@ function relevance(span, lower, stems) {
   if (stems.length === 0) return 0;
   const nameLower = span.name.toLowerCase();
   let score = 0;
+
   for (const s of stems) if (nameLower.includes(s)) score += 40;
+
   for (let i = span.start - 1; i < span.end; i++) {
     let hits = 0;
+
     for (const s of stems) if (lower[i].includes(s)) hits++;
     score += hits * hits * 5;
   }
+
   return score;
 }
 
@@ -30,30 +34,64 @@ function chooseExpanded(spans, lower, stems, raw, opts) {
   let budget = opts.maxChars;
   // Weak-match cutoff (as fff's weak-match detector): stop once relevance falls below 40% of the best span.
   const floor = stems.length > 0 ? Math.max(1, scored[0].r * 0.4) : 0;
+
   for (const { i, r, chars } of scored) {
     if (expanded.size >= opts.maxExpanded || r < floor) break;
+
     if (chars > budget) continue;
     expanded.add(i);
     budget -= chars;
   }
+
   return expanded;
 }
 
 function foldedLine(span) {
   const body = span.end - span.start;
   const sig = span.signature.replace(/\s*\{\s*$/, "");
+
   return String(span.start).padStart(5) + " " + sig + (body > 0 ? " … " + body + " lines" : "");
 }
 
 function expandedBlock(span, raw, opts) {
   const out = [];
+
   for (let l = span.start; l <= span.end; l++) out.push(String(l).padStart(5) + " " + raw[l - 1]);
   const refs = opts.references ? opts.references(span.name, span.start) : [];
+
   // Who uses this declaration: the relation a reader would otherwise grep for next.
   if (refs.length) out.push("      // used by: " + refs.slice(0, opts.maxRefs).join(", ") + (refs.length > opts.maxRefs ? " (+" + (refs.length - opts.maxRefs) + ")" : ""));
+
   return out.join("\n");
 }
 
+function focusedText(raw, lower, stems, relPath, opts) {
+  const hits = lower.map((line, i) => ({i, score: stems.filter(s => line.includes(s)).length}))
+    .filter(hit => hit.score > 0).sort((a, b) => b.score - a.score || a.i - b.i);
+
+  const parts = [];
+  const covered = new Set();
+  let budget = opts.maxChars;
+
+  for (const {i} of hits) {
+    if (parts.length >= opts.maxExpanded) break;
+
+    if (covered.has(i)) continue;
+    const start = Math.max(0, i - 3), end = Math.min(raw.length, i + 4);
+    const block = raw.slice(start, end).map((line, j) => String(start + j + 1).padStart(5) + " " + line).join("\n");
+
+    if (block.length > budget) continue;
+    parts.push(block); budget -= block.length;
+
+    for (let j = start; j < end; j++) covered.add(j);
+  }
+
+  const status = parts.length ? "focused text windows (not a complete file)"
+    : hits.length ? "matching text exceeds view budget; first match at line " + (hits[0].i + 1) : "no matching text";
+
+  return {text: "// " + relPath + " · " + status + "; read(path, line, count) for raw source\n" + parts.join("\n---\n"), expanded: parts.length, declarations: 0};
+}
+
 /**
  * @param entry index entry (text + cached lines/surface)
  * @param about question or symbol; empty ⇒ pure skeleton (every body folded)
@@ -63,18 +101,24 @@ export function outlineFile(entry, relPath, about, options = {}) {
   const { raw, lower } = WorkspaceIndex.linesOf(entry);
   const lineCount = raw.length;
   const spans = WorkspaceIndex.spansOf(entry).map((s) => ({ ...s, signature: raw[s.start - 1].trim() }));
-  if (spans.length === 0) return null; // no structure: caller falls back to plain text
   const stems = [...new Set(tokenizeQuery(about || "").tokens.map(stem))];
+
+  if (spans.length === 0) return about ? focusedText(raw, lower, stems, relPath, opts) : null;
   const expanded = chooseExpanded(spans, lower, stems, raw, opts);
 
   const parts = [];
   const headerEnd = Math.min(spans[0].start - 1, opts.headerLines);
+
   if (headerEnd > 0) {
     const header = raw.slice(0, headerEnd);
+
     if (header.length) parts.push(header.map((l, i) => String(i + 1).padStart(5) + " " + l).join("\n"));
+
     if (spans[0].start - 1 > opts.headerLines) parts.push("      … " + (spans[0].start - 1 - opts.headerLines) + " more header lines");
   }
+
   for (let i = 0; i < spans.length; i++) parts.push(expanded.has(i) ? expandedBlock(spans[i], raw, opts) : foldedLine(spans[i]));
   const title = "// " + relPath + " · " + lineCount + " lines · " + spans.length + " declarations · " + expanded.size + " expanded" + (about ? " for \"" + about + "\"" : "") + " · read(path, line, count) for a folded body";
+
   return { text: title + "\n" + parts.join("\n"), expanded: expanded.size, declarations: spans.length };
 }

package/src/context/repo-index.js

@@ -13,18 +13,27 @@ import { relativeSlash } from "../fs/workspace.js";
 
 // With a working fs.watch the list only refreshes on change; the TTL is the fallback when watching fails.
 const LIST_TTL_MS = 10_000;
+
 const WATCHED_TTL_MS = 5 * 60_000;
+
 const WATCH_DEBOUNCE_MS = 150;
+
 const MAX_INDEXED_FILES = 4000;
+
 const MAX_FILE_BYTES = 512 * 1024;
+
 const BINARY_EXT = new Set([
   ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".tgz", ".tar", ".bz2", ".xz", ".7z",
   ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".mov", ".wav", ".ogg", ".webm", ".wasm", ".class",
   ".jar", ".so", ".dylib", ".dll", ".exe", ".bin", ".o", ".a", ".node", ".lock", ".sqlite", ".sqlite3", ".db",
 ]);
+
 const REGEX_SPECIAL = /[.+^${}()|\\]/g;
+
 const IDENT_TOKEN = /[A-Za-z_$][\w$]*/g;
+
 const EMPTY = Object.freeze([]);
+
 const DEF_PATTERN = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)/;
 
 /** Declared identifier on a line (function/class/const/…), or ""; the same rule snap and grep use. */
@@ -39,13 +48,19 @@ function isTextCandidate(filePath) {
 /** Translate one glob token at index i → [regexSource, nextIndex]. */
 function globToken(glob, i) {
   const ch = glob[i];
+
   if (ch === "*" && glob[i + 1] === "*") {
     const slashAfter = glob[i + 2] === "/";
+
     return [slashAfter ? "(?:.*/)?" : ".*", i + (slashAfter ? 3 : 2)];
   }
+
   if (ch === "*") return ["[^/]*", i + 1];
+
   if (ch === "?") return ["[^/]", i + 1];
+
   if (ch === "{" || ch === "[") return globGroup(glob, i, ch);
+
   return [ch.replace(REGEX_SPECIAL, "\\$&"), i + 1];
 }
 
@@ -53,26 +68,31 @@ function globToken(glob, i) {
 function globGroup(glob, i, open) {
   const close = open === "{" ? "}" : "]";
   const end = glob.indexOf(close, i);
+
   if (end < 0) throw new SyntaxError("unclosed " + open + " in glob");
   const inner = glob.slice(i + 1, end);
   const source = open === "{" ? "(?:" + inner.split(",").map(globBody).join("|") + ")" : "[" + inner + "]";
+
   return [source, end + 1];
 }
 
 function globBody(glob) {
   let source = "";
   let i = 0;
+
   while (i < glob.length) {
     const [piece, next] = globToken(glob, i);
     source += piece;
     i = next;
   }
+
   return source;
 }
 
 /** gitignore-style glob (rg -g) → RegExp over a "/"-separated relative path. No slash ⇒ basename match anywhere. */
 export function globToRegExp(glob) {
   const body = globBody(glob);
+
   return new RegExp(glob.includes("/") ? "^" + body + "$" : "(?:^|/)" + body + "$");
 }
 
@@ -100,9 +120,11 @@ export class WorkspaceIndex {
   watch(root) {
     if (this.watchers.has(root)) return this.watchers.get(root);
     let ok = false;
+
     try {
       let timer = null;
-      const watcher = fs.watch(root, { recursive: true }, () => {
+
+      const watcher = fs.watch(root, { recursive: true, persistent: false }, () => {
         if (timer) return;
         timer = setTimeout(() => {
           timer = null;
@@ -110,39 +132,50 @@ export class WorkspaceIndex {
           this.gitModified.delete(root);
         }, WATCH_DEBOUNCE_MS);
       });
+
       watcher.on("error", () => {
         this.watchers.set(root, false);
         this.lists.clear();
       });
+
       if (isFunction(watcher.unref)) watcher.unref();
       ok = true;
     } catch {
       ok = false;
     }
+
     this.watchers.set(root, ok);
+
     return ok;
   }
 
   /** Paths git reports as modified/added/untracked (fff's git-status boost); one spawn per list refresh. */
   async modifiedFiles(root) {
     const cached = this.gitModified.get(root);
+
     if (cached) return cached;
     const set = new Set();
+
     try {
       const res = await this.runCommand(["git", "status", "--porcelain", "-z", "--untracked-files=all"], { cwd: root, timeoutMs: 5_000 });
+
       if (res.exitCode === 0) {
         for (const row of res.stdout.split("\0")) {
           if (row.length > 3) set.add(row.slice(3));
         }
       }
     } catch {}
+
     this.gitModified.set(root, set);
+
     return set;
   }
 
   mtimeSeconds(filePath) {
     const e = this.entries.get(filePath);
+
     if (e) return e.mtimeMs / 1000;
+
     try {
       return fs.statSync(filePath).mtimeMs / 1000;
     } catch {
@@ -155,16 +188,20 @@ export class WorkspaceIndex {
     const key = root + "\0" + (includeHidden ? "h" : "");
     const cached = this.lists.get(key);
     const ttl = this.watch(root) ? WATCHED_TTL_MS : LIST_TTL_MS;
+
     if (cached && Date.now() - cached.at < ttl) return cached.files;
     const args = ["rg", "--files"];
+
     if (includeHidden) args.push("--hidden");
     args.push("-g", "!.git/**", "-g", "!**/.git/**", root);
     let files = [];
     let error;
     let truncated = false;
     let missing = false;
+
     try {
       const res = await this.runCommand(args, { cwd: root, timeoutMs: 15_000, signal });
+
       if (res.exitCode !== 0 && res.exitCode !== 1) error = res.stderr.trim() || "rg exited with status " + res.exitCode;
       truncated = res.outputTruncated === true;
       const output = truncated && !res.stdout.endsWith("\n") ? res.stdout.slice(0, res.stdout.lastIndexOf("\n") + 1) : res.stdout;
@@ -174,7 +211,9 @@ export class WorkspaceIndex {
       error = err.message;
       missing = !fs.existsSync(root);
     }
+
     this.lists.set(key, { files, at: Date.now(), error, truncated, missing });
+
     return files;
   }
 
@@ -182,24 +221,31 @@ export class WorkspaceIndex {
   entry(filePath) {
     if (!isTextCandidate(filePath)) return null;
     let stat;
+
     try {
       stat = fs.statSync(filePath);
     } catch {
       this.entries.delete(filePath);
+
       return null;
     }
+
     if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null;
     const cached = this.entries.get(filePath);
+
     if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) return cached;
     let text;
+
     try {
       text = fs.readFileSync(filePath, "utf8");
     } catch {
       return null;
     }
+
     if (text.includes("\0")) return null;
     const created = { text, lower: text.toLowerCase(), mtimeMs: stat.mtimeMs, size: stat.size, ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
     this.entries.set(filePath, created);
+
     return created;
   }
 
@@ -214,13 +260,16 @@ export class WorkspaceIndex {
     const lower = [];
     const defNames = [];
     const idents = [];
+
     for (let i = 0; i < raw.length; i++) {
       const trimmed = raw[i].trim();
       lower[i] = trimmed.toLowerCase();
       defNames[i] = DEF_PATTERN.exec(trimmed)?.[2].toLowerCase() ?? "";
       idents[i] = trimmed.match(IDENT_TOKEN) || EMPTY;
     }
+
     entry.lines = { raw, lower, defNames, idents };
+
     return entry.lines;
   }
 
@@ -233,18 +282,23 @@ export class WorkspaceIndex {
     const { items, lineCount } = WorkspaceIndex.surfaceOf(entry);
     const { lower } = WorkspaceIndex.linesOf(entry);
     const spans = [];
+
     for (let i = 0; i < items.length; i++) {
       const start = items[i].line;
       let end = Math.min(i + 1 < items.length ? items[i + 1].line - 1 : lineCount, lineCount);
+
       while (end > start && lower[end - 1] === "") end--;
       spans.push({ start, end, name: items[i].name, kind: items[i].kind, isExport: items[i].isExport === true });
     }
+
     entry.spans = spans;
+
     return spans;
   }
 
   static surfaceOf(entry) {
     if (!entry.surface) entry.surface = extractStructuralSurface(entry.text, entry.ext);
+
     return entry.surface;
   }
 
@@ -256,12 +310,16 @@ export class WorkspaceIndex {
   /** Files whose lowercase text contains any (or every) needle; needles are lowercase. */
   filesContaining(files, needles, anyOf) {
     const hits = [];
+
     for (const filePath of files) {
       const e = this.entry(filePath);
+
       if (!e) continue;
       const found = anyOf ? needles.some((n) => e.lower.includes(n)) : needles.every((n) => e.lower.includes(n));
+
       if (found) hits.push(filePath);
     }
+
     return hits;
   }
 
@@ -269,16 +327,20 @@ export class WorkspaceIndex {
   grepRows(files, regex, root, overlayText = () => undefined) {
     const out = [];
     const nameRegex = new RegExp(regex.source, "i");
+
     for (const filePath of files) {
       const pending = overlayText(filePath);
       const e = pending === undefined ? this.entry(filePath) : WorkspaceIndex.fromText(filePath, pending);
+
       if (!e || !regex.test(e.text)) continue;
       const { raw, defNames } = WorkspaceIndex.linesOf(e);
       const rel = relativeSlash(root, filePath);
+
       for (let i = 0; i < raw.length; i++) {
         if (regex.test(raw[i])) out.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && nameRegex.test(defNames[i]) });
       }
     }
+
     return out;
   }
 }