pi-blackhole 0.2.2 → 0.3.0

package/src/core/format-recall.ts

@@ -1,4 +1,86 @@
-import type { SearchHit } from "./search-entries";
+import type { SearchHit, FileMatch, TouchedFile } from "./search-entries";
+
+// ── Path shortening ───────────────────────────────────────────────────────
+
+const CWD = process.cwd();
+
+/**
+ * Shorten an absolute file path for display:
+ * - If within cwd, return `./relative/path`
+ * - Otherwise, show last 3 path components with `.../` prefix
+ * - Short paths (≤3 components) returned as-is
+ */
+export function shortPath(fullPath: string): string {
+  // Normalize backslashes to forward slashes for cross-platform path handling
+  const normalized = fullPath.replace(/\\/g, "/");
+  const cwdNormalized = CWD.replace(/\\/g, "/");
+  if (normalized.startsWith(cwdNormalized + "/")) {
+    return "." + normalized.slice(cwdNormalized.length);
+  }
+  const parts = normalized.split("/");
+  if (parts.length > 3) {
+    return ".../" + parts.slice(-3).join("/");
+  }
+  return normalized;
+}
+
+// ── File indicator formatting ─────────────────────────────────────────────
+
+/** Render one file indicator line with shortened path. */
+function formatFileMatch(fm: FileMatch, index: number, isQuery: boolean): string {
+  const label = isQuery
+    ? (fm.lineCount === 1 ? "match" : "matches")
+    : (fm.lineCount === 1 ? "line" : "lines");
+  const displayPath = shortPath(fm.path);
+  let line = `  [${fm.toolName}] ${displayPath} — ${fm.lineCount} ${label}    use #${index}:${fm.path}`;
+  if (fm.snippet) {
+    line += `\n    | ${fm.snippet}`;
+  }
+  return line;
+}
+
+// ── Touched file output ───────────────────────────────────────────────────
+
+const TOUCHED_PAGE_SIZE = 5;
+
+/**
+ * Format aggregated "files touched" output.
+ */
+export function formatTouchedOutput(
+  touched: TouchedFile[],
+  page?: number,
+  pageSize?: number,
+): string {
+  if (touched.length === 0) {
+    return "No file operations found in session history.";
+  }
+
+  const ps = pageSize ?? TOUCHED_PAGE_SIZE;
+  const totalPages = Math.ceil(touched.length / ps);
+  const currentPage = Math.max(1, page ?? 1);
+  const start = (currentPage - 1) * ps;
+  const pageFiles = touched.slice(start, start + ps);
+
+  const header = totalPages > 1
+    ? `Page ${currentPage}/${totalPages} (${touched.length} total files)`
+    : `${touched.length} files touched`;
+
+  const lines = pageFiles.map((tf) => {
+    const displayPath = shortPath(tf.path);
+    const indices = tf.entries
+      .map((e) => `#${e.index} (${e.toolName})`)
+      .join(", ");
+    return `  ${displayPath}    ${indices}`;
+  });
+
+  let result = `${header}:\n\n${lines.join("\n")}`;
+
+  if (currentPage < totalPages) {
+    result += `\n\n--- Use page:${currentPage + 1} for more results ---`;
+  }
+
+  return result;
+}
 
 export const formatRecallOutput = (
   entries: SearchHit[],
@@ -18,9 +100,31 @@ export const formatRecallOutput = (
       : `Session history (${entries.length} entries):`;
 
   const lines = entries.map((e) => {
-    const fileSuffix = e.files?.length ? ` files:[${e.files.join(", ")}]` : "";
     const body = query && e.snippet ? e.snippet : e.summary;
-    return `#${e.index} [${e.role}]${fileSuffix} ${body}`;
+    let line = `#${e.index} [${e.role}]`;
+
+    // Entry text on next line if it's long or has fileMatches
+    if (e.fileMatches?.length) {
+      line += `\n  ${body}`;
+      // Only show top 3 file matches
+      const isQuery = Boolean(query);
+      const topFileMatches = e.fileMatches.slice(0, 3);
+      for (const fm of topFileMatches) {
+        line += `\n${formatFileMatch(fm, e.index, isQuery)}`;
+      }
+      if (e.fileMatches.length > 3) {
+        line += `\n  ...(${e.fileMatches.length - 3} more file matches)`;
+      }
+    } else if (e.files?.length) {
+      // Fallback: entries without fileMatches (e.g. expand-only path) still have
+      // the legacy .files array from renderMessage — show it in old format.
+      const fileSuffix = ` files:[${e.files.join(", ")}]`;
+      line += `${fileSuffix} ${body}`;
+    } else {
+      line += ` ${body}`;
+    }
+
+    return line;
   });
 
   return `${header}\n\n${lines.join("\n\n")}`;

package/src/core/load-messages.ts

@@ -34,9 +34,10 @@ export const loadAllMessages = (
 
     const allowed = !allowedEntryIds || allowedEntryIds.has(e.id);
     if (allowed) {
-      rendered.push(renderMessage(e.message, messageIndex, String(e.id), full));
+      const entryId = e.id != null ? String(e.id) : "";
+      rendered.push(renderMessage(e.message, messageIndex, entryId, full));
       rawMessages.push(e.message);
-      entryIds.push(String(e.id));
+      entryIds.push(entryId);
     }
     messageIndex++;
   }

package/src/core/recall-scope.ts

@@ -1,14 +1,25 @@
 export type RecallScope = "lineage" | "all";
+export type RecallMode = "hybrid" | "file" | "transcript" | "touched";
 
 const SCOPE_RE = /\bscope:(lineage|all)\b/i;
+const MODE_RE = /\bmode:(hybrid|file|transcript|touched)\b/i;
+
+const VALID_MODES = new Set(["hybrid", "file", "transcript", "touched"]);
 
 export const normalizeRecallScope = (scope?: unknown): RecallScope =>
   typeof scope === "string" && scope.toLowerCase() === "all" ? "all" : "lineage";
 
-export const parseRecallScope = (text: string): { scope: RecallScope; text: string } => {
-  const match = text.match(SCOPE_RE);
+export const normalizeRecallMode = (mode?: unknown): RecallMode =>
+  typeof mode === "string" && VALID_MODES.has(mode.toLowerCase())
+    ? (mode.toLowerCase() as RecallMode)
+    : "hybrid";
+
+export const parseRecallScope = (text: string): { scope: RecallScope; mode: RecallMode; text: string } => {
+  const scopeMatch = text.match(SCOPE_RE);
+  const modeMatch = text.match(MODE_RE);
   return {
-    scope: normalizeRecallScope(match?.[1]),
-    text: text.replace(SCOPE_RE, "").replace(/\s+/g, " ").trim(),
+    scope: normalizeRecallScope(scopeMatch?.[1]),
+    mode: normalizeRecallMode(modeMatch?.[1]),
+    text: text.replace(SCOPE_RE, "").replace(MODE_RE, "").replace(/\s+/g, " ").trim(),
   };
 };

package/src/core/search-entries.ts

@@ -6,13 +6,39 @@
  */
 import type { Message } from "@earendil-works/pi-ai";
 import type { RenderedEntry } from "./render-entries";
-import { textOf } from "./content";
+import { textOf, toolCallArgsText, isContentBearing } from "./content";
+import type { RecallMode } from "./recall-scope";
+
+export interface FileMatch {
+  /** Name of the tool (write, edit, hex_edit) */
+  toolName: string;
+  /** File path from the tool call arguments */
+  path: string;
+  /** Number of lines in the content that matched the query */
+  lineCount: number;
+  /** First matching line snippet (only populated for top matches) */
+  snippet?: string;
+}
+
+/** A file touched in one entry — used by mode:touched aggregation. */
+export interface FileTouch {
+  index: number;
+  toolName: string;
+}
+
+/** Aggregated view of a file touched across multiple entries. */
+export interface TouchedFile {
+  path: string;
+  entries: FileTouch[];
+}
 
 export interface SearchHit extends RenderedEntry {
   /** Context snippet around the first matched term (only when query provided) */
   snippet?: string;
   /** Number of query terms matched (for ranking) */
   matchCount?: number;
+  /** Per-file match indicators from content-bearing tool calls */
+  fileMatches?: FileMatch[];
 }
 
 const escapeRegex = (s: string): string =>
@@ -153,18 +179,131 @@ const lineSnippet = (text: string, regex: RegExp, contextLines = 2): string | un
   return parts.join("\n");
 };
 
-/** Build full searchable text for a message. */
-const fullText = (msg: Message): string => {
+/** Build full searchable text for a message, optionally filtered by mode. */
+const fullText = (msg: Message, mode?: RecallMode): string => {
   if ((msg as any).role === "bashExecution") {
+    if (mode === "file") return ""; // bash is not file content
     return `${(msg as any).command ?? ""} ${(msg as any).output ?? ""}`;
   }
-  return textOf(msg.content);
+  if (mode === "file") {
+    return toolCallArgsText(msg.content);
+  }
+  if (mode === "transcript") {
+    return textOf(msg.content);
+  }
+  // hybrid (default): both
+  const text = textOf(msg.content);
+  const toolArgs = toolCallArgsText(msg.content);
+  return toolArgs ? `${text}\n${toolArgs}` : text;
 };
 
+/**
+ * Extract searchable text from tool call arguments (content, edits, oldText, newText).
+ */
+function extractToolCallText(args: Record<string, unknown>): string {
+  let text = "";
+  if (typeof args.content === "string") text += args.content + "\n";
+  if (Array.isArray(args.edits)) {
+    for (const edit of args.edits) {
+      if (edit && typeof edit === "object") {
+        if (typeof edit.oldText === "string") text += edit.oldText + "\n";
+        if (typeof edit.newText === "string") text += edit.newText + "\n";
+      }
+    }
+  }
+  if (typeof args.oldText === "string" && !Array.isArray(args.edits)) text += args.oldText + "\n";
+  if (typeof args.newText === "string" && !Array.isArray(args.edits)) text += args.newText + "\n";
+  return text;
+}
+
+/**
+ * Compute file indicators from a message (no query — counts total lines per file).
+ */
+export function getFileIndicators(msg: Message): FileMatch[] {
+  if (!msg?.content || typeof msg.content === "string") return [];
+  const fileMatches: FileMatch[] = [];
+  for (const part of msg.content) {
+    if (!part || typeof part !== "object" || part.type !== "toolCall") continue;
+    const args = part.arguments as Record<string, unknown>;
+    if (!isContentBearing(args)) continue;
+
+    const path = ["path", "filePath", "file_path", "file"]
+      .map((k) => args[k])
+      .find((v): v is string => typeof v === "string")!;
+
+    const totalText = extractToolCallText(args);
+    const nonEmpty = totalText.split("\n").filter((l) => l.trim().length > 0);
+    fileMatches.push({
+      toolName: part.name || "",
+      path,
+      lineCount: nonEmpty.length,
+    });
+  }
+  return fileMatches;
+}
+
+function computeFileMatches(msg: Message | undefined, query: string): FileMatch[] {
+  if (!msg?.content || typeof msg.content === "string") return [];
+  const rawQuery = query.trim();
+  const hasQuery = rawQuery.length > 0;
+  if (!hasQuery) return getFileIndicators(msg as Message);
+  const regex = looksLikeRegex(rawQuery) ? safeRegex(rawQuery) : snippetRegex(rawQuery.split(/\s+/));
+  const fileMatches: FileMatch[] = [];
+
+  for (const part of msg.content) {
+    if (!part || typeof part !== "object" || part.type !== "toolCall") continue;
+    const args = part.arguments as Record<string, unknown>;
+    if (!isContentBearing(args)) continue;
+
+    // Path is guaranteed by isContentBearing check
+    const path = ["path", "filePath", "file_path", "file"]
+      .map((k) => args[k])
+      .find((v): v is string => typeof v === "string")!;
+
+    const searchText = extractToolCallText(args);
+    if (!searchText) continue;
+
+    const lines = searchText.split("\n");
+    const matchingLines = lines.filter((line) => regex.test(line));
+    if (matchingLines.length > 0) {
+      fileMatches.push({
+        toolName: part.name || "",
+        path,
+        lineCount: matchingLines.length,
+        snippet: matchingLines[0],
+      });
+    }
+  }
+
+  return fileMatches;
+}
+
+/** Aggregate file operations across all entries for mode:touched. */
+export function getTouchedFiles(
+  messages: Message[],
+  rendered: RenderedEntry[],
+): TouchedFile[] {
+  const map = new Map<string, TouchedFile>();
+  for (let i = 0; i < messages.length; i++) {
+    const msg = messages[i];
+    const indicators = getFileIndicators(msg);
+    for (const fm of indicators) {
+      const index = rendered[i]?.index ?? i;
+      if (!map.has(fm.path)) {
+        map.set(fm.path, { path: fm.path, entries: [] });
+      }
+      map.get(fm.path)!.entries.push({ index, toolName: fm.toolName });
+    }
+  }
+  return Array.from(map.values());
+}
+
 export const searchEntries = (
   entries: RenderedEntry[],
   messages: Message[],
   query?: string,
+  _page?: number,
+  mode?: RecallMode,
 ): SearchHit[] => {
   if (!query?.trim()) return entries;
 
@@ -178,12 +317,14 @@ export const searchEntries = (
     for (let i = 0; i < entries.length; i++) {
       const e = entries[i];
       const msg = messages[i];
-      const text = msg ? fullText(msg) : e.summary;
+      const text = msg ? fullText(msg, mode) : e.summary;
       const filePart = e.files?.join(" ") ?? "";
       const hay = `${e.role} ${text} ${filePart}`;
       if (regex.test(hay)) {
         const snip = lineSnippet(text, regex);
-        hits.push({ ...e, snippet: snip, matchCount: 1 });
+        const fileMatches = mode === "transcript" ? [] : computeFileMatches(msg, rawQuery);
+        const extra = fileMatches.length > 0 ? { fileMatches } : {};
+        hits.push({ ...e, snippet: snip, matchCount: 1, ...extra });
       }
     }
     return hits;
@@ -194,12 +335,14 @@ export const searchEntries = (
   const terms = filterStopwords(rawTerms);
   const snipRe = snippetRegex(terms);
 
-  // Build all docs for BM25 context
+  // Build all docs for BM25 context (cache fullText to avoid recomputing)
   const docs: string[] = [];
+  const fullTextCache: string[] = [];
   for (let i = 0; i < entries.length; i++) {
     const e = entries[i];
     const msg = messages[i];
-    const text = msg ? fullText(msg) : e.summary;
+    const text = msg ? fullText(msg, mode) : e.summary;
+    fullTextCache.push(text);
     const filePart = e.files?.join(" ") ?? "";
     docs.push(`${e.role} ${text} ${filePart}`);
   }
@@ -213,10 +356,12 @@ export const searchEntries = (
     const mc = countMatches(hay, terms);
     if (mc === 0) continue;
     const score = bm25Score(hay, terms, ctx);
-    const text = messages[i] ? fullText(messages[i]) : e.summary;
+    const text = fullTextCache[i];
     const snip = lineSnippet(text, snipRe);
+    const fileMatches = mode === "transcript" ? [] : computeFileMatches(messages[i], rawQuery);
+    const extra = fileMatches.length > 0 ? { fileMatches } : {};
     scored.push({
-      hit: { ...e, snippet: snip, matchCount: mc },
+      hit: { ...e, snippet: snip, matchCount: mc, ...extra },
       score,
     });
   }

package/src/core/unified-config.ts

@@ -44,6 +44,16 @@ export interface UnifiedConfig {
 	compactAfterTokens: number;
 	/** Observation pool token pressure for full fold. */
 	observationsPoolMaxTokens: number;
+	/** Target token budget for the observation pool (dropper aims here).
+	 *  Optional; defaults to half of observationsPoolMaxTokens when unset.
+	 *  Must be less than observationsPoolMaxTokens.
+	 *
+	 *  NOTE: Ported from upstream as forward-compat (no-op in our pool algorithm).
+	 *  Upstream renamed budgetTokens→targetTokens (52b5844) and uses this
+	 *  for their tokensOverTarget / avgTokensPerObservation drop calculation.
+	 *  We keep our ratio-based urgency algorithm; this knob exists so future
+	 *  lockstep iterations don't diverge on the config shape. */
+	observationsPoolTargetTokens: number;
 	/** Max prompt tokens for reflector model input (rolling window cap). */
 	reflectorInputMaxTokens: number;
 	/** Max prompt tokens for dropper model input (rolling window cap). */
@@ -96,6 +106,7 @@ export const DEFAULTS: UnifiedConfig = {
 	reflectAfterTokens: 25_000,
 	compactAfterTokens: 81_000,
 	observationsPoolMaxTokens: 20_000,
+	observationsPoolTargetTokens: 10_000,
 	reflectorInputMaxTokens: 80_000,
 	dropperInputMaxTokens: 80_000,
 	observerChunkMaxTokens: 40_000,
@@ -160,7 +171,7 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
 	if (typeof raw.debugLog === "boolean") c.debugLog = raw.debugLog;
 
 	// Positive integers
-	const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "observerPreambleMaxTokens", "agentMaxTurns"] as const;
+	const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "observationsPoolTargetTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "observerPreambleMaxTokens", "agentMaxTurns"] as const;
 	for (const k of numKeys) {
 		const v = positiveInt(raw[k]);
 		if (v !== undefined) (c as Record<string, unknown>)[k] = v;
@@ -238,7 +249,37 @@ export function loadUnifiedConfig(cwd: string): UnifiedConfig {
 		else if (["0", "false", "no", "off"].includes(v)) parsed.passive = false;
 	}
 
-	return { ...DEFAULTS, ...parsed };
+	// Merge defaults then override
+	const merged = { ...DEFAULTS, ...parsed };
+
+	// ── Validate all numeric fields ──
+	// Prevents NaN/undefined from leaking into runtime math.
+	const NUMERIC_KEYS: ReadonlyArray<keyof UnifiedConfig> = [
+		"observeAfterTokens", "reflectAfterTokens", "compactAfterTokens",
+		"observationsPoolMaxTokens", "observationsPoolTargetTokens",
+		"reflectorInputMaxTokens", "dropperInputMaxTokens",
+		"observerChunkMaxTokens", "observerPreambleMaxTokens",
+		"agentMaxTurns",
+	];
+	for (const k of NUMERIC_KEYS) {
+		const v = (merged as Record<string, unknown>)[k];
+		// observerPreambleMaxTokens=0 means "auto-compute from observerChunkMaxTokens (30%)"
+		// so 0 is valid for that field. All other numeric fields must be strictly positive.
+		const minVal = k === "observerPreambleMaxTokens" ? 0 : 1;
+		if (typeof v !== "number" || !Number.isFinite(v) || v < minVal) {
+			(merged as Record<string, unknown>)[k] = DEFAULTS[k];
+		}
+	}
+
+	// Derive observationsPoolTargetTokens if still unset or invalid (must be < max)
+	if (
+		merged.observationsPoolTargetTokens === undefined ||
+		merged.observationsPoolTargetTokens >= merged.observationsPoolMaxTokens
+	) {
+		merged.observationsPoolTargetTokens = Math.floor(merged.observationsPoolMaxTokens / 2);
+	}
+
+	return merged;
 }
 
 /**
@@ -260,7 +301,11 @@ export function saveUnifiedConfig(settings: Partial<UnifiedConfig>): boolean {
 
 /**
  * Ensure ~/.pi/agent/pi-blackhole/pi-blackhole-config.json exists with defaults.
- * If the file exists but is missing keys, fill them in.
+ *
+ * Only creates the file if it doesn't exist. Missing keys are filled at read
+ * time by loadUnifiedConfig() via { ...DEFAULTS, ...parsed } merge, so there
+ * is no need to keep the on-disk file "complete". This avoids a crash on
+ * read-only filesystems where the config is managed externally (e.g., Nix).
  */
 export function scaffoldConfig(): void {
 	try {
@@ -270,21 +315,7 @@ export function scaffoldConfig(): void {
 
 		if (!existsSync(path)) {
 			writeFileSync(path, `${JSON.stringify(DEFAULTS, null, 2)}\n`);
-			return;
-		}
-
-		const parsed = readJson(path);
-		if (!parsed || typeof parsed !== "object") return;
-
-		let changed = false;
-		const next: Record<string, unknown> = { ...parsed };
-		for (const [key, value] of Object.entries(DEFAULTS)) {
-			if (!(key in next)) {
-				next[key] = value;
-				changed = true;
-			}
 		}
-		if (changed) writeFileSync(path, `${JSON.stringify(next, null, 2)}\n`);
 	} catch (e) {
 		console.error("blackhole: config scaffold failed", e);
 	}