pi-blackhole 0.2.3 → 0.3.0

package/README.md

@@ -165,13 +165,13 @@ Set `memory: false` or run `/blackhole om-off` for pure pi-vcc compaction — no
 | `/blackhole-memory` | Pipeline status: token progress, worker counts, last errors |
 | `/blackhole-memory view` | Visible observations and reflections, copied to clipboard |
 | `/blackhole-memory full` | Complete recorded memory, copied to clipboard |
-| `/blackhole-recall <query>` | Search session history. Supports `page:N`, `scope:all` |
+| `/blackhole-recall <query>` | Search session history. Supports `page:N`, `scope:all`, `mode:file|transcript|touched` |
 
 ## Tools
 
 | Tool | Input | Returns |
 |---|---|---|
-| `recall` | `[12char hex]` — source evidence for an observation/reflection; `#N` — transcript entry by index; free text — BM25+regex ranked search | Source message(s) with timestamps |
+| `recall` | `[12char hex]` — source evidence; `#N` — expand entry; `#N:path` — drill-down file content; free text — BM25+regex ranked search; `mode:file|transcript|touched` — filter or aggregate | Source message(s) with timestamps; file content on drill-down; aggregated file list on touched mode |
 
 ---
 
@@ -317,7 +317,13 @@ Searches the raw session JSONL directly, bypassing compaction. Default scope is
 |---|---|
 | `[12char hex]` | Recover source evidence for an observation or reflection |
 | `#N` | Expand a transcript entry by index |
+| `#N:path` | Drill-down into file content from a tool call (e.g. `#42:auth.ts`) |
+| `#N:path:full` | Show all lines of a file (vs. default preview of first 30) |
+| `#N:path:offset:limit` | Paginated file content (e.g. `#42:auth.ts:30` = next 30 lines) |
 | Free text | BM25-ranked OR search, rare terms weighted higher |
+| `mode:file` | Search only write/edit file content (not transcript text) |
+| `mode:transcript` | Search only conversation text (not file content) |
+| `mode:touched` | Aggregate view of all files written/edited, grouped by path with entry indices |
 | Regex | Pattern search (e.g. `fork.*pi-vcc`, `hook\|inject`) |
 
 ### `/blackhole-recall` command (user-facing)
@@ -325,11 +331,14 @@ Searches the raw session JSONL directly, bypassing compaction. Default scope is
 Results are shown as a collapsible message and auto-fed to the agent as context.
 
 ```
-/blackhole-recall auth token              # active-lineage search, ranked
-/blackhole-recall auth token page:2       # paginated (5 results/page)
-/blackhole-recall hook|inject             # regex
-/blackhole-recall fail.*build scope:all   # regex across all lineages
-/blackhole-recall                         # recent 25 entries
+/blackhole-recall auth token                    # active-lineage search, ranked
+/blackhole-recall auth token page:2             # paginated (5 results/page)
+/blackhole-recall hook|inject                   # regex
+/blackhole-recall fail.*build scope:all         # regex across all lineages
+/blackhole-recall mode:file                     # search only write/edit file content
+/blackhole-recall mode:transcript               # search only conversation
+/blackhole-recall mode:touched                  # aggregate view of all files touched
+/blackhole-recall                               # recent 25 entries
 ```
 
 ---

package/index.ts

@@ -18,6 +18,51 @@ import { registerRecallTool } from "./src/tools/recall";
 import { Runtime } from "./src/om/runtime.js";
 
 export default (pi: ExtensionAPI) => {
+	// ── Bridge: capture custom provider stream functions for jiti-loaded agents ──
+	// pi-blackhole's consolidation agents are loaded via jiti with moduleCache: false,
+	// which creates a separate pi-ai instance whose apiProviderRegistry lacks custom
+	// providers (e.g., claude-bridge registered by other extensions). This bridge stores
+	// streamSimple functions in a Symbol.for() global so agents can access them without
+	// going through pi-ai's registry.
+	//
+	// There are two mechanisms:
+	// 1. Wrap pi.registerProvider to capture streamSimple at registration time.
+	//    This handles providers registered AFTER our factory runs.
+	// 2. On agent_start, scan modelRegistry.registeredProviders for any providers
+	//    that registered BEFORE our factory ran (different extension load order).
+	const PROVIDER_STREAMS_KEY = Symbol.for("pi-blackhole:provider-streams");
+	const providerStreams: Map<string, Function> = (globalThis as any)[PROVIDER_STREAMS_KEY] ??= new Map();
+
+	const origRegisterProvider = pi.registerProvider.bind(pi);
+	pi.registerProvider = (name: string, config: any) => {
+		if (config && config.streamSimple && config.api) {
+			providerStreams.set(config.api, config.streamSimple);
+		}
+		origRegisterProvider(name, config);
+	};
+
+	// Fallback: on agent_start, capture providers that registered before our wrapper
+	// (handles the case where pi-blackhole loads after another provider extension).
+	// Uses a dedicated flag instead of checking providerStreams.size so the scan
+	// always runs once even if the wrapper already captured some providers.
+	let hasScannedFallback = false;
+	pi.on("agent_start", (_event: unknown, ctx: any) => {
+		if (hasScannedFallback) return;
+		hasScannedFallback = true
+		// modelRegistry.registeredProviders is declared private in TypeScript but is a
+		// regular JS class field at runtime. We access it via bracket notation for
+		// future-proofing against potential #private migration.
+		const registry = (ctx as any)?.modelRegistry;
+		const registered: Map<string, any> | undefined = registry?.["registeredProviders"];
+		if (registered && typeof registered.forEach === "function") {
+			registered.forEach((config: any, _name: string) => {
+				if (config && config.streamSimple && config.api && !providerStreams.has(config.api)) {
+					providerStreams.set(config.api, config.streamSimple);
+				}
+			});
+		}
+	});
+
 	scaffoldSettings();
 
 	const omRuntime = new Runtime();

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "pi-blackhole",
-  "version": "0.2.3",
+  "version": "0.3.0",
+  "packageManager": "pnpm@11.2.2",
   "description": "Unified compaction + observational memory extension for Pi — compresses conversation context while preserving durable observations and reflections",
   "license": "MIT",
   "main": "index.ts",
@@ -14,7 +15,8 @@
     "example-config.json"
   ],
   "scripts": {
-    "check": "tsc --noEmit"
+    "check": "tsc --noEmit",
+    "lint": "eslint src/"
   },
   "keywords": [
     "pi-package",
@@ -53,8 +55,13 @@
     "@earendil-works/pi-ai": "0.75.4",
     "@earendil-works/pi-coding-agent": "0.75.4",
     "@earendil-works/pi-tui": "0.75.5",
-    "typebox": "^1.1.38",
-    "typescript": "^5.7.3",
-    "vitest": "^4.1.7"
+    "@typescript-eslint/eslint-plugin": "8.60.0",
+    "@typescript-eslint/parser": "8.60.0",
+    "eslint": "9.39.4",
+    "husky": "9.1.7",
+    "lint-staged": "15.5.2",
+    "typebox": "1.1.38",
+    "typescript": "5.9.3",
+    "vitest": "4.1.7"
   }
 }

package/src/commands/pi-vcc.ts

@@ -35,19 +35,29 @@ export const registerPiVccCommand = (pi: ExtensionAPI, runtime: Runtime) => {
 			if (trimmed === "om-off") {
 				const saved = saveUnifiedConfig({ memory: false });
 				runtime.config.memory = false;
-				ctx.ui.notify(
-					saved ? "Observational memory disabled. Use /blackhole om-on to re-enable." : "Failed to save config.",
-					"info",
-				);
+				if (saved) {
+					ctx.ui.notify("Observational memory disabled. Use /blackhole om-on to re-enable.", "info");
+				} else {
+					ctx.ui.notify(
+						"Failed to save config — the config file may be read-only (e.g., managed by Nix). " +
+						"Runtime state updated for this session only.",
+						"warning",
+					);
+				}
 				return;
 			}
 			if (trimmed === "om-on") {
 				const saved = saveUnifiedConfig({ memory: true });
 				runtime.config.memory = true;
-				ctx.ui.notify(
-					saved ? "Observational memory enabled." : "Failed to save config.",
-					"info",
-				);
+				if (saved) {
+					ctx.ui.notify("Observational memory enabled.", "info");
+				} else {
+					ctx.ui.notify(
+						"Failed to save config — the config file may be read-only (e.g., managed by Nix). " +
+						"Runtime state updated for this session only.",
+						"warning",
+					);
+				}
 				return;
 			}
 

package/src/commands/vcc-recall.ts

@@ -41,7 +41,7 @@ async function augmentWithObservations(
 export const registerVccRecallCommand = (pi: ExtensionAPI) => {
 	pi.registerCommand("blackhole-recall", {
 		description:
-			"Search session history. Defaults to active lineage; add scope:all for off-lineage branches. Usage: /blackhole-recall <query> [page:N] [scope:all]",
+			"Search session history. Defaults to active lineage. Usage: /blackhole-recall <query> [page:N] [scope:all] [mode:file|transcript]",
 		handler: async (args: string, ctx) => {
 			const sessionFile = ctx.sessionManager.getSessionFile();
 			if (!sessionFile) {
@@ -55,6 +55,7 @@ export const registerVccRecallCommand = (pi: ExtensionAPI) => {
 				parsed.scope === "lineage"
 					? getActiveLineageEntryIds(ctx.sessionManager)
 					: undefined;
+			const mode = parsed.mode;
 
 			if (!parsed.text) {
 				// No query: show recent entries
@@ -87,7 +88,7 @@ export const registerVccRecallCommand = (pi: ExtensionAPI) => {
 			}
 
 			const { rendered, rawMessages } = loadAllMessages(sessionFile, false, lineageEntryIds);
-			const allResults = searchEntries(rendered, rawMessages, query);
+			const allResults = searchEntries(rendered, rawMessages, query, undefined, mode);
 
 			const start = (page - 1) * PAGE_SIZE;
 			const pageResults = allResults.slice(start, start + PAGE_SIZE);

package/src/core/build-sections.ts

@@ -5,7 +5,7 @@
  * Unmodified.
  */
 import type { NormalizedBlock } from "../types";
-import { clip, clipSentence, firstLine, nonEmptyLines } from "./content";
+import { clipSentence, firstLine, nonEmptyLines } from "./content";
 import type { SectionData } from "../sections";
 import { extractGoals } from "../extract/goals";
 import { extractFiles } from "../extract/files";

package/src/core/content.ts

@@ -48,6 +48,76 @@ export const textParts = (content: Message["content"]): string[] => {
 export const textOf = (content: Message["content"]): string =>
   textParts(content).join("\n");
 
+/**
+ * Check if tool call arguments contain content-bearing data.
+ *
+ * A call is content-bearing if it has a path argument AND at least one
+ * large string/array field (content, edits, oldText, newText).
+ * This is a generic heuristic — not dependent on tool names.
+ */
+export const isContentBearing = (args: Record<string, unknown>): boolean => {
+  if (!args || typeof args !== "object") return false;
+  // Must have a path in one of the known keys
+  const hasPath = ["path", "filePath", "file_path", "file"].some((k) => typeof args[k] === "string");
+  if (!hasPath) return false;
+  // Must have at least one content-bearing field
+  if (typeof args.content === "string" && args.content.length > 0) return true;
+  if (Array.isArray(args.edits) && args.edits.length > 0) return true;
+  if (typeof args.oldText === "string" && args.oldText.length > 0 && !Array.isArray(args.edits)) return true;
+  if (typeof args.newText === "string" && args.newText.length > 0 && !Array.isArray(args.edits)) return true;
+  return false;
+};
+
+/**
+ * Extract textual content from tool call arguments (write, edit, hex_edit).
+ *
+ * Looks for content-bearing tool calls (those with a `path` argument and
+ * at least one large string/array field like `content`, `edits`, `oldText`, `newText`).
+ * Each call is capped at `maxBytesPerCall` to avoid inflating the search index.
+ */
+export const toolCallArgsText = (
+  content: Message["content"],
+  maxBytesPerCall = 10_240,
+): string => {
+  if (!content || typeof content === "string") return "";
+  const parts: string[] = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object" || part.type !== "toolCall") continue;
+    const args = part.arguments as Record<string, unknown>;
+    if (!isContentBearing(args)) continue;
+
+    let extracted = "";
+    if (typeof args.content === "string") {
+      extracted += args.content.slice(0, maxBytesPerCall) + "\n";
+    }
+    if (Array.isArray(args.edits)) {
+      for (const edit of args.edits) {
+        if (extracted.length >= maxBytesPerCall) break;
+        if (edit && typeof edit === "object") {
+          if (typeof edit.oldText === "string") {
+            extracted += edit.oldText.slice(0, Math.floor(maxBytesPerCall / 2)) + "\n";
+          }
+          if (extracted.length >= maxBytesPerCall) break;
+          if (typeof edit.newText === "string") {
+            extracted += edit.newText.slice(0, Math.floor(maxBytesPerCall / 2)) + "\n";
+          }
+        }
+      }
+    }
+    if (typeof args.oldText === "string" && !Array.isArray(args.edits)) {
+      extracted += args.oldText.slice(0, maxBytesPerCall) + "\n";
+    }
+    if (typeof args.newText === "string" && !Array.isArray(args.edits)) {
+      extracted += args.newText.slice(0, maxBytesPerCall) + "\n";
+    }
+
+    if (extracted) {
+      parts.push(extracted.slice(0, maxBytesPerCall));
+    }
+  }
+  return parts.join("\n");
+};
+
 /** Extract a snippet of ~`radius` chars around the first match of `term` in `text`. */
 export const snippet = (text: string, term: string, radius = 60): string | null => {
   const idx = text.toLowerCase().indexOf(term.toLowerCase());

package/src/core/drill-down.ts

@@ -0,0 +1,273 @@
+/**
+ * Drill-down: resolve #N:path syntax to tool call file content.
+ *
+ * Phase 3 of recall-progressive-discovery.
+ * Supports: #42:auth.ts (preview), #42:auth.ts:full (full content), #42:file (auto-select).
+ *
+ * Path matching notes:
+ * - The regex uses $ anchor so lazy .+? consumes the full path — spaces, dots, and
+ *   Windows drive-letter colons (C:\) all work correctly.
+ * - One edge case: a file path literally ending in ":full" (e.g. C:\file:full)
+ *   would be misinterpreted as the :full flag. This is extremely unlikely.
+ * - Inline queries like "check #42:auth.ts" are NOT drill-down — the ^ anchor
+ *   requires the entire query to be the drill-down pattern.
+ */
+import { isContentBearing } from "./content.js";
+import { loadAllMessages } from "./load-messages.js";
+
+// ── Types ─────────────────────────────────────────────────────────────────
+
+interface ContentBearingCall {
+  name: string;
+  path: string;
+  content?: string;
+  oldText?: string;
+  newText?: string;
+  edits?: Array<{ oldText?: string; newText?: string }>;
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────
+
+/** Extract the file path from tool call args, checking all known keys. */
+function extractPathFromArgs(args: Record<string, unknown>): string | null {
+  for (const key of ["path", "filePath", "file_path", "file"]) {
+    if (typeof args[key] === "string") return args[key] as string;
+  }
+  return null;
+}
+
+/**
+ * Find content-bearing tool calls that have a `path` argument and at least
+ * one content field (content, edits, oldText, newText).
+ * Uses the shared isContentBearing() heuristic from content.ts.
+ */
+function findContentBearingCalls(content: unknown[]): ContentBearingCall[] {
+  if (!Array.isArray(content)) return [];
+  const results: ContentBearingCall[] = [];
+  for (const part of content) {
+    if (!part || (part as any).type !== "toolCall") continue;
+    const args = (part as any).arguments ?? {};
+    if (!isContentBearing(args)) continue;
+    const path = extractPathFromArgs(args);
+    if (!path) continue;
+    const entry: ContentBearingCall = { name: (part as any).name ?? "", path };
+    if (typeof args.content === "string") entry.content = args.content;
+    if (Array.isArray(args.edits)) {
+      entry.edits = args.edits.filter((e: unknown): e is { oldText?: string; newText?: string } => e !== null && typeof e === "object");
+    }
+    if (typeof args.oldText === "string" && !Array.isArray(args.edits)) entry.oldText = args.oldText;
+    if (typeof args.newText === "string" && !Array.isArray(args.edits)) entry.newText = args.newText;
+    results.push(entry);
+  }
+  return results;
+}
+
+/** Format content for display with optional offset/limit slicing.
+ *
+ * When full=true, shows everything ignoring offset/limit.
+ * When offset/limit given, shows a window with "Lines X-Y (of Z)" header.
+ * When neither, shows preview (first 30 lines) with truncation hint.
+ */
+function formatToolCallContent(
+  tc: ContentBearingCall,
+  entryIndex: number,
+  options?: { full?: boolean; offset?: number; limit?: number },
+): string {
+  let body: string;
+  if (tc.content) {
+    body = tc.content;
+  } else if (tc.edits) {
+    body = tc.edits
+      .map((e, i) => `--- edit ${i + 1} ---\n${e.oldText ?? ""}\n--- becomes ---\n${e.newText ?? ""}`)
+      .join("\n\n");
+  } else if (tc.oldText && tc.newText) {
+    body = `--- old ---\n${tc.oldText}\n--- new ---\n${tc.newText}`;
+  } else {
+    body = "(no file content found in tool call arguments)";
+  }
+
+  const full = options?.full ?? false;
+  const offset = options?.offset;
+  const limit = options?.limit;
+  const allLines = body.split("\n");
+  const totalLines = allLines.length;
+  const previewLimit = 30;
+  const MAX_FULL_BYTES = 50 * 1024;
+
+  if (full) {
+    // Full content: capped at 50KB
+    if (Buffer.byteLength(body, "utf8") > MAX_FULL_BYTES) {
+      const truncated = body.slice(0, MAX_FULL_BYTES);
+      return `File: ${tc.path}
+Tool: ${tc.name}
+
+${truncated}
+
+... (${Buffer.byteLength(body, "utf8") - MAX_FULL_BYTES} more bytes — file exceeds 50KB display limit. Use #${entryIndex}:${tc.path}:${previewLimit} for next page.)`;
+    }
+    return `File: ${tc.path}
+Tool: ${tc.name}
+
+${body}`;
+  }
+
+  if (offset !== undefined) {
+    // Offset-based window: show slice
+    const startLine = Math.max(0, offset);
+    const maxLines = limit ?? 30;
+    const endLine = Math.min(startLine + maxLines, totalLines);
+    const visible = allLines.slice(startLine, endLine);
+    const displayStart = startLine + 1; // 1-indexed for user display
+
+    if (visible.length === 0) {
+      return `Offset ${startLine} is beyond file length ${totalLines}. Use #${entryIndex}:${tc.path} for the first ${previewLimit} lines.`;
+    }
+
+    let result = `File: ${tc.path}
+Tool: ${tc.name}
+Lines ${displayStart}-${endLine} (of ${totalLines}):
+
+`;
+    result += visible.join("\n");
+
+    if (endLine < totalLines) {
+      result += `\n\n--- Use #${entryIndex}:${tc.path}:${endLine} or #${entryIndex}:${tc.path}:${endLine}:${maxLines} for next ${maxLines} lines, #${entryIndex}:${tc.path}:full for complete ---`;
+    } else if (offset > 0) {
+      result += `\n\n(End of file)`;
+    }
+
+    return result;
+  }
+
+  // Default preview mode: first ${previewLimit} lines
+  if (totalLines > previewLimit) {
+    const preview = allLines.slice(0, previewLimit).join("\n");
+    return `File: ${tc.path}
+Tool: ${tc.name}
+
+${preview}
+
+...(${totalLines - previewLimit} more lines — use #${entryIndex}:${tc.path}:full for complete content, or #${entryIndex}:${tc.path}:${previewLimit} for next ${previewLimit} lines)`;
+  }
+
+  return `File: ${tc.path}
+Tool: ${tc.name}
+
+${body}`;
+}
+
+// ── Parse drill-down query ────────────────────────────────────────────────
+
+/**
+ * Pattern: #N:path, #N:path:full, #N:path:offset, or #N:path:offset:limit
+ * Group 1: index number
+ * Group 2: path (consumed lazily, expanded until suffix can match)
+ * Group 3: suffix — "full", a number (offset), or "offset:limit"
+ */
+const DRILLDOWN_PATTERN = /^#(\d+):(.+?)(?::(full|\d+(?::\d+)?))?$/;
+
+/**
+ * Parse a drill-down query like #42:auth.ts or #42:auth.ts:full.
+ * Returns null if the query doesn't match the drill-down pattern.
+ *
+ * Suffixes:
+ *   :full       → full content (no truncation)
+ *   :30         → offset 30 lines, default limit (30)
+ *   :30:20      → offset 30 lines, limit 20 lines
+ *   (none)      → preview first 30 lines
+ */
+export function parseDrillDown(
+  query: string,
+): { index: number; pathPattern: string; full: boolean; offset?: number; limit?: number } | null {
+  const match = query.match(DRILLDOWN_PATTERN);
+  if (!match) return null;
+  const index = parseInt(match[1], 10);
+  const pathPattern = match[2];
+  const suffix = match[3];
+
+  if (suffix === "full") {
+    return { index, pathPattern, full: true, offset: undefined, limit: undefined };
+  }
+
+  if (suffix !== undefined) {
+    // Parse "offset" or "offset:limit"
+    const parts = suffix.split(":");
+    const offset = parseInt(parts[0], 10);
+    const limit = parts[1] !== undefined ? parseInt(parts[1], 10) : undefined;
+    if (!Number.isNaN(offset)) {
+      return { index, pathPattern, full: false, offset, limit };
+    }
+  }
+
+  return { index, pathPattern, full: false, offset: undefined, limit: undefined };
+}
+
+// ── Main export ───────────────────────────────────────────────────────────
+
+/**
+ * Expand a drill-down query (#N:path) to tool call content.
+ *
+ * Offset/limit let you page through file content incrementally (like pi's read tool):
+ *   #42:auth.ts         → first 30 lines (preview)
+ *   #42:auth.ts:full    → all content
+ *   #42:auth.ts:30      → lines 31-60 (default limit 30)
+ *   #42:auth.ts:30:20   → lines 31-50 (custom limit 20)
+ *
+ * @param sessionFile - Path to the JSONL session file
+ * @param entryIndex - The message index (#N)
+ * @param pathPattern - File path substring to match (or "file" keyword)
+ * @param full - If true, return complete content without truncation
+ * @param offset - Line offset (0-indexed) for windowed content
+ * @param limit - Max lines to show (default 30 for windowed, ignored if full=true)
+ * @returns Formatted content string
+ */
+export function expandEntryFile(
+  sessionFile: string,
+  entryIndex: number,
+  pathPattern: string,
+  full = false,
+  offset?: number,
+  limit?: number,
+): string {
+  const { rawMessages } = loadAllMessages(sessionFile, true);
+
+  if (entryIndex < 0 || entryIndex >= rawMessages.length) {
+    return `Entry #${entryIndex} not found in session history.`;
+  }
+
+  const msg = rawMessages[entryIndex];
+  const content = msg.content as unknown[];
+  const calls = findContentBearingCalls(content);
+
+  // Special case: #42:file keyword
+  if (pathPattern === "file") {
+    if (calls.length === 0) {
+      return `No file content found in entry #${entryIndex}.`;
+    }
+    if (calls.length === 1) {
+      return formatToolCallContent(calls[0], entryIndex, { full, offset, limit });
+    }
+    // Multiple content-bearing calls — list them
+    const items = calls.map((tc) => `  [#${entryIndex}:${tc.path}] ${tc.name}(${tc.path})`);
+    return `Entry #${entryIndex} has ${calls.length} file operations:\n${items.join("\n")}\n\nUse #${entryIndex}:path to drill into a specific file.`;
+  }
+
+  const matched = calls.filter((tc) => tc.path.includes(pathPattern));
+
+  if (matched.length === 0) {
+    return `No file content found in entry #${entryIndex} for "${pathPattern}".`;
+  }
+
+  if (matched.length > 1) {
+    // Ambiguous match — list options instead of silently picking the first
+    const items = matched.map(
+      (tc) => `  [#${entryIndex}:${tc.path}] ${tc.name}(${tc.path})`,
+    );
+    return `Entry #${entryIndex} has ${matched.length} file operations matching "${pathPattern}":
+${items.join("\n")}
+
+Use #${entryIndex}:<more-specific-path> to drill into a specific file.`;
+  }
+
+  return formatToolCallContent(matched[0], entryIndex, { full, offset, limit });
+}