pi-blackhole 0.2.2 → 0.3.0

package/src/om/cooldown.ts

@@ -57,10 +57,17 @@ function readCooldownMap(): CooldownMap {
 }
 
 function writeCooldownMap(map: CooldownMap): void {
-	const path = cooldownPath();
-	const dir = dirname(path);
-	if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-	writeFileSync(path, `${JSON.stringify(map, null, 2)}\n`);
+	try {
+		const path = cooldownPath();
+		const dir = dirname(path);
+		if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+		writeFileSync(path, `${JSON.stringify(map, null, 2)}\n`);
+	} catch {
+		// Best-effort: cooldowns are advisory. Losing them means a rate-limited
+		// model might be retried before its cooldown window expires — slightly
+		// more API traffic, no data loss. This also prevents a process crash
+		// on read-only filesystems.
+	}
 }
 
 // ── API ─────────────────────────────────────────────────────────────────────

package/src/om/ledger/projection.ts

@@ -4,6 +4,7 @@
  * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/projection.ts)
  * Unmodified.
  */
+import { selectPriorObservations } from "./render-summary.js";
 import {
 	OM_FOLDED,
 	isMemoryDetails,
@@ -207,10 +208,21 @@ export function buildCompactionProjection(
 		0,
 	);
 	const fullFold = observationTokens >= config.observationsPoolMaxTokens;
-	const projection = fullFold
+	let projection = fullFold
 		? fullProjection(entries, firstKeptEntryId)
 		: normalProjection;
 
+	// Cap observations to budget using relevance-tiered + recency scoring.
+	// Even if the dropper determined some old observations are worth keeping,
+	// this safety valve ensures the compaction output never exceeds the pool
+	// token budget. Observations survive in the branch regardless.
+	if (config.observationsPoolMaxTokens > 0 && observationTokens >= config.observationsPoolMaxTokens) {
+		projection = {
+			observations: selectPriorObservations(projection.observations, config.observationsPoolMaxTokens),
+			reflections: projection.reflections,
+		};
+	}
+
 	const details: MemoryDetails = {
 		type: OM_FOLDED,
 		version: 1,

package/src/om/ledger/render-summary.ts

@@ -25,6 +25,57 @@ export function observationToSummaryLine(observation: Observation): string {
 	return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`;
 }
 
+/** Score an observation for cap/trim selection.
+ *  Relevance tier dominates: medium (5+) always outranks low (max 2).
+ *  Recency is based on position in the flat-mapped array (0 = oldest, N-1 = newest),
+ *  avoiding wall-clock dependency that punishes sessions spanning days or weeks. */
+export function scoreObservation(obs: Observation, index: number, total: number): number {
+	const base = obs.relevance === "high" || obs.relevance === "critical" ? 10
+		: obs.relevance === "medium" ? 5 : 1;
+	const recency = total > 1 ? index / (total - 1) : 1;
+	return base + recency;
+}
+
+/** Select observations up to a token budget, keeping all high-relevance items
+ *  unconditionally and filling the remaining budget with the best-scoring
+ *  medium and low observations (relevance-tiered + recency).
+ *
+ *  Reflections are never trimmed — they are inherently rare and always stay.
+ *  Observations stay in the branch either way; this only caps what is rendered
+ *  in the compaction summary output. */
+export function selectPriorObservations(observations: Observation[], maxTokens: number): Observation[] {
+	// Track original indices so we can restore chronological order after scoring
+	const indexed = observations.map((obs, i) => ({ obs, originalIndex: i }));
+	const high = indexed.filter(item => item.obs.relevance === "high" || item.obs.relevance === "critical");
+	const rest = indexed.filter(item => item.obs.relevance !== "high" && item.obs.relevance !== "critical");
+
+	// High always kept — consume budget first
+	let budget = maxTokens;
+	const selected = new Set<{ obs: Observation; originalIndex: number }>();
+	for (const item of high) {
+		const lineTokens = Math.ceil(observationToSummaryLine(item.obs).length / 4);
+		selected.add(item);
+		budget -= lineTokens;
+	}
+
+	// Score medium + low and select best within remaining budget
+	if (rest.length > 0 && budget > 0) {
+		const scored = rest.map((item, i) => ({ item, score: scoreObservation(item.obs, i, rest.length) }));
+		scored.sort((a, b) => b.score - a.score); // highest score first
+		for (const { item } of scored) {
+			const lineTokens = Math.ceil(observationToSummaryLine(item.obs).length / 4);
+			if (budget - lineTokens < 0) break;
+			selected.add(item);
+			budget -= lineTokens;
+		}
+	}
+
+	// Restore original chronological order before returning
+	return Array.from(selected)
+		.sort((a, b) => a.originalIndex - b.originalIndex)
+		.map(item => item.obs);
+}
+
 export function reflectionToSummaryLine(reflection: Reflection): string {
 	return `[${reflection.id}] ${reflection.content}`;
 }

package/src/om/pending.ts

@@ -14,10 +14,8 @@
  * concurrent pi sessions writing to a shared file.
  */
 import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { join } from "node:path";
 import { getAgentDir } from "@earendil-works/pi-coding-agent";
-import type { Observation, Reflection } from "./ledger/types.js";
-
 // ── Types ───────────────────────────────────────────────────────────────────
 
 export interface PendingObservation {
@@ -143,7 +141,13 @@ function writeSessionState(sessionId: string, state: PendingOMState): void {
 		}
 	} catch { /* best-effort — stale backup is optional */ }
 
-	writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
+	try {
+		writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
+	} catch {
+		// Best-effort: pending state loss means consolidation may re-process
+		// the same data on the next run — safe (idempotent by design).
+		// Prevents process crash on read-only filesystems.
+	}
 }
 
 /**

package/src/tools/recall.ts

@@ -4,13 +4,17 @@
  *
  * Created by pi-vcc-om. Replaces pi-vcc's vcc_recall and OM's standalone recall-observation.
  */
-import { Type } from "@earendil-works/pi-ai";
+import { Type } from "typebox";
+import { StringEnum } from "@earendil-works/pi-ai";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { loadAllMessages } from "../core/load-messages";
-import { searchEntries } from "../core/search-entries";
-import { formatRecallOutput } from "../core/format-recall";
+import { searchEntries, getFileIndicators, getTouchedFiles } from "../core/search-entries";
+import type { RenderedEntry } from "../core/render-entries";
+import type { SearchHit } from "../core/search-entries";
+import { formatRecallOutput, formatTouchedOutput } from "../core/format-recall";
 import { getActiveLineageEntryIds } from "../core/lineage";
-import { normalizeRecallScope } from "../core/recall-scope";
+import { normalizeRecallScope, normalizeRecallMode } from "../core/recall-scope";
+import { parseDrillDown, expandEntryFile } from "../core/drill-down.js";
 import {
 	recallMemorySources,
 	type Entry,
@@ -29,20 +33,63 @@ import {
 const DEFAULT_RECENT = 25;
 const PAGE_SIZE = 5;
 
-const invalidExpandIndices = (requested: number[], available: Set<number>): number[] =>
+export const invalidExpandIndices = (requested: number[], available: Set<number>): number[] =>
 	requested.filter((i) => !Number.isInteger(i) || !available.has(i));
 
-async function vccRecall(params: { query?: string; expand?: number[]; page?: number; scope?: "lineage" | "all" }, ctx: any) {
+/**
+ * Merge expanded (full-content) entries into search results.
+ * Overlapping entries get their summary replaced with full content.
+ * Non-overlapping expanded entries are appended. Results are sorted by index.
+ */
+export function mergeExpandedIntoSearchResults(
+	searchResults: SearchHit[],
+	expandedEntries: RenderedEntry[],
+): SearchHit[] {
+	if (expandedEntries.length === 0) return searchResults;
+
+	const expandedByIndex = new Map(expandedEntries.map((e) => [e.index, e]));
+
+	// Replace truncated summaries with full content for expanded indices
+	const merged = searchResults.map((r) => {
+		const full = expandedByIndex.get(r.index);
+		return full ? { ...r, summary: full.summary } : r;
+	});
+
+	// Append expand-only entries not already in search results
+	for (const fe of expandedEntries) {
+		if (!merged.some((r) => r.index === fe.index)) {
+			merged.push(fe as SearchHit);
+		}
+	}
+
+	// Maintain natural order by index
+	merged.sort((a, b) => a.index - b.index);
+	return merged;
+}
+
+async function vccRecall(params: { query?: string; expand?: number[]; page?: number; scope?: "lineage" | "all"; mode?: string }, ctx: any) {
 	const sessionFile = ctx.sessionManager.getSessionFile();
 	if (!sessionFile) {
 		return { content: [{ type: "text" as const, text: "No session file available." }], details: undefined };
 	}
 	const scope = normalizeRecallScope(params.scope);
+	const mode = normalizeRecallMode(params.mode);
 	const lineageEntryIds = scope === "lineage" ? getActiveLineageEntryIds(ctx.sessionManager) : undefined;
+
+	// ── "touched" mode: aggregate file operations ──
+	if (mode === "touched") {
+		const { rendered, rawMessages } = loadAllMessages(sessionFile, false, lineageEntryIds);
+		const touched = getTouchedFiles(rawMessages, rendered);
+		const text = formatTouchedOutput(touched, params.page);
+		return { content: [{ type: "text" as const, text }], details: undefined };
+	}
+
 	const expandSet = new Set(params.expand ?? []);
 	const hasExpand = expandSet.size > 0;
 
-	if (hasExpand && !params.query) {
+	// ── Pre-load full messages if expand is requested ──
+	let expandedFullEntries: RenderedEntry[] | undefined;
+	if (hasExpand) {
 		const { rendered: fullMsgs } = loadAllMessages(sessionFile, true, lineageEntryIds);
 		const requested = [...expandSet];
 		const byIndex = new Map(fullMsgs.map((m) => [m.index, m]));
@@ -50,34 +97,55 @@ async function vccRecall(params: { query?: string; expand?: number[]; page?: num
 		if (invalid.length > 0) {
 			return { content: [{ type: "text" as const, text: `Cannot expand indices outside ${scope === "all" ? "session history" : "active lineage"}: ${invalid.join(", ")}` }], details: undefined };
 		}
-		const expanded = requested.map((i) => byIndex.get(i)).filter((m): m is NonNullable<typeof m> => Boolean(m));
-		let output = (scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(expanded);
+		expandedFullEntries = requested.map((i) => byIndex.get(i)).filter((m): m is NonNullable<typeof m> => Boolean(m));
 
-		// Coupling: look up related OM observations
-		const expandedIds = expanded.map((e) => e.id).filter(Boolean);
-		if (expandedIds.length > 0) {
-			try {
-				const branchEntries = ctx.sessionManager.getBranch() as Entry[];
-				const obs = findObservationsForEntryIds(branchEntries, expandedIds);
-				const refs = findReflectionsForEntryIds(branchEntries, expandedIds);
-				if (obs.length > 0 || refs.length > 0) {
-					output += "\n\n" + formatRelatedObservations(obs, refs);
-				}
-			} catch { /* branch may not be available */ }
-		}
+		// Expand-only path (no query): return expanded entries immediately
+		if (!params.query) {
+			let output = (scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(expandedFullEntries);
 
-		return { content: [{ type: "text" as const, text: output }], details: undefined };
+			// Coupling: look up related OM observations
+			const expandedIds = expandedFullEntries.map((e) => e.id).filter(Boolean);
+			if (expandedIds.length > 0) {
+				try {
+					const branchEntries = ctx.sessionManager.getBranch() as Entry[];
+					const obs = findObservationsForEntryIds(branchEntries, expandedIds);
+					const refs = findReflectionsForEntryIds(branchEntries, expandedIds);
+					if (obs.length > 0 || refs.length > 0) {
+						output += "\n\n" + formatRelatedObservations(obs, refs);
+					}
+				} catch { /* branch may not be available */ }
+			}
+
+			return { content: [{ type: "text" as const, text: output }], details: undefined };
+		}
+		// With query: fall through to search, then merge expanded entries into results
 	}
 
 	const { rendered: msgs, rawMessages } = loadAllMessages(sessionFile, false, lineageEntryIds);
-	const allResults = params.query?.trim()
-		? searchEntries(msgs, rawMessages, params.query)
-		: msgs.slice(-DEFAULT_RECENT);
+	let allResults: SearchHit[] = params.query?.trim()
+		? searchEntries(msgs, rawMessages, params.query, undefined, mode)
+		: msgs.slice(-DEFAULT_RECENT).map((entry, i) => {
+				const msgIndex = Math.max(0, msgs.length - DEFAULT_RECENT) + i;
+				const msg = rawMessages[msgIndex];
+				if (msg) {
+					const indicators = getFileIndicators(msg);
+					if (indicators.length > 0) {
+						return { ...entry, fileMatches: indicators };
+					}
+				}
+				return entry;
+			});
+
+	// Merge expanded entries into full result set BEFORE pagination
+	// so pagination counts and positioning stay consistent
+	if (expandedFullEntries) {
+		allResults = mergeExpandedIntoSearchResults(allResults, expandedFullEntries);
+	}
 
 	if (params.query?.trim()) {
 		const page = Math.max(1, params.page ?? 1);
 		const start = (page - 1) * PAGE_SIZE;
-		const pageResults = allResults.slice(start, start + PAGE_SIZE);
+		const pageResults: SearchHit[] = allResults.slice(start, start + PAGE_SIZE);
 		const totalPages = Math.ceil(allResults.length / PAGE_SIZE);
 		const scopeSuffix = scope === "all" ? " (scope: all)" : "";
 		const header = totalPages > 1
@@ -86,6 +154,7 @@ async function vccRecall(params: { query?: string; expand?: number[]; page?: num
 		const footer = page < totalPages
 			? `\n--- Use page:${page + 1}${scope === "all" ? " with scope:'all'" : ""} for more results ---`
 			: "";
+
 		let output = formatRecallOutput(pageResults, params.query, header) + footer;
 
 		// Coupling: augment search results with related observations
@@ -104,6 +173,7 @@ async function vccRecall(params: { query?: string; expand?: number[]; page?: num
 		return { content: [{ type: "text" as const, text: output }], details: undefined };
 	}
 
+	// No query: show recent entries (expand already merged above)
 	const output = (scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(allResults, params.query);
 	return { content: [{ type: "text" as const, text: output }], details: undefined };
 }
@@ -160,40 +230,46 @@ export function registerRecallTool(pi: ExtensionAPI): void {
 		name: "recall",
 		label: "Recall",
 		description:
-			"Recall session history or memory evidence. This is text/pattern matching, NOT semantic search. Accepts:\n" +
-			"- A 12-char hex id [a1b2c3d4e5f6] to recover observation/reflection source evidence.\n" +
-			"- A #N entry index to expand a specific transcript entry from compacted output.\n" +
-			"- A text/regex query to search conversation content. Multi-word queries use BM25 ranking with stopword filtering.\n" +
-			"Search tips: use unique concrete words (file names, function names, exact phrases), not conceptual questions. Regex metacharacters (|, *, .) trigger regex mode. Default scope is active lineage; use scope:'all' for off-lineage branches.",
+			"Search session history and earlier lines omitted, file write/edit content by text/regex. " +
+			"Expand entries (#N), drill-down file content (#N:path) with paging, or aggregate touched files (mode:touched).",
 		promptSnippet:
-			"recall: Text/regex search (not semantic). Also: 12-char hex ids recover obs/reflection sources; #N indices expand transcript entries. Use concrete words for search. scope:'all' for off-lineage.",
+			"recall: Search session history + file write/edit content by text/regex. #N expand, #N:path drill-down with offset/limit paging, mode:file/transcript/touched.",
 		promptGuidelines: [
-			"Use recall with a 12-char hex id before making an important decision that depends on a compacted observation or reflection whose details are unclear.",
-			"Use recall with a search query when you need to find specific conversation content. Use unique concrete terms (file paths, function names, error messages, exact phrases), not conceptual questions. Example: 'merge_design.md' not 'what did we decide about merging'.",
-			"Use recall with #N to expand a transcript entry reference from compacted output.",
-			"After compaction, the summary includes observation/reflection ids in brackets. Use recall with those ids to recover full source evidence.",
-			"If you get no results, try fewer terms, use a distinctive single word, or use a regex pattern (e.g. 'fork.*pi-vcc').",
+			"Use recall — search is literal text/regex matching, NOT semantic. If no results, try different terms or a regex pattern. Set scope:'all' to search the full session.",
+			"Use recall — mode:file to search only write/edit file content, mode:transcript for conversation-only, mode:touched for aggregate view of all files written/edited across the session.",
+			"Use recall — drill-down supports paging: #42:auth.ts shows first 30 lines, #42:auth.ts:30 shows next 30, #42:auth.ts:full shows everything. Note: edit diffs are not indexed for text search — drill-down reads them from raw JSONL.",
+			"Use recall — when a drill-down path matches multiple files, options are listed. Narrow with a more specific path substring.",
+			"Use recall — hex observation/reflection ids (12-char hex) link memory evidence to session entries with cross-references for navigation.",
 		],
 		parameters: Type.Object({
 			query: Type.Optional(
-				Type.String({ description: "12-char hex id, #N entry index, or text/regex search terms. NOT semantic — use concrete words (file names, quoted phrases, identifiers). Regex metacharacters trigger regex mode. Multi-word = BM25-ranked OR." }),
+				Type.String({ description: "Text/regex search; #N expands entry; #N:path drills file (#N:file auto-selects); #N:path:full all lines; #N:path:offset:limit range; 12-char hex for observations. Only full-file writes indexed." }),
 			),
 			expand: Type.Optional(
-				Type.Array(Type.Number(), { description: "Entry indices to return full untruncated content for (blackhole format only)." }),
+				Type.Array(Type.Number(), { description: "Entry indices to return full untruncated content for. Standalone or with query." }),
 			),
 			page: Type.Optional(
-				Type.Number({ description: "Page number (1-based) for paginated search results. Default: 1." }),
+				Type.Number({ description: "Page number (1-based) for paginated results. Default: 1." }),
 			),
 			scope: Type.Optional(
-				Type.Union([
-					Type.Literal("lineage"),
-					Type.Literal("all"),
-				], { description: "Search scope. Default: lineage; all includes off-lineage branches." }),
+				StringEnum(["lineage", "all"] as const, { description: "Search scope. lineage = active lineage (default), all = entire session." }),
+			),
+			mode: Type.Optional(
+				StringEnum(["hybrid", "file", "transcript", "touched"] as const, { description: "What content to search. hybrid (default) = transcript + file indicators. file = write/edit file content only. transcript = conversation only. touched = aggregate files grouped by path (not per-entry search)." }),
 			),
 		}),
 		async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
 			const q = params.query?.trim();
 			// Dispatch by format
+			if (q && parseDrillDown(q)) {
+				const parsed = parseDrillDown(q)!;
+				const sessionFile = ctx.sessionManager.getSessionFile();
+				if (!sessionFile) {
+					return { content: [{ type: "text" as const, text: "No session file available." }], details: undefined };
+				}
+				const text = expandEntryFile(sessionFile, parsed.index, parsed.pathPattern, parsed.full, parsed.offset, parsed.limit);
+				return { content: [{ type: "text" as const, text }], details: undefined };
+			}
 			if (q && VCC_ENTRY_PATTERN.test(q)) {
 				// #N → expand entry indices
 				const match = q.match(VCC_ENTRY_PATTERN);

package/vitest.config.ts

@@ -1,18 +1,5 @@
 import { defineConfig } from "vitest/config";
 
-// ── pnpm global store paths for @earendil-works packages ──────────────────
-
-const GLOBAL_PNPM =
-	"/home/kovalik/.local/share/pnpm/global/5/.pnpm";
-
-const PKGS: Record<string, string> = {
-	"@earendil-works/pi-ai": `${GLOBAL_PNPM}/@earendil-works+pi-ai@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-ai`,
-	"@earendil-works/pi-ai/oauth": `${GLOBAL_PNPM}/@earendil-works+pi-ai@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-ai/oauth`,
-	"@earendil-works/pi-agent-core": `${GLOBAL_PNPM}/@earendil-works+pi-agent-core@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-agent-core`,
-	"@earendil-works/pi-coding-agent": `${GLOBAL_PNPM}/@earendil-works+pi-coding-agent@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent`,
-	"@earendil-works/pi-tui": `${GLOBAL_PNPM}/@earendil-works+pi-tui@0.75.4/node_modules/@earendil-works/pi-tui`,
-};
-
 export default defineConfig({
 	test: {
 		globals: true,
@@ -22,12 +9,7 @@ export default defineConfig({
 	},
 	resolve: {
 		alias: [
-			// Map @earendil-works/* to global pnpm store
-			...Object.entries(PKGS).map(([name, path]) => ({
-				find: new RegExp(`^${escapeRegex(name)}$`),
-				replacement: path,
-			})),
-			// Resolve .js → extension-less for our source files
+			// Resolve .js → extension-less for our TypeScript source files
 			{
 				find: /\.js$/,
 				replacement: "",
@@ -35,7 +17,3 @@ export default defineConfig({
 		],
 	},
 });
-
-function escapeRegex(s: string): string {
-	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}