pi-blackhole 0.2.0

package/src/om/pending.ts

@@ -0,0 +1,225 @@
+/**
+ * Pending OM state persistence.
+ *
+ * When `noAutoCompact` is enabled, observations, reflections, and dropper
+ * results are saved to disk instead of being appended to the conversation.
+ * Each new pipeline run replaces the previous result (latest subsumes earlier
+ * since every run processes all entries since the last actual branch append).
+ *
+ * On manual `/pi-vcc` trigger, pending entries are flushed to the branch
+ * and the file is cleared.
+ *
+ * Per-session files: each session gets its own <sessionId>-pending.json
+ * under ~/.pi/agent/pi-blackhole/. This eliminates race conditions from
+ * concurrent pi sessions writing to a shared file.
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export interface PendingObservation {
+	coversUpToId: string;
+	data: unknown;
+}
+
+export interface PendingReflection {
+	coversUpToId: string;
+	data: unknown;
+}
+
+export interface PendingDropped {
+	coversUpToId: string;
+	data: unknown;
+}
+
+export interface PendingOMState {
+	/** Latest observation run (replaced each time, not accumulated). */
+	observation?: PendingObservation;
+	/** Latest reflection run (replaced each time, not accumulated). */
+	reflection?: PendingReflection;
+	/** Latest dropper run (replaced each time, not accumulated). */
+	dropped?: PendingDropped;
+}
+
+// ── Persistence ─────────────────────────────────────────────────────────────
+
+const PENDING_DIR = "pi-blackhole";
+const PENDING_SUFFIX = "-pending.json";
+const STALE_SUFFIX = "-pending.stale.json";
+
+/** Build the path for a given session's pending file. */
+function pendingPath(sessionId: string): string {
+	return join(getAgentDir(), PENDING_DIR, `${sessionId}${PENDING_SUFFIX}`);
+}
+
+/** Build the path for a given session's stale pending file (backup of previous write). */
+function stalePath(sessionId: string): string {
+	return join(getAgentDir(), PENDING_DIR, `${sessionId}${STALE_SUFFIX}`);
+}
+
+/** Ensure the pending directory exists. */
+function ensureDir(): void {
+	const dir = join(getAgentDir(), PENDING_DIR);
+	if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+}
+
+function defaultState(): PendingOMState {
+	return {};
+}
+
+function isEmptyState(s: PendingOMState): boolean {
+	return !s.observation && !s.reflection && !s.dropped;
+}
+
+// ── Per-session file read/write ─────────────────────────────────────────────
+
+/**
+ * Read pending state for a specific session from its dedicated file.
+ * Returns default (empty) state if file doesn't exist or is corrupt.
+ */
+function readSessionState(sessionId: string): PendingOMState {
+	const path = pendingPath(sessionId);
+	if (!existsSync(path)) return defaultState();
+
+	try {
+		const raw = JSON.parse(readFileSync(path, "utf-8"));
+		if (isPendingOMState(raw)) return raw;
+		return defaultState();
+	} catch {
+		return defaultState();
+	}
+}
+
+/**
+ * Write pending state for a specific session to its dedicated file.
+ * Deletes the file if state is empty.
+ * Preserves the previous state as <sessionId>-pending.stale.json as backup.
+ */
+function writeSessionState(sessionId: string, state: PendingOMState): void {
+	const path = pendingPath(sessionId);
+	if (isEmptyState(state)) {
+		// Clear: remove both main and stale
+		try {
+			if (existsSync(path)) unlinkSync(path);
+		} catch { /* best-effort */ }
+		try {
+			const stale = stalePath(sessionId);
+			if (existsSync(stale)) unlinkSync(stale);
+		} catch { /* best-effort */ }
+		return;
+	}
+
+	ensureDir();
+
+	// Before writing new state, rename current file to stale as backup
+	try {
+		if (existsSync(path)) {
+			renameSync(path, stalePath(sessionId));
+		}
+	} catch { /* best-effort — stale backup is optional */ }
+
+	writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
+}
+
+/**
+ * Validate that an unknown value is a valid PendingOMState.
+ */
+function isPendingOMState(value: unknown): value is PendingOMState {
+	if (!value || typeof value !== "object") return false;
+	const v = value as Record<string, unknown>;
+	const hasObs = !!(v.observation && typeof v.observation === "object" && typeof (v.observation as any).coversUpToId === "string");
+	const hasRef = !!(v.reflection && typeof v.reflection === "object" && typeof (v.reflection as any).coversUpToId === "string");
+	const hasDrop = !!(v.dropped && typeof v.dropped === "object" && typeof (v.dropped as any).coversUpToId === "string");
+	return hasObs || hasRef || hasDrop;
+}
+
+// ── API ─────────────────────────────────────────────────────────────────────
+
+/**
+ * Save (replace) the latest observation result for a session.
+ * Each new run covers all entries since last branch append, so the latest
+ * result fully subsumes any previous one.
+ */
+export function savePendingObservation(sessionId: string, entry: PendingObservation): void {
+	const state = readSessionState(sessionId);
+	state.observation = entry;
+	writeSessionState(sessionId, state);
+}
+
+/**
+ * Save (replace) the latest reflection result for a session.
+ */
+export function savePendingReflection(sessionId: string, entry: PendingReflection): void {
+	const state = readSessionState(sessionId);
+	state.reflection = entry;
+	writeSessionState(sessionId, state);
+}
+
+/**
+ * Save (replace) the latest dropper result for a session.
+ */
+export function savePendingDropped(sessionId: string, entry: PendingDropped): void {
+	const state = readSessionState(sessionId);
+	state.dropped = entry;
+	writeSessionState(sessionId, state);
+}
+
+/**
+ * Check whether a coversUpToId matches the already-pending observation
+ * for the given session. Returns true if the chunk was already processed.
+ */
+export function isObservationChunkPending(sessionId: string, coversUpToId: string): boolean {
+	const s = readSessionState(sessionId);
+	return s.observation?.coversUpToId === coversUpToId;
+}
+
+/**
+ * Check whether a coversUpToId matches the already-pending reflection
+ * for the given session.
+ */
+export function isReflectionChunkPending(sessionId: string, coversUpToId: string): boolean {
+	const s = readSessionState(sessionId);
+	return s.reflection?.coversUpToId === coversUpToId;
+}
+
+/** Read the pending OM state for a specific session. */
+export function readPendingState(sessionId: string): PendingOMState {
+	return readSessionState(sessionId);
+}
+
+/** Clear the pending OM state for a specific session after flushing to branch. */
+export function clearPendingState(sessionId: string): void {
+	writeSessionState(sessionId, defaultState());
+}
+
+/** Check whether there is any pending OM state for a specific session. */
+export function hasPendingData(sessionId: string): boolean {
+	return !isEmptyState(readSessionState(sessionId));
+}
+
+/**
+ * List all session IDs that have pending data by scanning the pending directory
+ * for *-pending.json files.
+ */
+export function listPendingSessions(): string[] {
+	const dir = join(getAgentDir(), PENDING_DIR);
+	if (!existsSync(dir)) return [];
+
+	try {
+		const files = readdirSync(dir);
+		const sessions: string[] = [];
+		for (const file of files) {
+			if (!file.endsWith(PENDING_SUFFIX)) continue;
+			const sessionId = file.slice(0, -PENDING_SUFFIX.length);
+			if (!sessionId) continue;
+			// Verify the file actually has non-empty data
+			const state = readSessionState(sessionId);
+			if (!isEmptyState(state)) sessions.push(sessionId);
+		}
+		return sessions;
+	} catch {
+		return [];
+	}
+}

package/src/om/reverse-recall.ts

@@ -0,0 +1,130 @@
+/**
+ * Reverse recall — given session entry IDs, find related OM observations/reflections.
+ *
+ * This is the vcc→OM direction: when expanding session entries, look up
+ * observations whose sourceEntryIds contain those entry IDs.
+ */
+import {
+	indexLedger,
+	type Entry,
+} from "./ledger/recall.js";
+import { type RenderedEntry } from "../core/render-entries.js";
+
+// ── Types ─────────────────────────────────────────────────────────────────
+
+export interface RelatedObservation {
+	memoryId: string;
+	content: string;
+	timestamp: string;
+	relevance: string;
+	status: "active" | "dropped";
+	matchedEntryIds: string[];
+}
+
+export interface RelatedReflection {
+	memoryId: string;
+	content: string;
+}
+
+// ── Lookup ────────────────────────────────────────────────────────────────
+
+export function findObservationsForEntryIds(
+	entries: Entry[],
+	targetEntryIds: string[],
+): RelatedObservation[] {
+	if (targetEntryIds.length === 0) return [];
+	const { observations, droppedIds } = indexLedger(entries);
+	const targetSet = new Set(targetEntryIds);
+
+	const result: RelatedObservation[] = [];
+	for (const indexed of observations) {
+		const matched = indexed.observation.sourceEntryIds.filter((id) => targetSet.has(id));
+		if (matched.length > 0) {
+			result.push({
+				memoryId: indexed.observation.id,
+				content: indexed.observation.content,
+				timestamp: indexed.observation.timestamp,
+				relevance: indexed.observation.relevance,
+				status: droppedIds.has(indexed.observation.id) ? "dropped" : "active",
+				matchedEntryIds: matched,
+			});
+		}
+	}
+	return result;
+}
+
+export function findReflectionsForEntryIds(
+	entries: Entry[],
+	targetEntryIds: string[],
+): RelatedReflection[] {
+	if (targetEntryIds.length === 0) return [];
+	const { reflections } = indexLedger(entries);
+	// Reflections don't directly reference entry IDs — they reference observation IDs.
+	// So we only match indirectly: first find observations for these entry IDs,
+	// then find reflections that support those observations.
+	const { observations } = indexLedger(entries);
+	const targetSet = new Set(targetEntryIds);
+	const matchingObsIds = new Set<string>();
+	for (const indexed of observations) {
+		if (indexed.observation.sourceEntryIds.some((id) => targetSet.has(id))) {
+			matchingObsIds.add(indexed.observation.id);
+		}
+	}
+	if (matchingObsIds.size === 0) return [];
+	return reflections
+		.filter((r) => r.reflection.supportingObservationIds.some((id) => matchingObsIds.has(id)))
+		.map((r) => ({
+			memoryId: r.reflection.id,
+			content: r.reflection.content,
+		}));
+}
+
+// ── Formatters ────────────────────────────────────────────────────────────
+
+export function formatRelatedObservations(
+	observations: RelatedObservation[],
+	_reflections: RelatedReflection[],
+): string {
+	const parts: string[] = [];
+
+	if (observations.length > 0) {
+		parts.push("Related observations:");
+		for (const obs of observations) {
+			const dropped = obs.status === "dropped" ? " [dropped]" : "";
+			const entryRefs = obs.matchedEntryIds.length > 0
+				? ` (${obs.matchedEntryIds.join(", ")})`
+				: "";
+			parts.push(`  [${obs.memoryId}]${dropped} ${obs.timestamp} [${obs.relevance}] ${obs.content}${entryRefs}`);
+		}
+	}
+
+	return parts.join("\n");
+}
+
+/**
+ * Build a session-entry-id → message-index map from RenderedEntry[].
+ * Used to annotate source entries with their #N indices.
+ */
+export function buildIndexMap(rendered: RenderedEntry[]): Map<string, number> {
+	const map = new Map<string, number>();
+	for (const entry of rendered) {
+		if (entry.id && !map.has(entry.id)) {
+			map.set(entry.id, entry.index);
+		}
+	}
+	return map;
+}
+
+export function formatEntryIndexAnnotation(
+	sourceEntryIds: string[],
+	idToIndex: Map<string, number>,
+): string {
+	const indices: number[] = [];
+	for (const id of sourceEntryIds) {
+		const idx = idToIndex.get(id);
+		if (idx !== undefined) indices.push(idx);
+	}
+	if (indices.length === 0) return "";
+	indices.sort((a, b) => a - b);
+	return `(at index #${indices.join(", #")})`;
+}

package/src/om/runtime.ts

@@ -0,0 +1,241 @@
+/**
+ * Observational memory runtime — model resolution, consolidation lifecycle,
+ * cooldown integration, error tracking.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/runtime.ts)
+ * Modified by pi-vcc-om:
+ * - resolveModel iterates fallback chain (stage → fallbacks → base → session).
+ * - Skips cooled-down models (cooldown.ts).
+ * - recordRetryableError persists cooldown on API errors.
+ * - markConsolidationError sets 30s retry gate for failed runs.
+ */
+import { type Config, type ConfiguredModel, DEFAULTS, loadConfig } from "./config.js";
+import { isCooldownActive, recordCooldown, expireCooldowns, modelKey } from "./cooldown.js";
+
+export type ResolveResult =
+	| { ok: true; model: any; apiKey: string; headers?: Record<string, string>; cooldownApplied?: boolean }
+	| { ok: false; reason: string };
+
+type NotifyLevel = "warning" | "info" | "error";
+type Notify = (message: string, type?: NotifyLevel) => void;
+export type ConsolidationPhase = "observer" | "reflector" | "dropper";
+
+export interface ResolveCtx {
+	model: unknown;
+	modelRegistry: any;
+	hasUI: boolean;
+	ui?: { notify: Notify };
+	/** Primary stage model (from config). */
+	stageModel?: ConfiguredModel;
+	/** Fallback models for this stage (from config). */
+	stageFallbacks?: ConfiguredModel[];
+}
+
+export interface LaunchCtx {
+	hasUI: boolean;
+	ui?: { notify: Notify };
+}
+
+/** Default cooldown interval between failed consolidation runs (ms). */
+const CONSOLIDATION_RETRY_COOLDOWN_MS = 30_000;
+
+export class Runtime {
+	config: Config = { ...DEFAULTS };
+	configLoaded = false;
+	consolidationInFlight = false;
+	consolidationPromise: Promise<void> | null = null;
+	consolidationPhase: ConsolidationPhase | undefined;
+	compactInFlight = false;
+	compactHookInFlight = false;
+	resolveFailureNotified = false;
+	lastObserverError: string | undefined;
+	lastReflectorError: string | undefined;
+	lastDropperError: string | undefined;
+	/** Epoch ms of the last failed consolidation run (any stage). */
+	lastConsolidationErrorAt: number | undefined;
+
+	ensureConfig(cwd: string): void {
+		if (this.configLoaded) return;
+		this.config = loadConfig(cwd);
+		this.configLoaded = true;
+		expireCooldowns();
+	}
+
+	/**
+	 * Build the ordered model candidate list for a stage:
+	 * 1. Primary stage model (observerModel, reflectorModel, dropperModel)
+	 * 2. Stage fallbacks (observerFallbackModels, etc.)
+	 * 3. Base config.model
+	 *
+	 * Session model (ctx.model) is only used as the last resort inside resolveModel.
+	 */
+	private buildCandidateList(stageModel?: ConfiguredModel, stageFallbacks?: ConfiguredModel[]): ConfiguredModel[] {
+		const candidates: ConfiguredModel[] = [];
+		if (stageModel) candidates.push(stageModel);
+		if (stageFallbacks) candidates.push(...stageFallbacks);
+		if (this.config.model) candidates.push(this.config.model);
+		return candidates;
+	}
+
+	/**
+	 * Resolve a model for a consolidation stage.
+	 *
+	 * Tries the candidate list in order:
+	 * 1. Primary stage model → 2. Stage fallbacks → 3. Base config.model → 4. Session model.
+	 *
+	 * Skips models that are currently in a cooldown window.
+	 * On retryable error (after the agent runs), the model that failed is cooled down
+	 * and the next candidate is tried.  The caller must call `recordRetryableError`
+	 * after the API attempt to mark the failed model.
+	 *
+	 * Returns `ok: true` with the resolved model, or `ok: false` with a reason
+	 * if all candidates (including session model) are exhausted or unavailable.
+	 */
+	async resolveModel(ctx: ResolveCtx): Promise<ResolveResult> {
+		const candidates = this.buildCandidateList(ctx.stageModel, ctx.stageFallbacks);
+		const stageName = this.consolidationPhase ?? "unknown";
+
+		// Try configured candidates
+		for (const candidate of candidates) {
+			if (isCooldownActive(candidate)) {
+				if (ctx.hasUI && ctx.ui) {
+					ctx.ui.notify(
+						`Observational memory: ${stageName} skipping ${modelKey(candidate)} (cooldown active)`,
+						"info",
+					);
+				}
+				continue;
+			}
+
+			const configured = ctx.modelRegistry.find(candidate.provider, candidate.id);
+			if (!configured) {
+				if (ctx.hasUI && ctx.ui) {
+					ctx.ui.notify(
+						`Observational memory: ${stageName} model ${candidate.provider}/${candidate.id} not found`,
+						"warning",
+					);
+				}
+				continue;
+			}
+
+			const auth = await ctx.modelRegistry.getApiKeyAndHeaders(configured);
+			if (!auth.ok || !auth.apiKey) {
+				if (ctx.hasUI && ctx.ui) {
+					ctx.ui.notify(
+						`Observational memory: ${stageName} no auth for ${candidate.provider}`,
+						"warning",
+					);
+				}
+				continue;
+			}
+
+			return {
+				ok: true,
+				model: configured,
+				apiKey: auth.apiKey as string,
+				headers: auth.headers as Record<string, string> | undefined,
+				cooldownApplied: false,
+			};
+		}
+
+		// Fall back to session model
+		const sessionModel = ctx.model;
+		if (!sessionModel) {
+			return { ok: false, reason: `no model available for ${stageName} (all candidates exhausted, no session model)` };
+		}
+
+		const auth = await ctx.modelRegistry.getApiKeyAndHeaders(sessionModel);
+		if (!auth.ok || !auth.apiKey) {
+			const provider = (sessionModel as { provider?: string }).provider ?? "unknown";
+			return { ok: false, reason: `no API key for session model provider "${provider}"` };
+		}
+
+		return {
+			ok: true,
+			model: sessionModel,
+			apiKey: auth.apiKey as string,
+			headers: auth.headers as Record<string, string> | undefined,
+			cooldownApplied: false,
+		};
+	}
+
+	/**
+	 * Get the model config for the currently resolved model (used for cooldown recording).
+	 * Returns the candidate config if the model was from the candidate list,
+	 * or undefined if it's the session model.
+	 */
+	findCandidateConfig(resolvedModel: unknown, ctx: ResolveCtx): ConfiguredModel | undefined {
+		const candidates = this.buildCandidateList(ctx.stageModel, ctx.stageFallbacks);
+		const model = resolvedModel as { provider?: string; id?: string };
+		if (!model.provider || !model.id) return undefined;
+		return candidates.find((c) => c.provider === model.provider && c.id === model.id)
+			?? (this.config.model?.provider === model.provider && this.config.model?.id === model.id ? this.config.model : undefined);
+	}
+
+	/**
+	 * Record a retryable error for a model.  The model must be one of the candidates
+	 * (not the session model).  If it's the session model we don't cool it down.
+	 */
+	recordRetryableError(modelConfig: ConfiguredModel | undefined, error: unknown, stage: ConsolidationPhase): void {
+		if (!modelConfig) return;
+		const reason = error instanceof Error ? error.message : String(error || "unknown error");
+		recordCooldown(modelConfig, reason, stage);
+	}
+
+	/**
+	 * Record that a consolidation stage error occurred.
+	 * Sets the retry-gate timestamp so the next trigger is delayed.
+	 */
+	markConsolidationError(): void {
+		this.lastConsolidationErrorAt = Date.now();
+	}
+
+	/** Check if the consolidation retry gate is active (too soon after last error). */
+	isConsolidationRetryGated(): boolean {
+		if (!this.lastConsolidationErrorAt) return false;
+		return Date.now() - this.lastConsolidationErrorAt < CONSOLIDATION_RETRY_COOLDOWN_MS;
+	}
+
+	launchConsolidationTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> {
+		this.consolidationInFlight = true;
+		this.consolidationPhase = undefined;
+		const promise = this.launchTrackedTask(ctx, "consolidation", work, () => {
+			this.consolidationInFlight = false;
+			this.consolidationPhase = undefined;
+			if (this.consolidationPromise === promise) this.consolidationPromise = null;
+		});
+		this.consolidationPromise = promise;
+		return promise;
+	}
+
+	recordConsolidationStageError(ctx: LaunchCtx, phase: ConsolidationPhase, error: unknown): string {
+		const message = error instanceof Error ? error.message : String(error);
+		if (phase === "observer") this.lastObserverError = message;
+		if (phase === "reflector") this.lastReflectorError = message;
+		if (phase === "dropper") this.lastDropperError = message;
+		if (ctx.hasUI && ctx.ui) ctx.ui.notify(`Observational memory: ${phase} failed: ${message}`, "warning");
+		this.markConsolidationError();
+		return message;
+	}
+
+	private launchTrackedTask(
+		ctx: LaunchCtx,
+		label: string,
+		work: () => Promise<void>,
+		onFinally: (error: string | undefined) => void,
+	): Promise<void> {
+		const hasUI = ctx.hasUI;
+		const ui = ctx.ui;
+		return (async () => {
+			let errorMessage: string | undefined;
+			try {
+				await work();
+			} catch (error) {
+				errorMessage = error instanceof Error ? error.message : String(error);
+				if (hasUI && ui) ui.notify(`Observational memory: ${label} failed: ${errorMessage}`, "warning");
+			} finally {
+				onFinally(errorMessage);
+			}
+		})();
+	}
+}