pi-blackhole 0.2.0

package/src/om/consolidation.ts

@@ -0,0 +1,514 @@
+/**
+ * Consolidation pipeline — observer → reflector → dropper with fallback retry.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/hooks/consolidation-trigger.ts)
+ * Modified by pi-vcc-om:
+ * - Each stage retries through fallback models when any error occurs.
+ * - All errors record cooldown (so the failed model is skipped next iteration).
+ * - 30s retry gate prevents repeated failed runs (isConsolidationRetryGated).
+ */
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { runDropper } from "./agents/dropper/agent.js";
+import { runObserver } from "./agents/observer/agent.js";
+import { runReflector } from "./agents/reflector/agent.js";
+import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
+import type { ConfiguredModel } from "./config.js";
+import { debugLog, withDebugLogContext } from "./debug-log.js";
+import { type ResolveResult, type Runtime } from "./runtime.js";
+import { isRetryableError } from "./cooldown.js";
+import { serializeSourceAddressedBranchEntries } from "./serialize.js";
+import {
+	savePendingObservation,
+	savePendingReflection,
+	savePendingDropped,
+	isObservationChunkPending,
+} from "./pending.js";
+import {
+	OM_OBSERVATIONS_DROPPED,
+	OM_OBSERVATIONS_RECORDED,
+	OM_REFLECTIONS_RECORDED,
+	buildExistingObservationsSummary,
+	buildExistingReflectionsSummary,
+	buildObservationsDroppedData,
+	buildObservationsRecordedData,
+	buildReflectionsRecordedData,
+	earlierCoverageMarkerId,
+	foldLedger,
+	findLastCompactionIndex,
+	fullProjection,
+	isSourceEntry,
+	latestCoverageIndex,
+	latestCoverageMarkerId,
+	observationsCreatedAfterIndex,
+	observationToSummaryLine,
+	rawTokensSinceDropCoverage,
+	rawTokensSinceObservationCoverage,
+	rawTokensSinceReflectionCoverage,
+	reflectionToSummaryLine,
+	reflectionsCreatedAfterIndex,
+	type Entry,
+	type Reflection,
+} from "./ledger/index.js";
+
+type ResolvedModel = Extract<ResolveResult, { ok: true }>;
+
+type ConsolidationCtx = {
+	cwd: string;
+	hasUI: boolean;
+	ui?: { notify: (message: string, type?: "warning" | "info" | "error") => void };
+	model: unknown;
+	modelRegistry: any;
+	sessionManager: { getBranch: () => unknown; getSessionId: () => string };
+};
+
+type StageOutcome = "continue" | "abort";
+
+type ReflectorStageResult = {
+	outcome: StageOutcome;
+	sameRunReflections: Reflection[];
+	effectiveReflectionCoverageId?: string;
+};
+
+// Max attempts per stage (primary + all fallbacks the runtime will try internally).
+// Each call to resolveModel tries all non-cooldown candidates.  If the agent throws
+// a retryable error, we record cooldown and call resolveModel again (up to this many times).
+const MAX_STAGE_ATTEMPTS = 10;
+
+function sourceEntriesAfter(entries: Entry[], index: number): Entry[] {
+	return entries.slice(index + 1).filter(isSourceEntry);
+}
+
+/**
+ * Cap source entries to maxTokens by keeping newest entries first,
+ * walking backwards until the token budget is exceeded.
+ * Uses a conservative chars/4 heuristic for token estimation.
+ */
+function capSourceEntriesToTokens(entries: Entry[], maxTokens: number): Entry[] {
+	let totalTokens = 0;
+	const kept: Entry[] = [];
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		let chars = 0;
+		if (entry.type === "message" && entry.message) {
+			const msg = entry.message as any;
+			if (typeof msg.content === "string") chars = msg.content.length;
+			else if (Array.isArray(msg.content)) {
+				for (const block of msg.content) {
+					if (block.text) chars += block.text.length;
+				}
+			}
+		}
+		const estTokens = Math.ceil(chars / 4);
+		if (totalTokens + estTokens > maxTokens && kept.length > 0) break;
+		kept.unshift(entry);
+		totalTokens += estTokens;
+	}
+	return kept;
+}
+
+function appendEntry(pi: ExtensionAPI, customType: string, data: unknown): void {
+	pi.appendEntry(customType, data);
+}
+
+function mergeReflections(existing: Reflection[], additional: Reflection[]): Reflection[] {
+	const seen = new Set(existing.map((reflection) => reflection.id));
+	const merged = [...existing];
+	for (const reflection of additional) {
+		if (seen.has(reflection.id)) continue;
+		seen.add(reflection.id);
+		merged.push(reflection);
+	}
+	return merged;
+}
+
+function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
+	return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
+		|| rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens
+		|| rawTokensSinceDropCoverage(entries) >= runtime.config.reflectAfterTokens;
+}
+
+function stageModelConfig(runtime: Runtime, stage: "observer" | "reflector" | "dropper"): ConfiguredModel | undefined {
+	if (stage === "observer") return runtime.config.observerModel;
+	if (stage === "reflector") return runtime.config.reflectorModel;
+	return runtime.config.dropperModel;
+}
+
+function stageFallbackModels(runtime: Runtime, stage: "observer" | "reflector" | "dropper"): ConfiguredModel[] {
+	if (stage === "observer") return runtime.config.observerFallbackModels ?? [];
+	if (stage === "reflector") return runtime.config.reflectorFallbackModels ?? [];
+	return runtime.config.dropperFallbackModels ?? [];
+}
+
+function stageThinkingLevel(runtime: Runtime, stage: "observer" | "reflector" | "dropper"): ModelThinkingLevel {
+	const stageModel = stageModelConfig(runtime, stage);
+	return stageModel?.thinking ?? runtime.config.model?.thinking ?? "low";
+}
+
+function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "observer" | "reflector" | "dropper") => Promise<ResolvedModel | undefined> {
+	return async (stage) => {
+		const resolved = await runtime.resolveModel({
+			model: ctx.model,
+			modelRegistry: ctx.modelRegistry,
+			hasUI: ctx.hasUI,
+			ui: ctx.ui,
+			stageModel: stageModelConfig(runtime, stage),
+			stageFallbacks: stageFallbackModels(runtime, stage),
+		});
+		if (resolved.ok) {
+			runtime.resolveFailureNotified = false;
+			return resolved;
+		}
+		debugLog(`${stage}.model_unavailable`, { reason: resolved.reason });
+		if (!runtime.resolveFailureNotified && ctx.hasUI && ctx.ui) {
+			ctx.ui.notify(`Observational memory: ${stage} skipped — ${resolved.reason}`, "warning");
+			runtime.resolveFailureNotified = true;
+		}
+		return undefined;
+	};
+}
+
+// ── Trigger registration ────────────────────────────────────────────────────
+
+export function registerConsolidationTrigger(pi: ExtensionAPI, runtime: Runtime): void {
+	const launch = (_event: unknown, ctx: ConsolidationCtx) => {
+		maybeLaunchConsolidation(pi, runtime, ctx);
+	};
+	pi.on("agent_start", launch);
+	pi.on("turn_end", launch);
+}
+
+function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: ConsolidationCtx): void {
+	runtime.ensureConfig(ctx.cwd);
+	if (runtime.config.passive === true) return;
+	if (runtime.config.memory === false) return;
+	if (runtime.consolidationInFlight) return;
+	if (runtime.isConsolidationRetryGated()) return;
+
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	if (!anyStageDue(entries, runtime)) return;
+
+	const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
+	const consolidationCtx: ConsolidationCtx = {
+		cwd: ctx.cwd,
+		hasUI: ctx.hasUI,
+		ui: ctx.ui,
+		model: ctx.model,
+		modelRegistry: ctx.modelRegistry,
+		sessionManager: ctx.sessionManager,
+	};
+
+	void runtime.launchConsolidationTask(ctx, async () => withDebugLogContext({ enabled: runtime.config.debugLog === true, cwd: ctx.cwd, runId }, async () => {
+		await runConsolidationPipeline(pi, runtime, consolidationCtx);
+	}));
+}
+
+// ── Pipeline ─────────────────────────────────────────────────────────────────
+
+export async function runConsolidationPipeline(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+): Promise<void> {
+	const resolveModel = makeModelResolver(runtime, ctx);
+
+	runtime.consolidationPhase = "observer";
+	try {
+		const observerOutcome = await runObserverStage(pi, runtime, ctx, resolveModel);
+		if (observerOutcome === "abort") return;
+	} catch (error) {
+		debugLog("observer.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "observer", error) });
+		return;
+	}
+
+	runtime.consolidationPhase = "reflector";
+	let reflectorResult: ReflectorStageResult;
+	try {
+		reflectorResult = await runReflectorStage(pi, runtime, ctx, resolveModel);
+		if (reflectorResult.outcome === "abort") return;
+	} catch (error) {
+		debugLog("reflector.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "reflector", error) });
+		return;
+	}
+
+	runtime.consolidationPhase = "dropper";
+	try {
+		await runDropperStage(pi, runtime, ctx, resolveModel, reflectorResult.sameRunReflections, reflectorResult.effectiveReflectionCoverageId);
+	} catch (error) {
+		debugLog("dropper.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "dropper", error) });
+	}
+}
+
+// ── Observer stage (with fallback) ──────────────────────────────────────────
+
+async function runObserverStage(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
+): Promise<StageOutcome> {
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	const tokens = rawTokensSinceObservationCoverage(entries);
+	if (tokens < runtime.config.observeAfterTokens) return "continue";
+
+	const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
+	// Mid-session cold start: when no coverage marker exists (e.g., after compaction
+	// consumed the markers), fall back to the last compaction as the cutoff instead
+	// of processing everything including the compaction summary.
+	const effectiveStart = lastCoverageIdx >= 0 ? lastCoverageIdx : findLastCompactionIndex(entries);
+	let chunkEntries = sourceEntriesAfter(entries, effectiveStart);
+	const coversUpToId = chunkEntries.at(-1)?.id;
+	if (!coversUpToId) return "continue";
+
+	// Cap observer input to observerChunkMaxTokens (newest-to-oldest)
+	const maxChunkTokens = runtime.config.observerChunkMaxTokens;
+	if (tokens > maxChunkTokens) {
+		chunkEntries = capSourceEntriesToTokens(chunkEntries, maxChunkTokens);
+	}
+
+	const { text: chunk, sourceEntryIds } = serializeSourceAddressedBranchEntries(chunkEntries);
+	if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
+	const chunkTokens = Math.ceil(chunk.length / 4);
+
+	const memory = fullProjection(entries);
+	const priorReflections = memory.reflections.map(reflectionToSummaryLine);
+	const priorObservations = memory.observations.map(observationToSummaryLine);
+
+	// If noAutoCompact: skip if this exact chunk was already processed
+	const sessionId = ctx.sessionManager.getSessionId();
+	if (runtime.config.noAutoCompact && isObservationChunkPending(sessionId, coversUpToId)) {
+		debugLog("observer.pending_skip", { coversUpToId, sessionId });
+		return "continue";
+	}
+
+	for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
+		const resolved = await resolveModel("observer");
+		if (!resolved) return "abort";
+
+		if (ctx.hasUI) ctx.ui?.notify(
+			`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk (of ${tokens.toLocaleString()} accumulated)`,
+			"info",
+		);
+		debugLog("observer.start", { tokens, coversUpToId, sourceEntryIds, sourceEntryCount: sourceEntryIds.length, priorReflections: priorReflections.length, priorObservations: priorObservations.length });
+
+		try {
+			const result = await runObserver({
+				model: resolved.model as any,
+				apiKey: resolved.apiKey,
+				headers: resolved.headers,
+				priorReflections,
+				priorObservations,
+				chunk,
+				allowedSourceEntryIds: sourceEntryIds,
+				maxTurns: runtime.config.agentMaxTurns,
+				thinkingLevel: stageThinkingLevel(runtime, "observer"),
+			});
+
+			if (result.observations && result.observations.length > 0) {
+				const data = buildObservationsRecordedData(result.observations, coversUpToId);
+				if (!data) return "continue";
+				debugLog("observer.records", { count: result.observations.length, observationTokens: result.observations.reduce((s: number, o: any) => s + o.tokenCount, 0), coversUpToId });
+				if (runtime.config.noAutoCompact) {
+					savePendingObservation(sessionId, { coversUpToId, data });
+					debugLog("observer.pending", { count: result.observations.length, coversUpToId, sessionId });
+				} else {
+					appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
+					debugLog("observer.appended", { count: result.observations.length, coversUpToId });
+				}
+				if (ctx.hasUI) ctx.ui?.notify(`Observational memory: ${result.observations.length} observation${result.observations.length === 1 ? "" : "s"} recorded`, "info");
+				return "continue";
+			}
+
+			// No observations — diagnose the reason for the warning
+			const reason = result.emptyReason;
+			const reasonLabel = reason
+				? reason.kind === "tool_not_called"
+					? "model did not call the observation tool"
+					: reason.kind === "all_rejected"
+						? `${reason.count} observation(s) rejected for invalid sourceEntryIds`
+						: reason.kind === "all_duplicates"
+							? `${reason.count} observation(s) were duplicates of already-recorded entries`
+							: reason.kind === "empty_array"
+								? "model called the tool but submitted an empty observations array"
+								: "nothing new to record"
+				: "unknown reason";
+			const reasonLevel: "info" | "warning" = reason
+				? reason.kind === "no_new_content" || reason.kind === "all_duplicates"
+					? "info"
+					: "warning"
+				: "warning";
+			debugLog("observer.empty", { coversUpToId, reason: reason?.kind });
+			if (ctx.hasUI) ctx.ui?.notify(`Observational memory: no observations — ${reasonLabel}`, reasonLevel);
+			return "continue";
+		} catch (error) {
+			// Always try next fallback — don't abort pipeline for a single model failure.
+			// Record cooldown so resolveModel skips this model in the next iteration.
+			const candidateConfig = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "observer"), stageFallbacks: stageFallbackModels(runtime, "observer") });
+			runtime.recordRetryableError(candidateConfig, error, "observer");
+			debugLog("observer.error", { error: String(error), retryable: isRetryableError(error) });
+			// Continue loop — resolveModel will skip the cooled-down model
+			continue;
+		}
+	}
+
+	// All attempts exhausted
+	runtime.recordConsolidationStageError(ctx, "observer", new Error("Observer: all model candidates exhausted"));
+	return "abort";
+}
+
+// ── Reflector stage (with fallback) ─────────────────────────────────────────
+
+async function runReflectorStage(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	resolveModel: (stage: "reflector") => Promise<ResolvedModel | undefined>,
+): Promise<ReflectorStageResult> {
+	const sessionId = ctx.sessionManager.getSessionId();
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	const reflectionTokens = rawTokensSinceReflectionCoverage(entries);
+	if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
+
+	const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
+	if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
+
+	for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
+		const resolved = await resolveModel("reflector");
+		if (!resolved) return { outcome: "abort", sameRunReflections: [] };
+
+		// Compute ahead for an accurate notification
+		const folded = foldLedger(entries);
+		const lastReflectionIdx = latestCoverageIndex(entries, OM_REFLECTIONS_RECORDED);
+		const newObservations = observationsCreatedAfterIndex(entries, lastReflectionIdx);
+		const newReflections = reflectionsCreatedAfterIndex(entries, lastReflectionIdx);
+		const newItemsTokens = Math.ceil(
+			(newObservations.reduce((s, o) => s + o.content.length, 0) +
+				newReflections.reduce((s, r) => s + r.content.length, 0)) / 4
+		);
+		const summaryBudget = Math.floor(runtime.config.reflectorInputMaxTokens * 0.15) * 2;
+		const reflectorInputTokens = Math.min(newItemsTokens + summaryBudget, runtime.config.reflectorInputMaxTokens);
+		if (ctx.hasUI) ctx.ui?.notify(`Observational memory: reflector running (~${reflectionTokens.toLocaleString()} tokens accumulated, ~${reflectorInputTokens.toLocaleString()}-token input)`, "info");
+
+		try {
+			// Existing memory summaries for context (capped)
+			const existingReflectionsSummary = buildExistingReflectionsSummary(
+				folded.reflections,
+				Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
+			);
+			const existingObservationsSummary = buildExistingObservationsSummary(
+				folded.activeObservations.filter(o => !newObservations.some(no => no.id === o.id)),
+				Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
+			);
+
+			const reflections = await runReflector({
+				model: resolved.model as any,
+				apiKey: resolved.apiKey,
+				headers: resolved.headers,
+				reflections: newReflections,
+				observations: newObservations,
+				existingReflectionsSummary: existingReflectionsSummary || undefined,
+				existingObservationsSummary: existingObservationsSummary || undefined,
+				maxTurns: runtime.config.agentMaxTurns,
+				thinkingLevel: stageThinkingLevel(runtime, "reflector"),
+			});
+
+			if (!reflections || reflections.length === 0) return { outcome: "continue", sameRunReflections: [] };
+
+			const data = buildReflectionsRecordedData(reflections, observationCoverageId);
+			if (!data) return { outcome: "continue", sameRunReflections: [] };
+			if (runtime.config.noAutoCompact) {
+				savePendingReflection(sessionId, { coversUpToId: data.coversUpToId, data });
+			} else {
+				appendEntry(pi, OM_REFLECTIONS_RECORDED, data);
+			}
+			return {
+				outcome: "continue",
+				sameRunReflections: reflections,
+				effectiveReflectionCoverageId: data.coversUpToId,
+			};
+		} catch (error) {
+			const candidateConfig = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "reflector"), stageFallbacks: stageFallbackModels(runtime, "reflector") });
+			runtime.recordRetryableError(candidateConfig, error, "reflector");
+			debugLog("reflector.error", { error: String(error), retryable: isRetryableError(error) });
+			continue;
+		}
+	}
+
+	runtime.recordConsolidationStageError(ctx, "reflector", new Error("Reflector: all model candidates exhausted"));
+	return { outcome: "abort", sameRunReflections: [] };
+}
+
+// ── Dropper stage (with fallback) ───────────────────────────────────────────
+
+async function runDropperStage(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	resolveModel: (stage: "dropper") => Promise<ResolvedModel | undefined>,
+	sameRunReflections: Reflection[],
+	sameRunReflectionCoverageId: string | undefined,
+): Promise<StageOutcome> {
+	const sessionId = ctx.sessionManager.getSessionId();
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	const dropTokens = rawTokensSinceDropCoverage(entries);
+	if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
+
+	const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
+	if (!observationCoverageId) return "continue";
+
+	for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
+		const resolved = await resolveModel("dropper");
+		if (!resolved) return "abort";
+
+		// Compute ahead for an accurate notification
+		const folded = foldLedger(entries);
+		const lastDropIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_DROPPED);
+		const newObservations = observationsCreatedAfterIndex(entries, lastDropIdx);
+		const dropperNewObsTokens = Math.ceil(
+			newObservations.reduce((s, o) => s + o.content.length, 0) / 4
+		);
+		const dropperSummaryBudget = Math.floor(runtime.config.dropperInputMaxTokens * 0.2);
+		const dropperInputTokens = Math.min(dropperNewObsTokens + dropperSummaryBudget, runtime.config.dropperInputMaxTokens);
+		if (ctx.hasUI) ctx.ui?.notify(`Observational memory: dropper running (~${dropTokens.toLocaleString()} tokens accumulated, ~${dropperInputTokens.toLocaleString()}-token input)`, "info");
+
+		try {
+			// Existing active observations summary for context (capped)
+			const existingObservationsSummary = buildExistingObservationsSummary(
+				folded.activeObservations.filter(o => !newObservations.some(no => no.id === o.id)),
+				Math.floor(runtime.config.dropperInputMaxTokens * 0.2),
+			);
+			const reflectionsForDropper = mergeReflections(folded.reflections, sameRunReflections);
+
+			const droppedIds = await runDropper({
+				model: resolved.model as any,
+				apiKey: resolved.apiKey,
+				headers: resolved.headers,
+				reflections: reflectionsForDropper,
+				observations: newObservations,
+				existingObservationsSummary: existingObservationsSummary || undefined,
+				budgetTokens: runtime.config.observationsPoolMaxTokens,
+				maxTurns: runtime.config.agentMaxTurns,
+				thinkingLevel: stageThinkingLevel(runtime, "dropper"),
+			});
+			const latestReflectionCoverageId = latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
+			const effectiveReflectionCoverageId = sameRunReflectionCoverageId ?? latestReflectionCoverageId;
+			const coversUpToId = earlierCoverageMarkerId(entries, observationCoverageId, effectiveReflectionCoverageId);
+			const data = coversUpToId && droppedIds ? buildObservationsDroppedData(droppedIds, coversUpToId) : undefined;
+			if (data && coversUpToId) {
+				if (runtime.config.noAutoCompact) {
+					savePendingDropped(sessionId, { coversUpToId, data });
+				} else {
+					appendEntry(pi, OM_OBSERVATIONS_DROPPED, data);
+				}
+			}
+			return "continue";
+		} catch (error) {
+			const candidateConfig = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "dropper"), stageFallbacks: stageFallbackModels(runtime, "dropper") });
+			runtime.recordRetryableError(candidateConfig, error, "dropper");
+			debugLog("dropper.error", { error: String(error), retryable: isRetryableError(error) });
+			continue;
+		}
+	}
+
+	runtime.recordConsolidationStageError(ctx, "dropper", new Error("Dropper: all model candidates exhausted"));
+	return "abort";
+}

package/src/om/cooldown.ts

@@ -0,0 +1,130 @@
+/**
+ * Model cooldown persistence.
+ *
+ * When a model returns a retryable error (429, 5xx, timeout),
+ * we record a cooldown so it won't be retried for the configured duration.
+ * Cooldowns are persisted to `~/.pi/agent/pi-blackhole/pi-blackhole-cooldown.json`.
+ */
+
+/**
+ * Cooldown persistence for retryable API errors.
+ * 
+ * Created by pi-vcc-om. Records per-model cooldowns to disk so rate-limited
+ * or down models are skipped until their cooldown window expires.
+ *
+ * Key design:
+ * - isCooldownActive reads from disk every call (no in-memory cache needed).
+ * - recordCooldown writes to disk synchronously.
+ * - Cooldowns survive pi restarts via pi-blackhole/pi-blackhole-cooldown.json.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import type { OmModelConfig } from "../core/unified-config.js";
+
+// ── Persistence ─────────────────────────────────────────────────────────────
+
+const CONFIG_DIR = "pi-blackhole";
+const COOLDOWN_FILE = "pi-blackhole-cooldown.json";
+
+function cooldownPath(): string {
+	return join(getAgentDir(), CONFIG_DIR, COOLDOWN_FILE);
+}
+
+export interface CooldownEntry {
+	until: string; // ISO 8601 timestamp
+	reason: string;
+	stage: string; // "observer" | "reflector" | "dropper"
+}
+
+type CooldownMap = Record<string, CooldownEntry>;
+
+/** Provider/id key for cooldown lookup. */
+export function modelKey(model: OmModelConfig): string {
+	return `${model.provider}/${model.id}`;
+}
+
+// ── Load / save ─────────────────────────────────────────────────────────────
+
+function readCooldownMap(): CooldownMap {
+	const path = cooldownPath();
+	if (!existsSync(path)) return {};
+	try {
+		return JSON.parse(readFileSync(path, "utf-8"));
+	} catch {
+		return {};
+	}
+}
+
+function writeCooldownMap(map: CooldownMap): void {
+	const path = cooldownPath();
+	const dir = dirname(path);
+	if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+	writeFileSync(path, `${JSON.stringify(map, null, 2)}\n`);
+}
+
+// ── API ─────────────────────────────────────────────────────────────────────
+
+/**
+ * Check whether a model is currently cooled down.
+ * Expired entries are cleaned up lazily.
+ */
+export function isCooldownActive(model: OmModelConfig, now: Date = new Date()): boolean {
+	const map = readCooldownMap();
+	const key = modelKey(model);
+	const entry = map[key];
+	if (!entry) return false;
+
+	const until = new Date(entry.until);
+	if (isNaN(until.getTime())) return false;
+
+	if (now >= until) {
+		// Expired — clean up
+		delete map[key];
+		writeCooldownMap(map);
+		return false;
+	}
+	return true;
+}
+
+/**
+ * Record a cooldown for a model after a retryable error.
+ *
+ * @param model   The model that failed.
+ * @param reason  Human-readable error reason (e.g. "429 Too Many Requests").
+ * @param stage   Which pipeline stage failed ("observer" | "reflector" | "dropper").
+ */
+export function recordCooldown(model: OmModelConfig, reason: string, stage: string): void {
+	const hours = model.cooldownHours ?? 1;
+	const until = new Date(Date.now() + hours * 3_600_000).toISOString();
+	const map = readCooldownMap();
+	map[modelKey(model)] = { until, reason, stage };
+	writeCooldownMap(map);
+}
+
+/**
+ * Expire all cooldowns whose duration has passed.
+ * Call on session_start or config reload to clean up.
+ */
+export function expireCooldowns(): void {
+	const map = readCooldownMap();
+	const now = new Date();
+	let changed = false;
+	for (const [key, entry] of Object.entries(map)) {
+		const until = new Date(entry.until);
+		if (isNaN(until.getTime()) || now >= until) {
+			delete map[key];
+			changed = true;
+		}
+	}
+	if (changed) writeCooldownMap(map);
+}
+
+/** Regex matching retryable API error messages. */
+const RETRYABLE_ERROR_RE = /(?:\b|^)(?:overloaded|provider|rate\s*limit|too\s+many\s+requests|429|500|502|503|504|timeout|timed?\s*out|network\s*error|connection\s*error|service\s*unavailable|server\s*error|internal\s*error|fetch\s*failed|upstream|websocket\s*closed|retry)(?:\b|$)/i;
+
+/** Check whether an error string or Error indicates a retryable error. */
+export function isRetryableError(error: unknown): boolean {
+	const message = error instanceof Error ? error.message : String(error || "");
+	return RETRYABLE_ERROR_RE.test(message);
+}