pi-blackhole 0.3.9 → 0.4.0

package/src/om/configure-overlay.ts

@@ -9,7 +9,7 @@
 import { visibleWidth } from "./key-matcher.js";
 import { matchesKey, decodeKittyPrintable } from "@earendil-works/pi-tui";
 import { DEFAULTS } from "../core/unified-config.js";
-import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
 import { dirname } from "node:path";
 
 // ---------------------------------------------------------------------------
@@ -68,8 +68,12 @@ const FIELDS: FieldDef[] = [
 		helpText: "Max source entry tokens sent to observer per chunk" },
 	{ key: "observerPreambleMaxTokens", label: "Observer preamble max", type: "number", section: "Observational Memory",
 		helpText: "Preamble budget in manual compaction mode (0=auto-compute 30% of chunk)" },
+	{ key: "dropperPressureThreshold", label: "Dropper pressure threshold", type: "number", section: "Observational Memory",
+		helpText: "Fraction of reflectorInputMaxTokens that triggers pressure-driven dropper (0-1, default 0.70)" },
 	{ key: "agentMaxTurns", label: "Max turns per agent", type: "number", section: "Observational Memory",
 		helpText: "Shared turn cap for background memory agents" },
+	{ key: "fullFoldAlways", label: "Preserve OM on first compaction", type: "boolean", section: "Observational Memory",
+		helpText: "When true, early reflections/drops survive the first compaction in a fresh session" },
 
 	// ── Debug ──
 	{ key: "debug", label: "Debug snapshots", type: "boolean", section: "Debug",
@@ -111,6 +115,7 @@ const EMPTY_THEME: ThemeShim = {
 export interface OverlayResult {
 	saved: boolean;
 	path: string;
+	error?: string;
 }
 
 /**
@@ -136,9 +141,13 @@ export function createConfigureOverlay(
 
 	// Parse config file
 	let raw: Record<string, unknown>;
+	let fileError: string | undefined;
 	try {
 		raw = JSON.parse(readFileSync(configPath, "utf8")) as Record<string, unknown>;
-	} catch {
+	} catch (e) {
+		if (existsSync(configPath)) {
+			fileError = `Config file has invalid JSON: ${(e as Error).message}. Edit the file directly to fix it.`;
+		}
 		raw = {};
 	}
 
@@ -170,7 +179,16 @@ export function createConfigureOverlay(
 						break;
 					case "number": {
 						const num = Number(val);
-						updated[f.def.key] = (val && !isNaN(num)) ? num : raw[f.def.key];
+						if (val && !isNaN(num)) {
+							if (f.def.key === "dropperPressureThreshold") {
+								// Clamp to (0, 1] — runtime validates on reload
+								updated[f.def.key] = Math.min(1, Math.max(0.01, num));
+							} else {
+								updated[f.def.key] = num;
+							}
+						} else {
+							updated[f.def.key] = raw[f.def.key];
+						}
 						break;
 					}
 					case "enum":
@@ -238,8 +256,12 @@ export function createConfigureOverlay(
 
 		// Ctrl+S → save and close
 		if (matchesKey(data, "ctrl+s")) {
+			if (fileError) {
+				done({ saved: false, path: configPath, error: fileError });
+				return;
+			}
 			const saved = save();
-			done({ saved, path: configPath });
+			done({ saved, path: configPath, error: saved ? undefined : "Failed to write config file" });
 			return;
 		}
 
@@ -317,6 +339,16 @@ export function createConfigureOverlay(
 		lines.push(fg("border", `│ ${fg("accent", "Blackhole Configuration")}${" ".repeat(Math.max(0, innerW + 1 - 24))}│`));
 		lines.push(fg("border", `├${"─".repeat(w - 2)}┤`));
 
+		// Error banner when config file has invalid JSON
+		if (fileError) {
+			lines.push(fg("border", `│${" ".repeat(w - 2)}│`));
+			const errorPrefix = fg("error", " ERROR:");
+			const errorText = fg("error", fileError);
+			const errorLine = `│ ${errorPrefix} ${errorText}${" ".repeat(Math.max(0, innerW - visibleWidth(errorPrefix) - 1 - visibleWidth(errorText)))}│`;
+			lines.push(errorLine);
+			lines.push(fg("border", `│${" ".repeat(w - 2)}│`));
+		}
+
 		let currentSection = "";
 
 		for (let i = 0; i < fields.length; i++) {
@@ -375,7 +407,9 @@ export function createConfigureOverlay(
 
 		// Bottom hints
 		lines.push(fg("border", `│${" ".repeat(w - 2)}│`));
-		const hintText = " Ctrl+S save  \u2191\u2193 navigate  Enter toggle  Esc cancel ";
+		const hintText = fileError
+			? " Ctrl+S blocked  ↑↓ navigate  Esc cancel (fix JSON first) "
+			: " Ctrl+S save  \u2191\u2193 navigate  Enter toggle  Esc cancel ";
 		lines.push(fg("border", `│${fg("accent", hintText)}${" ".repeat(Math.max(1, innerW + 2 - visibleWidth(hintText)))}│`));
 		lines.push(fg("border", `╰${"─".repeat(w - 2)}╯`));
 

package/src/om/consolidation.ts

@@ -360,10 +360,8 @@ export function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (sta
 				const fallbackMsg = stageFallbacks.length === 0
 					? "no fallbacks configured"
 					: "no available fallbacks";
-				ctx.ui.notify(
-					`Observational memory: ${stage} skipped — model unavailable (cooldown set to 0, ${fallbackMsg}, will retry next run)`,
-					"info",
-				);
+				runtime.tryEmitInfo(true, ctx.ui,
+					`Observational memory: ${stage} skipped — model unavailable (cooldown set to 0, ${fallbackMsg}, will retry next run)`);
 			} else {
 				ctx.ui.notify(`Observational memory: ${stage} skipped — ${resolved.reason}`, "warning");
 			}
@@ -421,7 +419,7 @@ function validateCursors(entries: Entry[], runtime: Runtime): void {
 }
 
 function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: ConsolidationCtx): void {
-	runtime.ensureConfig(ctx.cwd);
+	runtime.ensureConfig(ctx.cwd, (msg) => ctx.ui?.notify?.(msg, "warning"));
 	if (runtime.config.memory === false) return;
 
 	// LEGACY: passive check — only applies when new keys are absent (unmigrated config)
@@ -617,10 +615,8 @@ async function runObserverStage(
 				if (idx >= 0) effectiveTokens = rawTokensAfterIndex(entries, idx);
 			}
 		}
-		if (ctx.hasUI) ctx.ui?.notify(
-			`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk (of ${effectiveTokens.toLocaleString()} accumulated)`,
-			"info",
-		);
+		runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+			`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk (of ${effectiveTokens.toLocaleString()} accumulated)`);
 		debugLog("observer.start", { tokens, coversUpToId, sourceEntryIds, sourceEntryCount: sourceEntryIds.length, priorReflections: priorReflections.length, priorObservations: priorObservations.length });
 
 		// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
@@ -633,7 +629,8 @@ async function runObserverStage(
 		if (observerEstimatedInput > effectiveObsCtx) {
 			debugLog("observer.context_window_exceeded", { estimatedInput: observerEstimatedInput, effectiveCtx: effectiveObsCtx, model: `${(resolved.model as any).provider}/${(resolved.model as any).id}` });
 			runtime.recordRetryableError(stageModelForThinking, new Error(`context window ${effectiveObsCtx} too small for estimated input ${observerEstimatedInput}`), "observer");
-			if (ctx.hasUI && ctx.ui) ctx.ui.notify(`Observational memory: observer skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveObsCtx.toLocaleString()} too small for ~${observerEstimatedInput.toLocaleString()}-token input)`, "info");
+			runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+				`Observational memory: observer skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveObsCtx.toLocaleString()} too small for ~${observerEstimatedInput.toLocaleString()}-token input)`);
 			continue;
 		}
 
@@ -662,7 +659,8 @@ async function runObserverStage(
 					debugLog("observer.appended", { count: result.observations.length, coversUpToId });
 				}
 				runtime.advanceCursor("observer", coversUpToId, "recorded");
-				if (ctx.hasUI) ctx.ui?.notify(`Observational memory: ${result.observations.length} observation${result.observations.length === 1 ? "" : "s"} recorded`, "info");
+				runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+					`Observational memory: ${result.observations.length} observation${result.observations.length === 1 ? "" : "s"} recorded`);
 				return "continue";
 			}
 
@@ -686,7 +684,12 @@ async function runObserverStage(
 				: "warning";
 			debugLog("observer.empty", { coversUpToId, reason: reason?.kind });
 			runtime.advanceCursor("observer", coversUpToId, "empty");
-			if (ctx.hasUI) ctx.ui?.notify(`Observational memory: no observations — ${reasonLabel}`, reasonLevel);
+			if (reasonLevel === "warning") {
+				if (ctx.hasUI) ctx.ui?.notify(`Observational memory: no observations — ${reasonLabel}`, "warning");
+			} else {
+				runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+					`Observational memory: no observations — ${reasonLabel}`);
+			}
 			return "continue";
 		} catch (error) {
 			// Always try next fallback — don't abort pipeline for a single model failure.
@@ -769,7 +772,8 @@ async function runReflectorStage(
 			}
 		}
 		debugLog("reflector.start", { tokens: effectiveReflectionTokens, inputTokens: reflectorInputTokens, newObsCount: newObservations.length, newRefCount: newReflections.length });
-		if (ctx.hasUI) ctx.ui?.notify(`Observational memory: reflector running (~${effectiveReflectionTokens.toLocaleString()} tokens accumulated, ~${reflectorInputTokens.toLocaleString()}-token input)`, "info");
+		runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+			`Observational memory: reflector running (~${effectiveReflectionTokens.toLocaleString()} tokens accumulated, ~${reflectorInputTokens.toLocaleString()}-token input)`);
 
 		// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
 		const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "reflector"), stageFallbacks: stageFallbackModels(runtime, "reflector") });
@@ -781,7 +785,8 @@ async function runReflectorStage(
 		if (reflectorEstimatedInput > effectiveRefCtx) {
 			debugLog("reflector.context_window_exceeded", { estimatedInput: reflectorEstimatedInput, effectiveCtx: effectiveRefCtx, model: `${(resolved.model as any).provider}/${(resolved.model as any).id}` });
 			runtime.recordRetryableError(stageModelForThinking, new Error(`context window ${effectiveRefCtx} too small for estimated input ${reflectorEstimatedInput}`), "reflector");
-			if (ctx.hasUI && ctx.ui) ctx.ui.notify(`Observational memory: reflector skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveRefCtx.toLocaleString()} too small for ~${reflectorEstimatedInput.toLocaleString()}-token input)`, "info");
+			runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+				`Observational memory: reflector skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveRefCtx.toLocaleString()} too small for ~${reflectorEstimatedInput.toLocaleString()}-token input)`);
 			continue;
 		}
 
@@ -917,7 +922,8 @@ async function runDropperStage(
 				if (idx >= 0) effectiveDropTokens = rawTokensAfterIndex(entries, idx);
 			}
 		}
-		if (ctx.hasUI) ctx.ui?.notify(`Observational memory: dropper running (~${effectiveDropTokens.toLocaleString()} tokens accumulated, ~${dropperInputTokens.toLocaleString()}-token input)`, "info");
+		runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+			`Observational memory: dropper running (~${effectiveDropTokens.toLocaleString()} tokens accumulated, ~${dropperInputTokens.toLocaleString()}-token input)`);
 
 		try {
 			// Existing active observations summary for context (capped).
@@ -948,7 +954,8 @@ async function runDropperStage(
 			if (dropperEstimatedInput > effectiveDropCtx) {
 				debugLog("dropper.context_window_exceeded", { estimatedInput: dropperEstimatedInput, effectiveCtx: effectiveDropCtx, model: `${(resolved.model as any).provider}/${(resolved.model as any).id}` });
 				runtime.recordRetryableError(stageModelForThinking, new Error(`context window ${effectiveDropCtx} too small for estimated input ${dropperEstimatedInput}`), "dropper");
-				if (ctx.hasUI && ctx.ui) ctx.ui.notify(`Observational memory: dropper skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveDropCtx.toLocaleString()} too small for ~${dropperEstimatedInput.toLocaleString()}-token input)`, "info");
+				runtime.tryEmitInfo(ctx.hasUI, ctx.ui,
+					`Observational memory: dropper skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveDropCtx.toLocaleString()} too small for ~${dropperEstimatedInput.toLocaleString()}-token input)`);
 				continue;
 			}
 

package/src/om/ledger/projection.ts

@@ -30,6 +30,7 @@ export type ProjectionDiff = {
 
 export type CompactionProjectionConfig = {
 	observationsPoolMaxTokens: number;
+	fullFoldAlways?: boolean;
 };
 
 export type CompactionProjection = Projection & {
@@ -197,7 +198,11 @@ export function buildCompactionProjection(
 	config: CompactionProjectionConfig,
 ): CompactionProjection {
 	const fullFoldBoundaryId = latestFullFoldBoundaryId(entries);
-	const maintenanceBoundary = fullFoldBoundaryId ? entryBoundary(fullFoldBoundaryId) : noneBoundary();
+	const maintenanceBoundary = fullFoldBoundaryId
+		? entryBoundary(fullFoldBoundaryId)
+		: config.fullFoldAlways
+			? entryBoundary(firstKeptEntryId)
+			: noneBoundary();
 	const normalProjection = foldProjection(entries, {
 		observationsBoundary: entryBoundary(firstKeptEntryId),
 		reflectionsBoundary: maintenanceBoundary,

package/src/om/runtime.ts

@@ -87,14 +87,47 @@ export class Runtime {
 	cursors: PipelineCursors = {};
 	/** Session ID for which cursors have been loaded/validated.  Undefined until first load. */
 	cursorsLoadedSessionId: string | undefined = undefined;
+	/** Info-notification gate: only the first info-level notification per turn/phase is emitted. */
+	hasEmittedInfoThisTurn = false;
 
-	ensureConfig(cwd: string): void {
+	/**
+	 * Emit an info-level notification if none has been emitted this turn/phase yet.
+	 * Returns true if emitted, false if suppressed (already emitted earlier).
+	 */
+	tryEmitInfo(hasUI: boolean, ui: { notify: Notify } | undefined, message: string): boolean {
+		if (!hasUI || !ui || typeof ui.notify !== "function") return false;
+		if (this.hasEmittedInfoThisTurn) return false;
+		this.hasEmittedInfoThisTurn = true;
+		try {
+			ui.notify(message, "info");
+		} catch {
+			// Stale extension context — harmless.
+		}
+		return true;
+	}
+
+	/** Reset the info gate — call at agent_start and agent_end to allow one
+	 *  notification per phase. */
+	resetInfoGate(): void {
+		this.hasEmittedInfoThisTurn = false;
+	}
+
+	ensureConfig(cwd: string, warn?: (message: string) => void): void {
 		if (this.configLoaded) return;
-		this.config = loadConfig(cwd);
+		this.config = loadConfig(cwd, warn);
 		this.configLoaded = true;
 		expireCooldowns();
 	}
 
+	/**
+	 * Force reload config from disk, discarding cached values.
+	 * Call this after external config changes (e.g., overlay save, manual edit).
+	 */
+	reloadConfig(cwd: string, warn?: (message: string) => void): void {
+		this.configLoaded = false;
+		this.ensureConfig(cwd, warn);
+	}
+
 	/**
 	 * Build the ordered model candidate list for a stage:
 	 * 1. Primary stage model (observerModel, reflectorModel, dropperModel)
@@ -139,24 +172,16 @@ export class Runtime {
 
 			// In-memory skip: model failed earlier in this stage with cooldownHours 0
 			if (this.failedInCycle.has(key)) {
-				if (ctx.hasUI && ctx.ui) {
-					ctx.ui.notify(
-						`Observational memory: ${stageName} skipping ${key} (failed this cycle, cooldown disabled)`,
-						"info",
-					);
-				}
+				this.tryEmitInfo(ctx.hasUI, ctx.ui,
+					`Observational memory: ${stageName} skipping ${key} (failed this cycle, cooldown disabled)`);
 				continue;
 			}
 
 			if (isCooldownActive(candidate)) {
-				if (ctx.hasUI && ctx.ui) {
-					const entry = getCooldownEntry(candidate);
-					const reason = entry ? `: ${entry.reason}` : "";
-					ctx.ui.notify(
-						`Observational memory: ${stageName} skipping ${key} (cooldown${reason})`,
-						"info",
-					);
-				}
+				const entry = getCooldownEntry(candidate);
+				const reason = entry ? `: ${entry.reason}` : "";
+				this.tryEmitInfo(ctx.hasUI, ctx.ui,
+					`Observational memory: ${stageName} skipping ${key} (cooldown${reason} — details in cooldown log)`);
 				continue;
 			}
 
@@ -172,7 +197,8 @@ export class Runtime {
 			}
 
 			const auth = await ctx.modelRegistry.getApiKeyAndHeaders(configured);
-			if (!auth.ok || !auth.apiKey) {
+			const hasAuth = ctx.modelRegistry.hasConfiguredAuth?.(configured) ?? true;
+			if (!auth.ok || !hasAuth) {
 				if (ctx.hasUI && ctx.ui) {
 					ctx.ui.notify(
 						`Observational memory: ${stageName} no auth for ${candidate.provider}`,
@@ -185,7 +211,7 @@ export class Runtime {
 			return {
 				ok: true,
 				model: configured,
-				apiKey: auth.apiKey as string,
+				apiKey: (auth.apiKey as string) ?? "",
 				headers: auth.headers as Record<string, string> | undefined,
 				cooldownApplied: false,
 			};
@@ -199,29 +225,25 @@ export class Runtime {
 			}
 
 			const auth = await ctx.modelRegistry.getApiKeyAndHeaders(sessionModel);
-			if (!auth.ok || !auth.apiKey) {
+			const hasAuth = ctx.modelRegistry.hasConfiguredAuth?.(sessionModel) ?? true;
+			if (!auth.ok || !hasAuth) {
 				const provider = (sessionModel as { provider?: string }).provider ?? "unknown";
-				return { ok: false, reason: `no API key for session model provider "${provider}"` };
+				return { ok: false, reason: `no auth for session model provider "${provider}"` };
 			}
 
 			return {
 				ok: true,
 				model: sessionModel,
-				apiKey: auth.apiKey as string,
+				apiKey: (auth.apiKey as string) ?? "",
 				headers: auth.headers as Record<string, string> | undefined,
 				cooldownApplied: false,
 			};
 		}
 
 		// All configured candidates exhausted and session fallback disabled —
-		// skip the stage entirely.  Info-level to match cooldown-disabled pattern.
-		// Set resolveFailureNotified so the consolidation layer doesn't duplicate.
-		if (ctx.hasUI && ctx.ui) {
-			ctx.ui.notify(
-				`Observational memory: ${stageName} skipped — all candidates failed (sessionFallback disabled, won't use main model)`,
-				"info",
-			);
-		}
+		// skip the stage entirely.
+		this.tryEmitInfo(ctx.hasUI, ctx.ui,
+			`Observational memory: ${stageName} skipped — all candidates failed (sessionFallback disabled, won't use main model)`);
 		this.resolveFailureNotified = true;
 
 		return { ok: false, reason: `no model available for ${stageName} (all candidates exhausted, sessionFallback disabled)` };
@@ -256,8 +278,13 @@ export class Runtime {
 			this.failedInCycle.add(modelKey(modelConfig));
 			return;
 		}
-		const reason = error instanceof Error ? error.message : String(error || "unknown error");
-		recordCooldown(modelConfig, reason, stage);
+		const rawReason = error instanceof Error ? error.message : String(error || "unknown error");
+		// Strip trailing JSON body from API error messages for display cleanliness.
+		// To avoid stripping non-JSON braces like "{host}", only strip if the text
+		// after the brace pair consists solely of whitespace (i.e. JSON is at end).
+		// The full rawReason is NOT stored in the cooldown log — only this brief form.
+		const brief = rawReason.replace(/\s*\{[\s\S]*?\}\s*$/, "").trim();
+		recordCooldown(modelConfig, brief, stage);
 	}
 
 	/**