pi-blackhole 0.3.7 → 0.3.9

package/README.md

@@ -1,20 +1,6 @@
 # pi-blackhole
 
-> [!IMPORTANT]
-> **Blackhole is the default compaction engine** (`compactionEngine: "blackhole"`, `compaction: "auto"`). Auto-compaction fires at the configured threshold using blackhole's pipeline — both auto-trigger and Pi's `/compact` command use it. No additional setup needed. To work automatically, blackhole needs to register its hook so it overrides /compact with the `blackhole` compaction engine. Opting out or opting for manual compaction with blackhole can be done as below:
->
-> | Setting | Auto-trigger after threshold | `/compact` (Pi built-in) |
-> |---|---|---|
-> | `"auto"` + `"blackhole"` (default) | blackhole handles ✓ | blackhole handles |
-> | `"auto"` + `"pi-default"` | Pi handles | Pi handles |
-> | `"manual"` + any | skipped | Pi handles ✓ |
-> | `"off"` + any | skipped (Pi handles) | Pi handles ✓ |
->
-> The `/blackhole` command always uses blackhole's pipeline regardless of settings.
->
-> **Upgrading from an older version?** This version replaces legacy keys (`passive`, `noAutoCompact`, `overrideDefaultCompaction`) with `compaction`, `compactionEngine`, and `tailBehavior`. Automatic migration runs at startup — old configs continue to work. Best effort was made to preserve existing behavior, but review [`MIGRATION-GUIDE.md`](MIGRATION-GUIDE.md) if something behaves differently.
->
-> See [`CONFIG.md`](CONFIG.md) for the full reference.
+
 
 **Algorithmic compaction + session-aware observational memory for [Pi](https://github.com/badlogic/pi-mono) — in one unified extension.**
 
@@ -291,6 +277,7 @@ Everything else has sensible defaults.
 | `observationsPoolTargetTokens` | `10000` | Target size dropper aims for after pruning (derived: half of pool max) |
 | `reflectorInputMaxTokens` | `80000` | Max reflector input budget |
 | `dropperInputMaxTokens` | `80000` | Max dropper input budget |
+| `dropperPressureThreshold` | `0.70` | Fraction of `reflectorInputMaxTokens` at which dropper runs even without new data (pressure relief valve) |
 | `agentMaxTurns` | `16` | Max agent-loop turns per worker per run |
 | `debug` | `false` | Pre-compaction snapshot to `/tmp/pi-blackhole-debug.json` |
 | `debugLog` | `false` | Continuous JSONL debug log to `~/.pi/agent/pi-blackhole/debug.ndjson` |
@@ -313,7 +300,8 @@ Paste the appropriate block into your config to match your main session model's
   "observerPreambleMaxTokens": 0,
   "observationsPoolMaxTokens": 8000,
   "reflectorInputMaxTokens": 30000,
-  "dropperInputMaxTokens": 30000
+  "dropperInputMaxTokens": 30000,
+  "dropperPressureThreshold": 0.70
 }
 ```
 
@@ -330,7 +318,8 @@ These are the built-in defaults. If you reset your config, these are what you ge
   "observerPreambleMaxTokens": 0,
   "observationsPoolMaxTokens": 20000,
   "reflectorInputMaxTokens": 80000,
-  "dropperInputMaxTokens": 80000
+  "dropperInputMaxTokens": 80000,
+  "dropperPressureThreshold": 0.70
 }
 ```
 
@@ -345,7 +334,8 @@ These are the built-in defaults. If you reset your config, these are what you ge
   "observerPreambleMaxTokens": 0,
   "observationsPoolMaxTokens": 40000,
   "reflectorInputMaxTokens": 160000,
-  "dropperInputMaxTokens": 160000
+  "dropperInputMaxTokens": 160000,
+  "dropperPressureThreshold": 0.70
 }
 ```
 

package/example-config.json

@@ -95,6 +95,7 @@
   "observationsPoolTargetTokens": 10000,
   "reflectorInputMaxTokens": 80000,
   "dropperInputMaxTokens": 80000,
+  "dropperPressureThreshold": 0.70,
   "observerChunkMaxTokens": 40000,
   "observerPreambleMaxTokens": 0,
   "agentMaxTurns": 16,
@@ -110,6 +111,7 @@
     "  compactionEngine (blackhole|pi-default) — who handles it",
     "  tailBehavior (pi-default|minimal) — how much stays visible",
     "  sessionFallback (true|false) — if false, skip session model as last-resort fallback",
+    "  dropperPressureThreshold (0-1, default 0.70) — pressure relief valve: fraction of reflectorInputMaxTokens where dropper fires even without new data",
     "",
     "Backward compat: old keys (overrideDefaultCompaction, noAutoCompact, passive)",
     "are migrated to new keys at load time.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-blackhole",
-  "version": "0.3.7",
+  "version": "0.3.9",
   "packageManager": "pnpm@11.2.2",
   "description": "Unified compaction + observational memory extension for Pi — compresses conversation context while preserving durable observations and reflections",
   "license": "MIT",

package/src/core/unified-config.ts

@@ -90,6 +90,11 @@ export interface UnifiedConfig {
 	reflectorInputMaxTokens: number;
 	/** Max prompt tokens for dropper model input (rolling window cap). */
 	dropperInputMaxTokens: number;
+	/** Pressure threshold for dropper.  When active observation pool tokens exceed
+	 *  this fraction of reflectorInputMaxTokens, the dropper runs even without new
+	 *  observations/reflections (to keep the pool pruned).
+	 *  Default 0.70 (70%). Must be in range (0, 1]. */
+	dropperPressureThreshold: number;
 	/** Max source entries tokens sent to observer per chunk. */
 	observerChunkMaxTokens: number;
 	/** Max preamble tokens (CURRENT REFLECTIONS / OBSERVATIONS) in the observer prompt.
@@ -149,6 +154,7 @@ export const DEFAULTS: UnifiedConfig = {
 	observationsPoolTargetTokens: 10_000,
 	reflectorInputMaxTokens: 80_000,
 	dropperInputMaxTokens: 80_000,
+	dropperPressureThreshold: 0.70,
 	observerChunkMaxTokens: 40_000,
 	observerPreambleMaxTokens: 0,
 	agentMaxTurns: 16,
@@ -238,6 +244,11 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
 
 	// Numeric fields — use nonNegativeInt for observerPreambleMaxTokens (0 = auto)
 	const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "observationsPoolTargetTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "observerPreambleMaxTokens", "agentMaxTurns"] as const;
+
+	// dropperPressureThreshold: fractional, must be in (0, 1]
+	if (typeof raw.dropperPressureThreshold === "number" && Number.isFinite(raw.dropperPressureThreshold) && raw.dropperPressureThreshold > 0 && raw.dropperPressureThreshold <= 1) {
+		c.dropperPressureThreshold = raw.dropperPressureThreshold;
+	}
 	for (const k of numKeys) {
 		// observerPreambleMaxTokens accepts 0 (auto-compute); everything else must be > 0
 		const validator = k === "observerPreambleMaxTokens" ? nonNegativeInt : positiveInt;
@@ -414,6 +425,11 @@ export function loadUnifiedConfig(cwd: string): UnifiedConfig {
 
 
 
+	// Validate dropperPressureThreshold — must be in (0, 1]
+	if (typeof merged.dropperPressureThreshold !== "number" || !Number.isFinite(merged.dropperPressureThreshold) || merged.dropperPressureThreshold <= 0 || merged.dropperPressureThreshold > 1) {
+		merged.dropperPressureThreshold = DEFAULTS.dropperPressureThreshold;
+	}
+
 	// Derive observationsPoolTargetTokens if still unset or invalid (must be < max)
 	if (
 		merged.observationsPoolTargetTokens === undefined ||

package/src/om/compaction-trigger.ts

@@ -27,6 +27,17 @@ function notifySafely(hasUI: boolean, ui: any, message: string, level: "info" |
 }
 
 export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
+	pi.on("agent_start", () => {
+		// A new turn is starting — abort any pending auto-compaction wait.
+		// The new turn's own agent_end will re-evaluate the threshold and
+		// schedule a fresh wait if compaction is still needed.
+		if (runtime.autoCompactionController) {
+			runtime.autoCompactionController.abort();
+			runtime.autoCompactionController = null;
+			runtime.compactInFlight = false;
+		}
+	});
+
 	pi.on("agent_end", (event: any, ctx: any) => {
 		try {
 			handleAgentEnd(event, ctx, runtime);
@@ -132,17 +143,51 @@ function handleAgentEnd(event: any, ctx: any, runtime: Runtime): void {
 			"info",
 		);
 
-		runtime.compactInFlight = true;
-		dbg("compaction_trigger.scheduled", { compactInFlight: runtime.compactInFlight });
+	runtime.compactInFlight = true;
+	const controller = new AbortController();
+	runtime.autoCompactionController = controller;
+	const signal = controller.signal;
+	dbg("compaction_trigger.scheduled", { compactInFlight: runtime.compactInFlight });
+
+	// Issue #31: keep waiting for the agent to become idle instead of bailing
+	// after the first non-idle check. The agent may need a few hundred ms to
+	// finish async work from other extension handlers (e.g. pi-rewind's
+	// checkpoint I/O) before it is truly idle. The only legitimate cancellation
+	// is the agent_start handler above aborting the controller.
+	void (async () => {
+		try {
+			// Yield to the event loop first — matches the historical
+			// setTimeout(0) deferral that lets other agent_end listeners run.
+			await new Promise((resolve) => setTimeout(resolve, 0));
+
+			// Poll isIdle() every 200ms until it returns true. No max-retries
+			// cap: the user can be reading the response for arbitrarily long.
+			// ctx.compact() itself aborts any in-flight agent operation, so we
+			// must wait until the agent is truly idle.
+			let isIdle = false;
+			while (!isIdle) {
+				if (signal.aborted) {
+					dbg("compaction_trigger.microtask.bail", { reason: "aborted_agent_start" });
+					return;
+				}
 
-		setTimeout(() => {
-			dbg("compaction_trigger.microtask.enter", {});
-			try {
 				// Validate session identity — bail if the session was replaced/reloaded.
-				const currentSessionId = ctx.sessionManager.getSessionId();
+				let currentSessionId: string;
+				try {
+					currentSessionId = ctx.sessionManager.getSessionId();
+				} catch (error) {
+					if (isStaleExtensionContextError(error)) {
+						runtime.compactInFlight = false;
+						runtime.autoCompactionController = null;
+						dbg("compaction_trigger.microtask.bail", { reason: "stale_ctx" });
+						return;
+					}
+					throw error;
+				}
 				dbg("compaction_trigger.microtask.session_check", { currentSessionId, expectedSessionId: sessionId, match: currentSessionId === sessionId });
 				if (currentSessionId !== sessionId) {
 					runtime.compactInFlight = false;
+					runtime.autoCompactionController = null;
 					dbg("compaction_trigger.microtask.bail", { reason: "session_changed" });
 					notifySafely(
 						hasUI,
@@ -153,60 +198,76 @@ function handleAgentEnd(event: any, ctx: any, runtime: Runtime): void {
 					return;
 				}
 
-				const isIdle = ctx.isIdle();
+				isIdle = ctx.isIdle();
 				dbg("compaction_trigger.microtask.idle_check", { isIdle });
 				if (!isIdle) {
-					runtime.compactInFlight = false;
-					dbg("compaction_trigger.microtask.bail", { reason: "not_idle" });
-					notifySafely(
-						hasUI,
-						ui,
-						"Observational memory: compaction deferred — agent became busy before compaction",
-						"info",
-					);
-					return;
-				}
-				const currentEntries = ctx.sessionManager.getBranch() as Entry[];
-				const currentTokens = rawTokensSinceLastCompaction(currentEntries);
-				dbg("compaction_trigger.microtask.recheck_tokens", { currentTokens, threshold: runtime.config.compactAfterTokens, ok: currentTokens >= runtime.config.compactAfterTokens });
-				if (currentTokens < runtime.config.compactAfterTokens) {
-					runtime.compactInFlight = false;
-					dbg("compaction_trigger.microtask.bail", { reason: "pressure_relieved", currentTokens, threshold: runtime.config.compactAfterTokens });
-					notifySafely(
-						hasUI,
-						ui,
-						"Observational memory: compaction skipped — another compaction already ran before deferred compaction",
-						"info",
-					);
-					return;
-				}
-
-				dbg("compaction_trigger.microtask.calling_compact", {});
-				ctx.compact({
-					onComplete: (result: any) => {
-						runtime.compactInFlight = false;
-						dbg("compaction_trigger.onComplete", { result: !!result });
-						notifySafely(hasUI, ui, "Observational memory: compaction complete", "info");
-					},
-					onError: (error: { message: string }) => {
-						runtime.compactInFlight = false;
-						dbg("compaction_trigger.onError", { message: error?.message ?? String(error) });
-						if (error.message === "Compaction cancelled") {
-							// We already notified the user with the real reason before returning { cancel: true }.
+					// Sleep in 50ms slices so agent_start aborts are noticed quickly.
+					// A single 200ms await would let the loop run for 200ms after
+					// the user typed — too long, since we want compaction to wait
+					// only for the agent to settle, not for a full tick.
+					const sliceMs = 50;
+					const end = Date.now() + 200;
+					while (Date.now() < end) {
+						if (signal.aborted) {
+							dbg("compaction_trigger.microtask.bail", { reason: "aborted_agent_start" });
 							return;
 						}
-						notifySafely(hasUI, ui, `Observational memory: ${error.message}`, "error");
-					},
-				});
-			} catch (error) {
-				runtime.compactInFlight = false;
-				const msg = getErrorMessage(error);
-				if (isStaleExtensionContextError(error)) {
-					dbg("compaction_trigger.microtask.bail", { reason: "stale_ctx", message: msg });
-					return;
+						await new Promise((resolve) => setTimeout(resolve, sliceMs));
+					}
 				}
-				dbg("compaction_trigger.microtask.error", { message: msg });
-				notifySafely(hasUI, ui, `Observational memory: compact threw: ${msg}`, "error");
 			}
-	}, 0);
+
+			if (signal.aborted) {
+				dbg("compaction_trigger.microtask.bail", { reason: "aborted_agent_start" });
+				return;
+			}
+
+			const currentEntries = ctx.sessionManager.getBranch() as Entry[];
+			const currentTokens = rawTokensSinceLastCompaction(currentEntries);
+			dbg("compaction_trigger.microtask.recheck_tokens", { currentTokens, threshold: runtime.config.compactAfterTokens, ok: currentTokens >= runtime.config.compactAfterTokens });
+			if (currentTokens < runtime.config.compactAfterTokens) {
+				runtime.compactInFlight = false;
+				runtime.autoCompactionController = null;
+				dbg("compaction_trigger.microtask.bail", { reason: "pressure_relieved", currentTokens, threshold: runtime.config.compactAfterTokens });
+				notifySafely(
+					hasUI,
+					ui,
+					"Observational memory: compaction skipped — another compaction already ran before deferred compaction",
+					"info",
+				);
+				return;
+			}
+
+			dbg("compaction_trigger.microtask.calling_compact", {});
+			// Compaction is now actually starting — clear the controller so
+			// agent_start doesn't abort an in-progress compact.
+			runtime.autoCompactionController = null;
+			ctx.compact({
+				onComplete: (result: any) => {
+					runtime.compactInFlight = false;
+					dbg("compaction_trigger.onComplete", { result: !!result });
+					notifySafely(hasUI, ui, "Observational memory: compaction complete", "info");
+				},
+				onError: (error: { message: string }) => {
+					runtime.compactInFlight = false;
+					dbg("compaction_trigger.onError", { message: error?.message ?? String(error) });
+					if (error.message === "Compaction cancelled") {
+						// We already notified the user with the real reason before returning { cancel: true }.
+						return;
+					}
+					notifySafely(hasUI, ui, `Observational memory: ${error.message}`, "error");
+				},
+			});
+		} catch (error) {
+			runtime.compactInFlight = false;
+			runtime.autoCompactionController = null;
+			const msg = getErrorMessage(error);
+			if (isStaleExtensionContextError(error)) {
+				dbg("compaction_trigger.microtask.bail", { reason: "stale_ctx", message: msg });
+				return;
+			}
+			dbg("compaction_trigger.microtask.error", { message: msg });
+			notifySafely(hasUI, ui, `Observational memory: compact threw: ${msg}`, "error");
+		}
+	})();
 }

package/src/om/consolidation.ts

@@ -178,10 +178,148 @@ function pendingObservationsCreatedAfter(
 	return newObs;
 }
 
-function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
-	return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
-		|| rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens
-		|| rawTokensSinceDropCoverage(entries) >= runtime.config.reflectAfterTokens;
+/** Cursor-aware stage-due check.  Uses cursors when available; falls back to
+ *  legacy coverage markers when cursors are absent (cold start, fork recovery).
+ *
+ *  In compaction: "manual" mode, the branch has no OM markers — observations
+ *  live in the per‑session pending file.  `pending` provides the pool fullness
+ *  and new‑data visibility that the reflector/dropper checks need. */
+export function anyStageDue(entries: Entry[], runtime: Runtime, pending?: PendingOMState): boolean {
+	const config = runtime.config;
+	const cursors = runtime.cursors ?? {};
+
+	// ── Observer ──────────────────────────────────────────────────────────
+	const observerDue = (() => {
+		const cursor = cursors.observer;
+		if (!cursor) {
+			return rawTokensSinceObservationCoverage(entries) >= config.observeAfterTokens;
+		}
+		const idx = entryIndexForId(entries, cursor.entryId);
+		const tokensSince = idx >= 0
+			? rawTokensAfterIndex(entries, idx)
+			: rawTokensSinceObservationCoverage(entries);
+		return tokensSince >= config.observeAfterTokens;
+	})();
+
+	// ── Reflector ─────────────────────────────────────────────────────────
+	const reflectorDue = (() => {
+		const cursor = cursors.reflector;
+		if (!cursor) {
+			return rawTokensSinceReflectionCoverage(entries) >= config.reflectAfterTokens;
+		}
+		const idx = entryIndexForId(entries, cursor.entryId);
+		if (idx < 0) {
+			return rawTokensSinceReflectionCoverage(entries) >= config.reflectAfterTokens;
+		}
+		// Must have enough accumulated tokens before considering reflector
+		const tokensSince = rawTokensAfterIndex(entries, idx);
+		if (tokensSince < config.reflectAfterTokens) {
+			return false;
+		}
+		// Check for new observation batches after the cursor
+		for (let i = idx + 1; i < entries.length; i++) {
+			const e = entries[i];
+			if (e.type === "custom" && e.customType === OM_OBSERVATIONS_RECORDED) {
+				// Skip if this marker's coversUpToId is at or before the cursor
+				// — data it covers was already processed.
+				const markerCoversUpTo: string | undefined = (e as any).data?.coversUpToId;
+				if (markerCoversUpTo) {
+					const markerCoversIdx = entryIndexForId(entries, markerCoversUpTo);
+					if (markerCoversIdx >= 0 && markerCoversIdx <= idx) continue;
+				}
+				return true;
+			}
+		}
+		// In manual mode, also check pending observation batches that arrived
+		// after the cursor (since branch has no OM markers).
+		if (pending) {
+			const pendingBatches = pending.observationBatches ?? [];
+			for (const batch of pendingBatches) {
+				if (batch.coversUpToId) {
+					const batchIdx = entryIndexForId(entries, batch.coversUpToId);
+					if (batchIdx >= 0 && batchIdx > idx) return true;
+				}
+			}
+		}
+		return false;
+	})();
+
+	// ── Dropper ───────────────────────────────────────────────────────────
+	// Short‑circuit: only compute dropperDue when observer and reflector are
+	// both not due — if either is due, the pipeline launches anyway.
+	const dropperDue = observerDue || reflectorDue ? false : (() => {
+		// Compute active observation pool tokens (branch + pending in manual mode)
+		const folded = foldLedger(entries);
+		let poolTokens = folded.activeObservations.reduce(
+			(s: number, o: Observation) => s + (o.tokenCount ?? 0),
+			0,
+		);
+		// In manual mode, include pending observation batches
+		if (pending) {
+			const pendingBatches = pending.observationBatches ?? [];
+			for (const batch of pendingBatches) {
+				poolTokens += ((batch.data as any)?.observations ?? []).reduce(
+					(s: number, o: any) => s + (o.tokenCount ?? 0), 0,
+				);
+			}
+		}
+		const fullnessVsPool = config.observationsPoolMaxTokens > 0
+			? poolTokens / config.observationsPoolMaxTokens
+			: 0;
+
+		// Must have at least 10% fullness to consider dropper
+		if (fullnessVsPool < 0.10) return false;
+
+		// Pressure check: pool ≥ threshold × reflectorInputMaxTokens
+		const pressure = poolTokens >= config.dropperPressureThreshold * config.reflectorInputMaxTokens;
+		if (pressure) return true;
+
+		// New data check: new obs or ref batches after dropper cursor
+		const cursor = cursors.dropper;
+		if (!cursor) {
+			// In manual mode, pending batches are the only source of new‑data
+			// visibility (branch has no OM markers).
+			const hasPendingNewData = pending
+				? (pending.observationBatches?.length ?? 0) > 0 || (pending.reflectionBatches?.length ?? 0) > 0
+				: false;
+			if (hasPendingNewData) return true;
+			return rawTokensSinceDropCoverage(entries) >= config.reflectAfterTokens;
+		}
+		const idx = entryIndexForId(entries, cursor.entryId);
+		if (idx < 0) {
+			return rawTokensSinceDropCoverage(entries) >= config.reflectAfterTokens;
+		}
+		// Must have enough accumulated tokens before considering dropper
+		const tokensSince = rawTokensAfterIndex(entries, idx);
+		if (tokensSince < config.reflectAfterTokens) {
+			return false;
+		}
+		for (let i = idx + 1; i < entries.length; i++) {
+			const e = entries[i];
+			if (e.type === "custom" && (e.customType === OM_OBSERVATIONS_RECORDED || e.customType === OM_REFLECTIONS_RECORDED)) {
+				const markerCoversUpTo: string | undefined = (e as any).data?.coversUpToId;
+				if (markerCoversUpTo) {
+					const markerCoversIdx = entryIndexForId(entries, markerCoversUpTo);
+					if (markerCoversIdx >= 0 && markerCoversIdx <= idx) continue;
+				}
+				return true;
+			}
+		}
+		// In manual mode, also check pending batches after the cursor
+		if (pending) {
+			const pendingObs = pending.observationBatches ?? [];
+			const pendingRef = pending.reflectionBatches ?? [];
+			for (const batch of [...pendingObs, ...pendingRef]) {
+				if (batch.coversUpToId) {
+					const batchIdx = entryIndexForId(entries, batch.coversUpToId);
+					if (batchIdx >= 0 && batchIdx > idx) return true;
+				}
+			}
+		}
+		return false;
+	})();
+
+	return observerDue || reflectorDue || dropperDue;
 }
 
 function stageModelConfig(runtime: Runtime, stage: "observer" | "reflector" | "dropper"): ConfiguredModel | undefined {
@@ -245,6 +383,43 @@ export function registerConsolidationTrigger(pi: ExtensionAPI, runtime: Runtime)
 	pi.on("turn_end", launch);
 }
 
+/** Validate cursors against the current branch.  If a cursor's entry ID no longer
+ *  exists in the branch (fork, navigation, compaction), fall back to the best
+ *  available coverage marker for that stage. */
+function validateCursors(entries: Entry[], runtime: Runtime): void {
+	const cursors = runtime.cursors ?? {};
+
+	// Observer: fall back to latest OM_OBSERVATIONS_RECORDED marker
+	if (cursors.observer && entryIndexForId(entries, cursors.observer.entryId) < 0) {
+		const markerId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
+		if (markerId) {
+			cursors.observer = { entryId: markerId, state: "initial" };
+		} else {
+			delete cursors.observer;
+		}
+	}
+
+	// Reflector: fall back to latest OM_REFLECTIONS_RECORDED marker
+	if (cursors.reflector && entryIndexForId(entries, cursors.reflector.entryId) < 0) {
+		const markerId = latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
+		if (markerId) {
+			cursors.reflector = { entryId: markerId, state: "initial" };
+		} else {
+			delete cursors.reflector;
+		}
+	}
+
+	// Dropper: fall back to latest OM_OBSERVATIONS_DROPPED marker
+	if (cursors.dropper && entryIndexForId(entries, cursors.dropper.entryId) < 0) {
+		const markerId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_DROPPED);
+		if (markerId) {
+			cursors.dropper = { entryId: markerId, state: "initial" };
+		} else {
+			delete cursors.dropper;
+		}
+	}
+}
+
 function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: ConsolidationCtx): void {
 	runtime.ensureConfig(ctx.cwd);
 	if (runtime.config.memory === false) return;
@@ -256,8 +431,28 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 	if (runtime.consolidationInFlight) return;
 	if (runtime.isConsolidationRetryGated()) return;
 
+	// Load and validate cursors from pending file (once per session; re-load on fork)
+	const sessionId = ctx.sessionManager.getSessionId();
+	if (runtime.cursorsLoadedSessionId !== sessionId) {
+		if (typeof runtime.loadCursorsFromPending === "function") {
+			runtime.loadCursorsFromPending(sessionId);
+		}
+		const entries = ctx.sessionManager.getBranch() as Entry[];
+		validateCursors(entries, runtime);
+		runtime.cursorsLoadedSessionId = sessionId;
+		const c = runtime.cursors ?? {};
+		debugLog("cursor.loaded", {
+			observer: c.observer ?? null,
+			reflector: c.reflector ?? null,
+			dropper: c.dropper ?? null,
+		});
+	}
+
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	if (!anyStageDue(entries, runtime)) return;
+	// In manual mode, the branch has no OM markers — pending state provides
+	// pool fullness and new‑data visibility for reflector/dropper checks.
+	const pending = runtime.config.noAutoCompact ? readPendingState(sessionId) : undefined;
+	if (!anyStageDue(entries, runtime, pending)) return;
 
 	const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
 	const consolidationCtx: ConsolidationCtx = {
@@ -314,6 +509,16 @@ export async function runConsolidationPipeline(
 	} catch (error) {
 		debugLog("dropper.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "dropper", error) });
 	}
+
+	// Flush cursors to pending file after all stages complete (non‑blocking)
+	const sessionId = ctx.sessionManager.getSessionId();
+	runtime.scheduleCursorFlush(sessionId);
+	const c = runtime.cursors ?? {};
+	debugLog("cursor.saved", {
+		observer: c.observer ?? null,
+		reflector: c.reflector ?? null,
+		dropper: c.dropper ?? null,
+	});
 }
 
 // ── Observer stage (with fallback) ──────────────────────────────────────────
@@ -325,14 +530,27 @@ async function runObserverStage(
 	resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
 ): Promise<StageOutcome> {
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	const tokens = rawTokensSinceObservationCoverage(entries);
-	if (tokens < runtime.config.observeAfterTokens) return "continue";
-
-	const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
-	// Mid-session cold start: when no coverage marker exists (e.g., after compaction
-	// consumed the markers), fall back to the last compaction as the cutoff instead
-	// of processing everything including the compaction summary.
-	const effectiveStart = lastCoverageIdx >= 0 ? lastCoverageIdx : findLastCompactionIndex(entries);
+	const sessionId = ctx.sessionManager.getSessionId();
+
+	// Determine start index: cursor takes priority, fall back to coverage markers
+	const observerCursor = runtime.getCursor("observer");
+	let effectiveStart: number;
+	if (observerCursor) {
+		const cursorIdx = entryIndexForId(entries, observerCursor.entryId);
+		effectiveStart = cursorIdx >= 0 ? cursorIdx : latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
+	} else {
+		const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
+		effectiveStart = lastCoverageIdx >= 0 ? lastCoverageIdx : findLastCompactionIndex(entries);
+	}
+
+	const tokens = effectiveStart >= 0 ? rawTokensAfterIndex(entries, effectiveStart) : 0;
+	if (tokens < runtime.config.observeAfterTokens) {
+		// Not due — advance cursor to last source entry so we don't re-check immediately
+		const lastSourceId = [...entries].reverse().find((e: Entry) => isSourceEntry(e))?.id;
+		if (lastSourceId) runtime.advanceCursor("observer", lastSourceId, "not_due");
+		return "continue";
+	}
+
 	let chunkEntries = sourceEntriesAfter(entries, effectiveStart);
 
 	// Cap observer input to observerChunkMaxTokens (newest-to-oldest)
@@ -349,8 +567,6 @@ async function runObserverStage(
 	if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
 	const chunkTokens = Math.ceil(chunk.length / 4);
 
-	const sessionId = ctx.sessionManager.getSessionId();
-
 	const memory = fullProjection(entries);
 	let priorReflections = memory.reflections.map(reflectionToSummaryLine);
 	let priorObservations = memory.observations.map(observationToSummaryLine);
@@ -436,7 +652,7 @@ async function runObserverStage(
 
 			if (result.observations && result.observations.length > 0) {
 				const data = buildObservationsRecordedData(result.observations, coversUpToId);
-				if (!data) return "continue";
+				if (!data) { runtime.advanceCursor("observer", coversUpToId, "empty"); return "continue"; }
 				debugLog("observer.records", { count: result.observations.length, observationTokens: result.observations.reduce((s: number, o: any) => s + o.tokenCount, 0), coversUpToId });
 				if (runtime.config.noAutoCompact) {
 					savePendingObservation(sessionId, { coversUpToId, data });
@@ -445,6 +661,7 @@ async function runObserverStage(
 					appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
 					debugLog("observer.appended", { count: result.observations.length, coversUpToId });
 				}
+				runtime.advanceCursor("observer", coversUpToId, "recorded");
 				if (ctx.hasUI) ctx.ui?.notify(`Observational memory: ${result.observations.length} observation${result.observations.length === 1 ? "" : "s"} recorded`, "info");
 				return "continue";
 			}
@@ -468,6 +685,7 @@ async function runObserverStage(
 					: "warning"
 				: "warning";
 			debugLog("observer.empty", { coversUpToId, reason: reason?.kind });
+			runtime.advanceCursor("observer", coversUpToId, "empty");
 			if (ctx.hasUI) ctx.ui?.notify(`Observational memory: no observations — ${reasonLabel}`, reasonLevel);
 			return "continue";
 		} catch (error) {
@@ -502,15 +720,15 @@ async function runReflectorStage(
 		const pending = readPendingState(sessionId);
 		// Check any accumulated batch for unprocessed observations, not just the latest
 		const hasPendingObs = (pending.observationBatches ?? []).some((b: any) => (b.data as any)?.observations?.length);
-		if (!hasPendingObs) return { outcome: "continue", sameRunReflections: [] };
+		if (!hasPendingObs) { runtime.advanceCursor("reflector", entries.at(-1)?.id ?? "unknown", "skipped"); return { outcome: "continue", sameRunReflections: [] }; }
 		observationCoverageId = pending.observation?.coversUpToId;
 		if (pending.reflection?.coversUpToId) {
 			const obsIdx = entryIndexForId(entries, pending.observation?.coversUpToId ?? "");
 			const refIdx = entryIndexForId(entries, pending.reflection.coversUpToId);
-			if (obsIdx >= 0 && refIdx >= 0 && obsIdx <= refIdx) return { outcome: "continue", sameRunReflections: [] };
+			if (obsIdx >= 0 && refIdx >= 0 && obsIdx <= refIdx) { runtime.advanceCursor("reflector", pending.reflection.coversUpToId, "skipped"); return { outcome: "continue", sameRunReflections: [] }; }
 			if (refIdx >= 0) {
 				reflectionTokens = rawTokensAfterIndex(entries, refIdx);
-				if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
+				if (reflectionTokens < runtime.config.reflectAfterTokens) { runtime.advanceCursor("reflector", pending.reflection.coversUpToId, "not_due"); return { outcome: "continue", sameRunReflections: [] }; }
 			} else {
 				reflectionTokens = rawTokensSinceObservationCoverage(entries);
 			}
@@ -519,9 +737,9 @@ async function runReflectorStage(
 		}
 	} else {
 		reflectionTokens = rawTokensSinceReflectionCoverage(entries);
-		if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
+		if (reflectionTokens < runtime.config.reflectAfterTokens) { runtime.advanceCursor("reflector", entries.at(-1)?.id ?? "unknown", "not_due"); return { outcome: "continue", sameRunReflections: [] }; }
 		observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
-		if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
+		if (!observationCoverageId) { runtime.advanceCursor("reflector", entries.at(-1)?.id ?? "unknown", "skipped"); return { outcome: "continue", sameRunReflections: [] }; }
 	}
 
 	for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
@@ -550,6 +768,7 @@ async function runReflectorStage(
 				if (idx >= 0) effectiveReflectionTokens = rawTokensAfterIndex(entries, idx);
 			}
 		}
+		debugLog("reflector.start", { tokens: effectiveReflectionTokens, inputTokens: reflectorInputTokens, newObsCount: newObservations.length, newRefCount: newReflections.length });
 		if (ctx.hasUI) ctx.ui?.notify(`Observational memory: reflector running (~${effectiveReflectionTokens.toLocaleString()} tokens accumulated, ~${reflectorInputTokens.toLocaleString()}-token input)`, "info");
 
 		// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
@@ -597,16 +816,26 @@ async function runReflectorStage(
 				thinkingLevel: stageThinkingLevel(runtime, "reflector", stageModelForThinking),
 			});
 
-			if (!reflections || reflections.length === 0) return { outcome: "continue", sameRunReflections: [] };
-			if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
+			if (!reflections || reflections.length === 0) {
+				runtime.advanceCursor("reflector", observationCoverageId ?? entries.at(-1)?.id ?? "unknown", "empty");
+				return { outcome: "continue", sameRunReflections: [] };
+			}
+			if (!observationCoverageId) {
+				runtime.advanceCursor("reflector", entries.at(-1)?.id ?? "unknown", "empty");
+				return { outcome: "continue", sameRunReflections: [] };
+			}
 
 			const data = buildReflectionsRecordedData(reflections, observationCoverageId);
-			if (!data) return { outcome: "continue", sameRunReflections: [] };
+			if (!data) {
+				runtime.advanceCursor("reflector", observationCoverageId, "empty");
+				return { outcome: "continue", sameRunReflections: [] };
+			}
 			if (runtime.config.noAutoCompact) {
 				savePendingReflection(sessionId, { coversUpToId: data.coversUpToId, data });
 			} else {
 				appendEntry(pi, OM_REFLECTIONS_RECORDED, data);
 			}
+			runtime.advanceCursor("reflector", data.coversUpToId, "recorded");
 			return {
 				outcome: "continue",
 				sameRunReflections: reflections,
@@ -642,15 +871,15 @@ async function runDropperStage(
 		const pending = readPendingState(sessionId);
 		// Check any accumulated batch for unprocessed observations, not just the latest
 		const hasPendingObs = (pending.observationBatches ?? []).some((b: any) => (b.data as any)?.observations?.length);
-		if (!hasPendingObs) return "continue";
+		if (!hasPendingObs) { runtime.advanceCursor("dropper", entries.at(-1)?.id ?? "unknown", "skipped"); return "continue"; }
 		observationCoverageId = pending.observation?.coversUpToId;
 		if (pending.dropped?.coversUpToId) {
 			const obsIdx = entryIndexForId(entries, pending.observation?.coversUpToId ?? "");
 			const dropIdx = entryIndexForId(entries, pending.dropped.coversUpToId);
-			if (obsIdx >= 0 && dropIdx >= 0 && obsIdx <= dropIdx) return "continue";
+			if (obsIdx >= 0 && dropIdx >= 0 && obsIdx <= dropIdx) { runtime.advanceCursor("dropper", pending.dropped.coversUpToId, "skipped"); return "continue"; }
 			if (dropIdx >= 0) {
 				dropTokens = rawTokensAfterIndex(entries, dropIdx);
-				if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
+				if (dropTokens < runtime.config.reflectAfterTokens) { runtime.advanceCursor("dropper", pending.dropped.coversUpToId, "not_due"); return "continue"; }
 			} else {
 				dropTokens = rawTokensSinceDropCoverage(entries);
 			}
@@ -659,9 +888,9 @@ async function runDropperStage(
 		}
 	} else {
 		dropTokens = rawTokensSinceDropCoverage(entries);
-		if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
+		if (dropTokens < runtime.config.reflectAfterTokens) { runtime.advanceCursor("dropper", entries.at(-1)?.id ?? "unknown", "not_due"); return "continue"; }
 		observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
-		if (!observationCoverageId) return "continue";
+		if (!observationCoverageId) { runtime.advanceCursor("dropper", entries.at(-1)?.id ?? "unknown", "skipped"); return "continue"; }
 	}
 
 	for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
@@ -746,6 +975,10 @@ async function runDropperStage(
 				} else {
 					appendEntry(pi, OM_OBSERVATIONS_DROPPED, data);
 				}
+				runtime.advanceCursor("dropper", coversUpToId, "recorded");
+			} else {
+				// No drops selected (maxDropsAllowed=0 or LLM returned no candidates)
+				runtime.advanceCursor("dropper", coversUpToId ?? observationCoverageId ?? entries.at(-1)?.id ?? "unknown", "empty");
 			}
 			return "continue";
 		} catch (error) {

package/src/om/pending.ts

@@ -59,6 +59,12 @@ export interface PendingOMState {
 	 * /blackhole flush.
 	 */
 	droppedBatches?: PendingDropped[];
+	/** Pipeline progress cursors — persist across restarts and fork recovery. */
+	cursors?: {
+		observer?: { entryId: string; state: string };
+		reflector?: { entryId: string; state: string };
+		dropper?: { entryId: string; state: string };
+	};
 }
 
 // ── Persistence ─────────────────────────────────────────────────────────────
@@ -88,6 +94,8 @@ function defaultState(): PendingOMState {
 }
 
 function isEmptyState(s: PendingOMState): boolean {
+	const hasCursors = s.cursors && (s.cursors.observer || s.cursors.reflector || s.cursors.dropper);
+	if (hasCursors) return false;
 	return !s.observation && !s.reflection && !s.dropped
 		&& (!s.observationBatches || s.observationBatches.length === 0)
 		&& (!s.reflectionBatches || s.reflectionBatches.length === 0)
@@ -159,7 +167,9 @@ function isPendingOMState(value: unknown): value is PendingOMState {
 	const hasDrop = !!(v.dropped && typeof v.dropped === "object" && typeof (v.dropped as any).coversUpToId === "string");
 	// Also accept states with only batch arrays (no singular fields)
 	const hasBatches = Array.isArray(v.observationBatches) || Array.isArray(v.reflectionBatches) || Array.isArray(v.droppedBatches);
-	return hasObs || hasRef || hasDrop || hasBatches;
+	// Accept cursor-only states (no observations/reflections yet, but persisted)
+	const hasCursors = !!(v.cursors && typeof v.cursors === "object");
+	return hasObs || hasRef || hasDrop || hasBatches || hasCursors;
 }
 
 /**
@@ -268,6 +278,20 @@ export function hasPendingData(sessionId: string): boolean {
 	return !isEmptyState(readSessionState(sessionId));
 }
 
+/** Read cursors from pending state for a session. */
+export function readPendingCursors(sessionId: string): PendingOMState["cursors"] {
+	const state = readSessionState(sessionId);
+	return state.cursors;
+}
+
+/** Write cursors to pending state for a session (replaces existing cursors).
+ *  Uses assignment (not merge) so deletions from validateCursors persist. */
+export function writePendingCursors(sessionId: string, cursors: PendingOMState["cursors"]): void {
+	const state = readSessionState(sessionId);
+	state.cursors = { ...cursors };
+	writeSessionState(sessionId, state);
+}
+
 /**
  * List all session IDs that have pending data by scanning the pending directory
  * for *-pending.json files.

package/src/om/runtime.ts

@@ -11,6 +11,8 @@
  */
 import { type Config, type ConfiguredModel, DEFAULTS, loadConfig } from "./config.js";
 import { isCooldownActive, getCooldownEntry, recordCooldown, expireCooldowns, modelKey } from "./cooldown.js";
+import { readPendingCursors, writePendingCursors } from "./pending.js";
+import type { PendingOMState } from "./pending.js";
 
 export type ResolveResult =
 	| { ok: true; model: any; apiKey: string; headers?: Record<string, string>; cooldownApplied?: boolean }
@@ -20,6 +22,19 @@ type NotifyLevel = "warning" | "info" | "error";
 type Notify = (message: string, type?: NotifyLevel) => void;
 export type ConsolidationPhase = "observer" | "reflector" | "dropper";
 
+export type CursorState = "initial" | "recorded" | "empty" | "error" | "skipped" | "not_due";
+
+export interface PipelineCursor {
+	entryId: string;
+	state: CursorState;
+}
+
+export interface PipelineCursors {
+	observer?: PipelineCursor;
+	reflector?: PipelineCursor;
+	dropper?: PipelineCursor;
+}
+
 export interface ResolveCtx {
 	model: unknown;
 	modelRegistry: any;
@@ -54,6 +69,10 @@ export class Runtime {
 	failedInCycle: Set<string> = new Set();
 	compactInFlight = false;
 	compactHookInFlight = false;
+	/** AbortController for the pending auto-compaction wait loop, or null if none.
+	 * Set when handleAgentEnd schedules a wait; cleared on abort, success, or terminal bail.
+	 * agent_start handlers read this to abort the pending wait when a new turn starts. */
+	autoCompactionController: AbortController | null = null;
 	resolveFailureNotified = false;
 	lastObserverError: string | undefined;
 	lastReflectorError: string | undefined;
@@ -64,6 +83,10 @@ export class Runtime {
 	compactionStats: { summarized: number; kept: number; keptTokensEst: number } | null = null;
 	/** Whether the most recent compaction was triggered by /blackhole (vs auto-compact). */
 	compactWasPiVcc = false;
+	/** In‑memory pipeline cursors — authoritative copy for gating decisions. */
+	cursors: PipelineCursors = {};
+	/** Session ID for which cursors have been loaded/validated.  Undefined until first load. */
+	cursorsLoadedSessionId: string | undefined = undefined;
 
 	ensureConfig(cwd: string): void {
 		if (this.configLoaded) return;
@@ -251,6 +274,57 @@ export class Runtime {
 		return Date.now() - this.lastConsolidationErrorAt < CONSOLIDATION_RETRY_COOLDOWN_MS;
 	}
 
+	/** Get the current cursor for a pipeline stage. */
+	getCursor(stage: ConsolidationPhase): PipelineCursor | undefined {
+		return this.cursors[stage];
+	}
+
+	/** Advance a stage's cursor to a new entry ID with the given state. */
+	advanceCursor(stage: ConsolidationPhase, entryId: string, state: CursorState): void {
+		this.cursors[stage] = { entryId, state };
+	}
+
+	/** Load cursors from the per‑session pending file into the in‑memory map. */
+	loadCursorsFromPending(sessionId: string): void {
+		try {
+			const stored = readPendingCursors(sessionId);
+			if (!stored) return;
+			if (stored.observer?.entryId && stored.observer?.state) {
+				this.cursors.observer = { entryId: stored.observer.entryId, state: stored.observer.state as CursorState };
+			}
+			if (stored.reflector?.entryId && stored.reflector?.state) {
+				this.cursors.reflector = { entryId: stored.reflector.entryId, state: stored.reflector.state as CursorState };
+			}
+			if (stored.dropper?.entryId && stored.dropper?.state) {
+				this.cursors.dropper = { entryId: stored.dropper.entryId, state: stored.dropper.state as CursorState };
+			}
+		} catch {
+			// Best‑effort: missing or corrupt files are harmless.
+		}
+	}
+
+	/** Save in‑memory cursors to the per‑session pending file (synchronous, for tests). */
+	saveCursorsToPending(sessionId: string): void {
+		try {
+			writePendingCursors(sessionId, this.cursors as PendingOMState["cursors"]);
+		} catch {
+			// Best‑effort: graceful degradation on read‑only filesystems.
+		}
+	}
+
+	/** Schedule an async flush of cursors to the pending file.
+	 *  Uses a micro‑task to avoid blocking the pipeline. */
+	scheduleCursorFlush(sessionId: string): void {
+		const cursors = { ...this.cursors };
+		queueMicrotask(() => {
+			try {
+				writePendingCursors(sessionId, cursors as PendingOMState["cursors"]);
+			} catch {
+				// Best‑effort: graceful degradation.
+			}
+		});
+	}
+
 	launchConsolidationTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> {
 		this.consolidationInFlight = true;
 		this.consolidationPhase = undefined;