pi-mega-compact 0.8.16 → 0.8.18

package/extensions/mega-events/agent-handlers.ts

@@ -19,6 +19,7 @@ import { evaluateAndUnlockAchievements } from "../../src/store/sqlite/game-achie
 import { isMegaCache } from "../../src/game/scoring.js";
 import { resolveRepoRoot } from "../mega-config.js";
 import { classifyError } from "./error-classifier.js";
+import { safeSendUserMessage } from "./send-safe.js";
 
 /** Register agent/turn tracking event handlers. */
 export function registerAgentHandlers(
@@ -215,7 +216,7 @@ export function registerAgentHandlers(
 					const nudgeMsg = lengthStop && !didDurableTrim
 						? "[mega-compact] the last response hit the output-token cap; continue from where it stopped."
 						: "[mega-compact] continue from the compacted context above.";
-					pi.sendUserMessage(nudgeMsg);
+					await safeSendUserMessage(pi, nudgeMsg);
 				}
 			} catch {
 				/* non-fatal: a failed nudge never blocks */
@@ -346,6 +347,48 @@ export function registerAgentHandlers(
 						sessionId: runtime.rt.sessionId,
 						turnIndex: event.turnIndex,
 					});
+				} else if (category === 'context-overflow') {
+					// (4b) context-window overflow 400 ("too long... even after compaction").
+					// NOT a blind retry: re-submitting the same oversized prompt would just
+					// re-400 and busy-loop. Reset the counters (this turn is terminal, not
+					// retryable in the S38 sense) and force ONE best-effort re-compact with
+					// the debounce bypassed + the same race-guard (lastNativeCompactAt
+					// cooldown + deferred setTimeout re-check) as the agent_end durable
+					// trim. Resume after that forced compact is handled by the existing
+					// nudgeResume() inside session_before_compact (it fires after
+					// driveNativeCompaction supplies the compaction) — do NOT add a
+					// separate nudge here.
+					runtime.rt.errorRetryCount = 0;
+					runtime.rt.consecutiveErrors = 0;
+					runtime.dashboard.event('context_overflow', {
+						turnIndex: event.turnIndex,
+						sessionId: runtime.rt.sessionId,
+					});
+					runtime.logger.warn('context-overflow', {
+						sessionId: runtime.rt.sessionId,
+						turnIndex: event.turnIndex,
+					});
+					if (config.auto) {
+						const now2 = Date.now();
+						const cooldownMs2 = config.raceGuardStrict ? 30_000 : 10_000;
+						const sinceCompact2 = now2 - (runtime.rt.lastNativeCompactAt ?? 0);
+						if (sinceCompact2 >= cooldownMs2 && !piCompactWouldNoop(ctx)) {
+							runtime.debounceUntil = now2 + 0;
+							const stamp2 = runtime.rt.lastNativeCompactAt;
+							const liveSid2 = runtime.rt.sessionId;
+							setTimeout(() => {
+								try {
+									if (runtime.rt.sessionId !== liveSid2) return; // session reset
+									const since3 = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
+									if (runtime.rt.lastNativeCompactAt !== stamp2 && since3 < cooldownMs2) return;
+									if (piCompactWouldNoop(ctx)) return;
+									ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. Forced re-compact after a context-overflow 400.
+								} catch {
+									/* non-fatal */
+								}
+							}, 500);
+						}
+					}
 				} else {
 					// (5) transient or permanent — retry with exponential backoff.
 					// S38.7: hard-stop switch — bypass ALL retry logic when set.
@@ -421,8 +464,9 @@ export function registerAgentHandlers(
 									count: runtime.rt.errorRetryCount,
 									max,
 								});
-								// PREVENT-PI-003: user-role sendUserMessage only.
-								pi.sendUserMessage(
+								// PREVENT-PI-003: user-role sendUserMessage only (queued + catch-guarded).
+								await safeSendUserMessage(
+									pi,
 									'[mega-compact] the last turn ended with an error; please retry.',
 								);
 							}

package/extensions/mega-events/compact-handlers.ts

@@ -22,6 +22,7 @@ import type { MegaConfig } from "../mega-config.js";
 import { recordScore, getDedupStats } from "../../src/store/sqlite.js";
 import { evaluateAndUnlockAchievements } from "../../src/store/sqlite/game-achievements.js";
 import { resolveRepoRoot } from "../mega-config.js";
+import { safeSendUserMessage } from "./send-safe.js";
 
 /**
  * Build a minimal fallback compaction so pi never runs its throwing compact().
@@ -57,13 +58,19 @@ function fallbackCompaction(
 /**
  * Debounced resume-nudge: restart the agent loop after a compaction (which
  * may have stopped it). Idempotent — one nudge per 30s, never blocks.
+ *
+ * Uses safeSendUserMessage ({ deliverAs: 'followUp' } + catch-guard) so that a
+ * nudge fired during session_before_compact (which is mid-prompt-submission,
+ * so the agent can be busy) QUEUES instead of throwing
+ * "Agent is already processing. Specify streamingBehavior (steer or followUp)".
  */
-function nudgeResume(pi: ExtensionAPI, runtime: MegaRuntime): void {
+async function nudgeResume(pi: ExtensionAPI, runtime: MegaRuntime): Promise<void> {
 	try {
 		const now = Date.now();
 		if (now >= runtime.resumeNudgeUntil) {
 			runtime.resumeNudgeUntil = now + 30_000;
-			pi.sendUserMessage(
+			await safeSendUserMessage(
+				pi,
 				"[mega-compact] continue from the compacted context above.",
 			);
 		}
@@ -113,7 +120,7 @@ export function registerCompactHandlers(
 						tokensBefore: result.compaction.tokensBefore,
 						summaryTokens: result.compaction.estimatedTokensAfter,
 					});
-					nudgeResume(pi, runtime);
+					await nudgeResume(pi, runtime);
 					return { compaction: result.compaction };
 				}
 				// FIX "compacts but doesn't resume" + "Nothing to compact" regression:
@@ -134,7 +141,7 @@ export function registerCompactHandlers(
 						tokensBefore: fb.compaction.tokensBefore,
 						reason: event.reason,
 					});
-					nudgeResume(pi, runtime);
+					await nudgeResume(pi, runtime);
 					return { compaction: fb.compaction };
 				}
 			} catch (err) {

package/extensions/mega-events/error-classifier.ts

@@ -12,12 +12,13 @@
  * exclusively (its agent_end nudge path is separate and must not be doubled).
  *
  * @param message  the event.message (a pi AgentMessage) or an error string
- * @returns 'transient' | 'permanent' | 'compaction-noop' | null (success/unknown)
+ * @returns 'transient' | 'permanent' | 'compaction-noop' | 'context-overflow' | null (success/unknown)
  */
 export function classifyError(message: unknown):
 	| 'transient'
 	| 'permanent'
 	| 'compaction-noop'
+	| 'context-overflow'
 	| null {
 	// Resolve a searchable text blob from a pi AgentMessage or raw string.
 	let text = '';
@@ -81,6 +82,29 @@ export function classifyError(message: unknown):
 	if (/compaction failed/.test(s)) return 'compaction-noop';
 	if (/nothing to compact/.test(s)) return 'compaction-noop';
 	if (/auto[\s-]?compaction failed/.test(s)) return 'compaction-noop';
+	// --- context-overflow (ORDER BEFORE generic transient!) ---
+	// A 400 from the model meaning "the prompt is bigger than the context window."
+	// Catches BOTH provider phrasings so the classification does not depend on
+	// which backend the router landed on:
+	//   - pi/Anthropic wrapper: "too long for this model's context window even
+	//     after compaction. Reduce the conversation length..." ->
+	//     too long | context window | even after compaction | reduce the conversation
+	//   - OpenAI/OpenRouter provider-side (often wrapped in "All targets failed:
+	//     <model>. Last error: ..."): "This model's maximum context length is N
+	//     tokens. Your request requires at least M tokens. Please reduce your
+	//     input or max_tokens." -> maximum context length | context length
+	//     exceeded | requires at least N tokens | reduce your input
+	// All of these carry `invalid_request_error` (UNDERSCORE, not 'invalid
+	// request' space) and would otherwise fall through to the generic
+	// `s.includes('error')` transient branch below, misclassifying as
+	// 'transient' and firing up to 5 blind retry nudges that re-submit the same
+	// oversized prompt -> re-400 -> busy-loop. 'context-overflow' instead forces
+	// ONE deferred re-compact (debounce-bypassed, race-guarded) and fires NO
+	// blind retry nudge. The forced re-compact is shaped by the existing
+	// session_before_compact durable trim (it cannot lower pi's firstKeptEntryId).
+	if (/too long|context window|maximum context length|context length exceeded|requires at least \d+ tokens|even after compaction|reduce the conversation|reduce your input/.test(s)) {
+		return 'context-overflow';
+	}
 	// --- transient (retryable) ---
 	if (s.includes('error') && !/\b(permanent|invalid request|malformed|bad request|auth|unauthorized|invalid (api )?key|permission)\b/.test(s)) {
 		return 'transient'; // generic pi stopReason 'error' / 'aborted'

package/extensions/mega-events/send-safe.ts

@@ -0,0 +1,37 @@
+/**
+ * send-safe.ts — queue-safe wrapper for pi.sendUserMessage.
+ *
+ * All extension-initiated user-role nudges (resume after compaction, length-stop
+ * continue, error-retry) MUST go through this wrapper. It:
+ *   1. Passes { deliverAs: 'followUp' } so that when the agent is busy (e.g. a
+ *      resume nudge fired during session_before_compact, which is mid-prompt
+ *      submission) pi QUEUES the message instead of throwing
+ *      "Agent is already processing. Specify streamingBehavior (steer or
+ *      followUp) to queue the message" (pi agent-session.js:830).
+ *   2. Awaits the call and catch-guards it so a failed/queued nudge never throws
+ *      or produces an unhandled rejection — it must never block the agent loop.
+ *
+ * PREVENT-PI-003: user-role sendUserMessage only (no role:'system' injection).
+ * PREVENT-PI-004: local pi ctx call, no network.
+ */
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+/**
+ * safeSendUserMessage — await + catch-guard + queue-safe wrapper for
+ * pi.sendUserMessage. Never throws; never produces an unhandled rejection.
+ */
+export async function safeSendUserMessage(
+	pi: ExtensionAPI,
+	content: string,
+): Promise<void> {
+	try {
+		await (
+			pi.sendUserMessage as (
+				c: string,
+				o?: { deliverAs?: "steer" | "followUp" },
+			) => Promise<void> | void
+		)(content, { deliverAs: "followUp" });
+	} catch {
+		/* non-fatal: a failed/queued nudge never blocks the agent loop */
+	}
+}

package/extensions/mega-events.ts

@@ -17,3 +17,4 @@ export * from "./mega-events/context-handler.js";
 export * from "./mega-events/compact-handlers.js";
 export * from "./mega-events/perf-handler.js";
 export * from "./mega-events/error-classifier.js";
+export * from "./mega-events/send-safe.js";

package/extensions/mega-runtime/state.ts

@@ -29,6 +29,8 @@ import {
 	getCacheHitTokensSaved,
 	getGameState,
 	recordPerfSample,
+	recordSessionHeartbeat,
+	appendTokenSample,
 	type ModelSnapshot,
 	type GameState,
 } from "../../src/store/sqlite.js";
@@ -575,6 +577,35 @@ export class MegaRuntime {
 		} as DashboardSnapshot);
 		const perfDiskMs = this.dashboard.lastWriteMs;
 
+		// S39: record a session heartbeat + token sample into the shared
+		// machine-wide index.sqlite so the dashboard can show a real-time
+		// stacked-memory graph across all active pi processes. Behind the
+		// material-change gate (this code only runs when sig changed). Non-fatal
+		// try/catch mirrors the recordPerfSample pattern below. Skip the token
+		// sample when lastCtxTokens is null (no context data yet).
+		try {
+			const repo = resolveRepoRoot(ctx?.cwd ?? this.currentStateDir) ?? this.currentStateDir;
+			recordSessionHeartbeat(
+				process.pid,
+				this.rt.sessionId,
+				repo,
+				this.currentStateDir,
+				this.lastCtxWindow || 0,
+			);
+			if (this.lastCtxTokens != null) {
+				appendTokenSample(
+					this.rt.sessionId,
+					repo,
+					this.lastCtxTokens,
+					this.lastCtxPercent ?? 0,
+					this.lastCtxWindow || 0,
+					join(this.currentStateDir, "events.log"),
+				);
+			}
+		} catch {
+			/* non-fatal: S39 monitoring must never block the snapshot path */
+		}
+
 		// Live stats widget above the editor
 		if (ctx) {
 			// ── gather widget data (computed per snapshot, rendered per frame) ────

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.8.16",
+  "version": "0.8.18",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",

package/src/store/sqlite/global-index.test.ts

@@ -0,0 +1,222 @@
+/**
+ * global-index.test.ts — S39 session heartbeats + token_samples round-trip.
+ *
+ * Verifies recordSessionHeartbeat + appendTokenSample + readActiveSessions +
+ * readSessionTimeseries + pruneStaleSessions + pruneTokenSamples +
+ * clearSessionHeartbeat, with particular focus on the Step 5 events.log
+ * session_sample JSONL line shape (the SSE real-time push wire format).
+ *
+ * Pi-agnostic. Uses an isolated mock index dir (never the real user
+ * `~/.mega-compact-index/` — guardrail G7). PREVENT-PI-004 (zero network):
+ * every operation is local SQLite + local FS append.
+ */
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
+
+import {
+	recordSessionHeartbeat,
+	appendTokenSample,
+	readActiveSessions,
+	readSessionTimeseries,
+	pruneStaleSessions,
+	pruneTokenSamples,
+	clearSessionHeartbeat,
+	closeIndexStore,
+} from "./global-index.js";
+
+describe("global-index session time-series (S39)", () => {
+	let indexDir: string;
+	let eventsLogPath: string;
+
+	before(() => {
+		indexDir = mkdtempSync(join(tmpdir(), "mc-global-idx-"));
+		eventsLogPath = join(indexDir, "events.log");
+	});
+
+	after(() => {
+		closeIndexStore();
+		rmSync(indexDir, { recursive: true, force: true });
+	});
+
+	it("recordSessionHeartbeat + readActiveSessions: heartbeat row appears", () => {
+		recordSessionHeartbeat(
+			12_345,
+			"sess-abc-1",
+			"/home/u/repos/proj-a",
+			"/home/u/.pi/state-a",
+			200_000,
+			indexDir,
+		);
+		const sessions = readActiveSessions(indexDir);
+		assert.equal(sessions.length, 1);
+		const s = sessions[0];
+		assert.equal(s.pid, 12_345);
+		assert.equal(s.sessionId, "sess-abc-1");
+		assert.equal(s.repoRoot, "/home/u/repos/proj-a");
+		assert.equal(s.stateDir, "/home/u/.pi/state-a");
+		assert.equal(s.ctxWindow, 200_000);
+		assert.ok(s.lastSeen > 0);
+		// No token sample yet → tokens/percent are null.
+		assert.equal(s.tokens, null);
+		assert.equal(s.percent, null);
+	});
+
+	it("appendTokenSample writes a token_samples row + a session_sample JSONL line", () => {
+		appendTokenSample(
+			"sess-abc-1",
+			"/home/u/repos/proj-a",
+			42_000,
+			21.5,
+			200_000,
+			eventsLogPath,
+			indexDir,
+		);
+		// The token_samples row should now be the latest sample for this session.
+		const sessions = readActiveSessions(indexDir);
+		const s = sessions.find((x) => x.sessionId === "sess-abc-1");
+		assert.ok(s, "session must still appear in readActiveSessions");
+		assert.equal(s!.tokens, 42_000);
+		assert.equal(s!.percent, 21.5);
+
+		// The events.log JSONL line should exist and match the wire format
+		// described by the SseSessionSample contract: an object with `ts`
+		// (ISO 8601 string), `type: 'session_sample'`, and the payload fields.
+		assert.ok(existsSync(eventsLogPath), "events.log should exist");
+		const lines = readFileSync(eventsLogPath, "utf-8")
+			.trim()
+			.split("\n")
+			.filter(Boolean);
+		assert.equal(lines.length, 1, "exactly one JSONL line expected");
+		const parsed = JSON.parse(lines[0]) as Record<string, unknown>;
+		assert.equal(parsed["type"], "session_sample");
+		assert.equal(parsed["sessionId"], "sess-abc-1");
+		assert.equal(parsed["tokens"], 42_000);
+		assert.equal(parsed["percent"], 21.5);
+		// Step 5 contract mirroring: `ts` is an ISO 8601 string (matches every
+		// other SseEvent variant's `ts: string` and the DashboardEmitter
+		// events.log pattern), NOT a numeric ms value.
+		assert.equal(
+			typeof parsed["ts"],
+			"string",
+			"ts must be an ISO 8601 string (matches other SseEvent variants)",
+		);
+		// Must be ISO-parseable (NaN if invalid → typeof won't be number).
+		const tsMs = Date.parse(parsed["ts"] as string);
+		assert.ok(
+			Number.isFinite(tsMs),
+			"ts must be a valid ISO 8601 timestamp",
+		);
+		// And recent (within the last 5s).
+		assert.ok(
+			Math.abs(Date.now() - tsMs) < 5_000,
+			"ts must be within ~5s of now",
+		);
+	});
+
+	it("readSessionTimeseries returns per-session series + totals in chronological order", () => {
+		// Two sessions, two samples each — fully within the rolling window.
+		// sess-xyz-9 needs its own heartbeat row so readActiveSessions can later
+		// surface it (heartbeats are the primary key, not the samples).
+		recordSessionHeartbeat(
+			67_890,
+			"sess-xyz-9",
+			"/home/u/repos/proj-b",
+			"/home/u/.pi/state-b",
+			200_000,
+			indexDir,
+		);
+		appendTokenSample(
+			"sess-abc-1",
+			"/home/u/repos/proj-a",
+			50_000,
+			25,
+			200_000,
+			null, // no events.log line for this assertion
+			indexDir,
+		);
+		appendTokenSample(
+			"sess-xyz-9",
+			"/home/u/repos/proj-b",
+			10_000,
+			5,
+			200_000,
+			null,
+			indexDir,
+		);
+		appendTokenSample(
+			"sess-xyz-9",
+			"/home/u/repos/proj-b",
+			30_000,
+			15,
+			200_000,
+			null,
+			indexDir,
+		);
+		const since = Date.now() - 60_000;
+		const result = readSessionTimeseries(since, indexDir);
+
+		const seriesIds = result.series.map((s) => s.sessionId).sort();
+		assert.deepEqual(seriesIds, ["sess-abc-1", "sess-xyz-9"]);
+
+		const xyz = result.series.find((s) => s.sessionId === "sess-xyz-9");
+		assert.ok(xyz, "sess-xyz-9 series must exist");
+		assert.equal(xyz!.data.length, 2);
+		assert.equal(xyz!.data[0].tokens, 10_000);
+		assert.equal(xyz!.data[1].tokens, 30_000);
+		// Stable hash-based color from the SESSION_COLORS palette.
+		assert.match(xyz!.color, /^#[0-9a-f]{6}$/i);
+
+		// Totals: each timestamp summed across sessions.
+		assert.ok(result.totals.length >= 2);
+		for (const t of result.totals) {
+			assert.ok(typeof t.ts === "number" && t.ts > 0);
+			assert.ok(typeof t.tokens === "number" && t.tokens >= 0);
+		}
+		// Totals are sorted ascending by ts.
+		for (let i = 1; i < result.totals.length; i++) {
+			assert.ok(
+				result.totals[i].ts >= result.totals[i - 1].ts,
+				"totals must be chronologically ascending",
+			);
+		}
+	});
+
+	it("pruneStaleSessions clears heartbeats older than the cutoff (and only those)", () => {
+		// At this point we have 2 sessions; both are fresh (lastSeen ≈ now).
+		const beforeCount = readActiveSessions(indexDir).length;
+		assert.equal(beforeCount, 2);
+
+		// With a 0 ms cutoff, both are stale → pruned.
+		const pruned = pruneStaleSessions(0, indexDir);
+		assert.equal(pruned, 2, "both heartbeats should be pruned");
+		assert.equal(readActiveSessions(indexDir).length, 0);
+	});
+
+	it("pruneTokenSamples clears token_samples older than the cutoff", () => {
+		// Re-seed two samples; the previously-written ones were for two
+		// sessions (sess-abc-1 + sess-xyz-9). The prunedStaleSessions call
+		// above left token_samples untouched (separate table).
+		const sinceBefore = Date.now() - 60_000;
+		const before = readSessionTimeseries(sinceBefore, indexDir);
+		const beforeCount = before.series.reduce((n, s) => n + s.data.length, 0);
+		assert.ok(beforeCount >= 3, "expected to find the seeded samples");
+
+		const npurged = pruneTokenSamples(0, indexDir);
+		assert.ok(npurged >= 3, "should have pruned the seeded samples");
+
+		const after = readSessionTimeseries(sinceBefore, indexDir);
+		const afterCount = after.series.reduce((n, s) => n + s.data.length, 0);
+		assert.equal(afterCount, 0, "no samples should remain after pruning");
+	});
+
+	it("clearSessionHeartbeat is a no-op on missing rows (never throws)", () => {
+		const before = readActiveSessions(indexDir).length;
+		assert.doesNotThrow(() =>
+			clearSessionHeartbeat(99_999, "nonexistent-session", indexDir),
+		);
+		assert.equal(readActiveSessions(indexDir).length, before);
+	});
+});