pi-mega-compact 0.9.2 → 0.10.1

package/extensions/mega-events/session-handlers.ts

@@ -18,6 +18,8 @@ import {
 } from "../mega-pipeline.js";
 import { recallMemoriesAndInline } from "../../src/recall.js";
 import { vectorStats } from "../../src/vectorStore.js";
+import { openTurnStore } from "../../src/store/turns/connection.js";
+import { openIntentQueue } from "../../src/intent.js";
 import type { MegaConfig } from "../mega-config.js";
 
 /** Register session lifecycle event handlers. */
@@ -148,6 +150,27 @@ export function registerSessionHandlers(
 	// ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
 	pi.on("before_agent_start", async (event, ctx) => {
 		runtime.captureModel(ctx); // most reliable point ctx.model is populated
+		// S52A: consume any pending rewind intents the dashboard queued. The host
+		// polls the intent queue here (the only safe lifecycle point) and marks each
+		// consumed. Best-effort + non-fatal — the store never calls back into the
+		// host (ledger protocol). A real rewind (reloading pi's conversation to turn
+		// N) is host-specific; this consumes the intent so the round-trip completes.
+		try {
+			if (config.turnsDbEnabled) {
+				const tdb = openTurnStore(runtime.currentStateDir);
+				const q = openIntentQueue(tdb);
+				const pending = q.pendingIntents();
+				for (const intent of pending) {
+					q.consumeIntent(intent.id);
+					runtime.dashboard.event("rewind_intent_consumed", {
+						conversationId: intent.conversationId,
+						targetTurnIndex: intent.targetTurnIndex,
+					});
+				}
+			}
+		} catch {
+			/* non-fatal: intent polling never breaks the agent loop */
+		}
 		const cpBlock = runtime.pendingRecallBlock;
 		const memBlock = runtime.pendingMemoryRecallBlock;
 		if (!cpBlock && !memBlock) return;

package/extensions/mega-metrics-cmds.ts

@@ -0,0 +1,114 @@
+/**
+ * mega-metrics-cmds.ts — S50C per-turn memory-quality /commands.
+ *
+ * Registers /mega-metrics and /mega-fork, backed by the pi-agnostic
+ * src/metrics/turns.ts rollups and src/fork.ts primitive over the isolated
+ * S49 turns.db. Read-only (PREVENT-PI-001/002 — these never mutate memory,
+ * drop ranges, or compaction); local SQLite only (PREVENT-PI-004);
+ * parameterized queries (PREVENT-002). Both require turnsDbEnabled (the
+ * isolated store); they no-op with guidance on the legacy main-db path.
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { MegaRuntime } from "./mega-runtime.js";
+import type { MegaConfig } from "./mega-config.js";
+import { turnStoreFor, mainDbFor } from "./mega-turn-store.js";
+import { turnMetrics, conversationMetrics } from "../src/metrics/index.js";
+import { forkFromConversation, ForkError } from "../src/fork.js";
+import { ensureConversationIdFor } from "./mega-turn-store.js";
+
+function pct(r: number): string {
+	return `${(r * 100).toFixed(1)}%`;
+}
+
+/** Register the /mega-metrics + /mega-fork commands. */
+export function registerMetricsCommands(
+	pi: ExtensionAPI,
+	runtime: MegaRuntime,
+	config: MegaConfig,
+): void {
+	pi.registerCommand("mega-metrics", {
+		description:
+			"Show per-turn + per-conversation memory-quality metrics (recall reuse, dedup ratio, compression ratio). Usage: /mega-metrics [conversationId]",
+		handler: async (args: string, ctx: ExtensionContext) => {
+			try {
+				runtime.bindRepo(ctx.cwd);
+				const stateDir = runtime.currentStateDir;
+				const store = turnStoreFor(config, stateDir);
+				if (!store) {
+					ctx.ui.notify(
+						"[mega-compact] /mega-metrics requires the isolated turns.db (set turnsDbEnabled on).",
+					);
+					return;
+				}
+				const conv =
+					args.trim() ||
+					ensureConversationIdFor(config, runtime.rt.sessionId, stateDir);
+				const mainDb = mainDbFor(stateDir);
+				const perTurn = turnMetrics(store, mainDb, conv);
+				if (perTurn.length === 0) {
+					ctx.ui.notify(`[mega-compact] no turns recorded for conversation ${conv}.`);
+					return;
+				}
+				const agg = conversationMetrics(store, mainDb, conv);
+				ctx.ui.notify(
+					`[mega-compact] conversation ${conv} — ${agg.turnCount} turns, ${agg.epochCount} epoch(s), total recall ${agg.totalRecall}, total raw msgs ${agg.totalRawMessages}`,
+				);
+				ctx.ui.notify(
+					`  avg dedup unique ratio ${pct(agg.avgDedupUniqueRatio)} · avg compression ratio ${pct(agg.avgCompressionRatio)}`,
+				);
+				for (const t of perTurn) {
+					ctx.ui.notify(
+						`  turn ${t.turnIndex} (epoch ${t.epochId ?? "—"}): recall ${t.recallCount}, raw ${t.rawMessageCount}, dedup ${pct(t.dedupUniqueRatio)}, compress ${pct(t.compressionRatio)}`,
+					);
+				}
+			} catch (e) {
+				ctx.ui.notify(`[mega-compact] /mega-metrics failed: ${String(e)}`);
+			}
+		},
+	});
+
+	pi.registerCommand("mega-fork", {
+		description:
+			"Fork the current conversation at turn N into a new child conversation, rehydrating that turn's injected checkpoints (recall-to-point, not window replay). Usage: /mega-fork <turnIndex> [conversationId]",
+		handler: async (args: string, ctx: ExtensionContext) => {
+			try {
+				runtime.bindRepo(ctx.cwd);
+				const stateDir = runtime.currentStateDir;
+				const store = turnStoreFor(config, stateDir);
+				if (!store) {
+					ctx.ui.notify(
+						"[mega-compact] /mega-fork requires the isolated turns.db (set turnsDbEnabled on).",
+					);
+					return;
+				}
+				const parts = args.trim().split(/\s+/).filter(Boolean);
+				const turnIndex = Number.parseInt(parts[0] ?? "", 10);
+				if (!Number.isFinite(turnIndex)) {
+					ctx.ui.notify("[mega-compact] /mega-fork needs a turn index, e.g. /mega-fork 3");
+					return;
+				}
+				const conv =
+					parts[1] ||
+					ensureConversationIdFor(config, runtime.rt.sessionId, stateDir);
+				const out = forkFromConversation(store, conv, turnIndex);
+				runtime.logger.info("fork", {
+					parent: conv,
+					child: out.childConversationId,
+					turnIndex,
+					checkpoints: out.checkpointIds,
+				});
+				ctx.ui.notify(
+					`[mega-compact] forked ${conv} @ turn ${turnIndex} → ${out.childConversationId}\n` +
+						`  replay set (${out.checkpointIds.length} checkpoint(s)): ${out.checkpointIds.join(", ")}`,
+				);
+			} catch (e) {
+				if (e instanceof ForkError) {
+					ctx.ui.notify(`[mega-compact] /mega-fork: ${e.message}`);
+				} else {
+					ctx.ui.notify(`[mega-compact] /mega-fork failed: ${String(e)}`);
+				}
+			}
+		},
+	});
+}

package/extensions/mega-pipeline/recall.ts

@@ -8,15 +8,25 @@
 
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
-import { recallAndInline, recallAndInlineAsync, formatRecallBlock, type RecallInjectResult } from "../../src/recall.js";
+import {
+	recallAndInline,
+	recallAndInlineAsync,
+	formatRecallBlock,
+	type RecallInjectResult,
+} from "../../src/recall.js";
 import { normalizeSessionId } from "../../src/store.js";
-import { incRecallInjected, incCacheHitTokens, getIndexDir } from "../../src/store/sqlite.js";
-import { ensureConversationId, recordTurn, recordTurnRecall, type RecallSource } from "../../src/store/sqlite/turns.js";
 import {
-  type MegaRuntime,
-  C,
-} from "../mega-runtime.js";
-import { type MegaConfig } from "../mega-config.js";
+	incRecallInjected,
+	incCacheHitTokens,
+	getIndexDir,
+} from "../../src/store/sqlite.js";
+import {
+	ensureConversationIdFor,
+	recordTurnWrite,
+	recordRecallWrite,
+} from "../mega-turn-store.js";
+import { type MegaRuntime, C } from "../mega-runtime.js";
+import type { MegaConfig } from "../mega-config.js";
 
 /**
  * Unified recall (Layer 5). The ONE path that injects. Returns the recall
@@ -24,73 +34,101 @@ import { type MegaConfig } from "../mega-config.js";
  * or report it (command).
  */
 export function doRecall(
-  runtime: MegaRuntime,
-  config: MegaConfig,
-  ctx: ExtensionContext,
-  query: string,
-  source: "resume" | "command",
+	runtime: MegaRuntime,
+	config: MegaConfig,
+	ctx: ExtensionContext,
+	query: string,
+	source: "resume" | "command",
 ) {
-  runtime.bindRepo(ctx.cwd);
-  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-  // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
-  // are already resident in the session, so recall never re-injects context the
-  // model can already see. Best-effort — an empty window just skips dedupe.
-  const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
-  const result = recallAndInline(
-    {
-      sessionId: sid,
-      query,
-      limit: config.autoInlineK,
-      source,
-      skipInjected: true,
-      recallMaxTokens: config.recallMaxTokens,
-      windowDedupe: config.windowDedupe,
-      liveWindow,
-      dedupSim: config.dedupSim,
-    },
-    runtime.store,
-  );
-  runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
-  if (!result.empty && result.toInject.length > 0) {
-    const top = result.toInject[0];
-    const scorePct = Math.round((top.score ?? 0) * 100);
-    const files = top.checkpoint.filesModified ?? [];
-    const label = files.length ? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ") : top.checkpoint.checkpointId;
-    runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
-    runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
-  }
-  if (result.toInject.length > 0) {
-    let sumTokens = 0; for (const h of result.toInject) sumTokens += h.checkpoint.tokenEstimate;
-    runtime.rt.recallInjections += result.toInject.length;
-    runtime.rt.cacheHitTokens += sumTokens;
-    incRecallInjected(result.toInject.length, runtime.currentStateDir);
-    incCacheHitTokens(sumTokens, runtime.currentStateDir);
-    // S43: record recall provenance — which checkpoints/summaries served this
-    // turn, their score + source path. Linked to the turn row written at
-    // turn_end via the conversation+turnIndex. Best-effort + non-fatal.
-    try {
-      const convId = ensureConversationId(sid, runtime.currentStateDir);
-      const turnId = recordTurn({
-        conversationId: convId,
-        sessionId: sid,
-        turnIndex: runtime.currentTurn,
-        startedAt: Date.now(),
-      }, runtime.currentStateDir);
-      recordTurnRecall(
-        turnId,
-        result.toInject.map((h) => ({
-          checkpointId: h.checkpoint.checkpointId,
-          score: h.score,
-          source: (h.raptorLevel !== undefined ? "raptor" : h.repoId ? "cross-repo" : "flat") as RecallSource,
-          raptorLevel: h.raptorLevel,
-        })),
-        runtime.currentStateDir,
-      );
-    } catch {
-      /* non-fatal: recall provenance never breaks the recall path */
-    }
-  }
-  return result;
+	runtime.bindRepo(ctx.cwd);
+	const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+	// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
+	// are already resident in the session, so recall never re-injects context the
+	// model can already see. Best-effort — an empty window just skips dedupe.
+	const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
+	const result = recallAndInline(
+		{
+			sessionId: sid,
+			query,
+			limit: config.autoInlineK,
+			source,
+			skipInjected: true,
+			recallMaxTokens: config.recallMaxTokens,
+			windowDedupe: config.windowDedupe,
+			liveWindow,
+			dedupSim: config.dedupSim,
+		},
+		runtime.store,
+	);
+	runtime.dashboard.event("recall", {
+		source,
+		query: query.slice(0, 120),
+		injected: result.toInject.length,
+		empty: result.empty,
+	});
+	if (!result.empty && result.toInject.length > 0) {
+		const top = result.toInject[0];
+		const scorePct = Math.round((top.score ?? 0) * 100);
+		const files = top.checkpoint.filesModified ?? [];
+		const label = files.length
+			? files
+					.map((f) => f.split("/").pop() ?? f)
+					.slice(0, 2)
+					.join(", ")
+			: top.checkpoint.checkpointId;
+		runtime.pushTicker(
+			`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`,
+		);
+		runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
+	}
+	if (result.toInject.length > 0) {
+		let sumTokens = 0;
+		for (const h of result.toInject) sumTokens += h.checkpoint.tokenEstimate;
+		runtime.rt.recallInjections += result.toInject.length;
+		runtime.rt.cacheHitTokens += sumTokens;
+		incRecallInjected(result.toInject.length, runtime.currentStateDir);
+		incCacheHitTokens(sumTokens, runtime.currentStateDir);
+		// S43: record recall provenance — which checkpoints/summaries served this
+		// turn, their score + source path. Linked to the turn row written at
+		// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
+		try {
+			const convId = ensureConversationIdFor(
+				config,
+				sid,
+				runtime.currentStateDir,
+			);
+			const turnId = recordTurnWrite(
+				config,
+				{
+					conversationId: convId,
+					sessionId: sid,
+					turnIndex: runtime.currentTurn,
+					role: "assistant",
+					startedAt: Date.now(),
+				},
+				runtime.currentStateDir,
+			);
+			recordRecallWrite(
+				config,
+				turnId,
+				result.toInject.map((h) => ({
+					checkpointId: h.checkpoint.checkpointId,
+					score: h.score,
+					source:
+						h.raptorLevel !== undefined
+							? "raptor"
+							: h.repoId
+								? "cross-repo"
+								: "flat",
+					raptorLevel: h.raptorLevel,
+				})),
+				runtime.currentStateDir,
+			);
+		} catch {
+			/* non-fatal: recall provenance never breaks the recall path */
+		}
+	}
+	return result;
 }
 
 /**
@@ -105,72 +143,96 @@ export function doRecall(
  * the same-repo result unchanged.
  */
 export async function doRecallAsync(
-  runtime: MegaRuntime,
-  config: MegaConfig,
-  ctx: ExtensionContext,
-  query: string,
-  source: "resume" | "command",
-  opts: { crossRepo?: boolean } = {},
+	runtime: MegaRuntime,
+	config: MegaConfig,
+	ctx: ExtensionContext,
+	query: string,
+	source: "resume" | "command",
+	opts: { crossRepo?: boolean } = {},
 ): Promise<RecallInjectResult> {
-  runtime.bindRepo(ctx.cwd);
-  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-  const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
-  // Sync same-repo first (fast, never blocks).
-  const sameRepo = recallAndInline(
-    {
-      sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
-      recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
-      liveWindow, dedupSim: config.dedupSim,
-    },
-    runtime.store,
-  );
-  if (!config.crossRepoEnabled || !opts.crossRepo) return sameRepo;
-  if (sameRepo.toInject.length >= config.autoInlineK) return sameRepo; // same-repo satisfied
-  // Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
-  try {
-    const x = await recallAndInlineAsync(
-      {
-        sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
-        recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
-        liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
-        // F2: resolve the machine-wide index dir via the shared resolver so the
-        // cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
-        // unset. The env var still wins when set (getIndexDir checks it first);
-        // the default (~/.mega-compact-index) is the same DB mega-commands and the
-        // dashboard read, so injection counts stay consistent. Without this, a
-        // bare `process.env` read returns undefined → cross-repo hits re-inject in
-        // every new session (the global injected-set is never consulted).
-        globalIndexDir: getIndexDir(),
-      },
-      runtime.store,
-    );
-    runtime.dashboard.event("recall-crossrepo", {
-      source, query: query.slice(0, 120), injected: x.toInject.length,
-      sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
-    });
-    // Merge, dedup by checkpointId, respect the same token cap by reformatting.
-    const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
-    const merged = [...sameRepo.toInject];
-    for (const h of x.toInject) {
-      if (!seen.has(h.checkpoint.checkpointId)) { merged.push(h); seen.add(h.checkpoint.checkpointId); }
-    }
-    const block = merged.length ? formatRecallBlock(merged) : "";
-    if (merged.length > 0) {
-      let sumTokens = 0; for (const h of merged) sumTokens += h.checkpoint.tokenEstimate;
-      runtime.rt.recallInjections += merged.length;
-      runtime.rt.cacheHitTokens += sumTokens;
-      incRecallInjected(merged.length, runtime.currentStateDir);
-      incCacheHitTokens(sumTokens, runtime.currentStateDir);
-    }
-    return {
-      toInject: merged,
-      report: merged.map((h) => `  • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
-      block,
-      empty: merged.length === 0,
-    };
-  } catch {
-    return sameRepo; // cross-repo failure → same-repo only (non-fatal)
-  }
+	runtime.bindRepo(ctx.cwd);
+	const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+	const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
+	// Sync same-repo first (fast, never blocks).
+	const sameRepo = recallAndInline(
+		{
+			sessionId: sid,
+			query,
+			limit: config.autoInlineK,
+			source,
+			skipInjected: true,
+			recallMaxTokens: config.recallMaxTokens,
+			windowDedupe: config.windowDedupe,
+			liveWindow,
+			dedupSim: config.dedupSim,
+		},
+		runtime.store,
+	);
+	if (!config.crossRepoEnabled || !opts.crossRepo) return sameRepo;
+	if (sameRepo.toInject.length >= config.autoInlineK) return sameRepo; // same-repo satisfied
+	// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
+	try {
+		const x = await recallAndInlineAsync(
+			{
+				sessionId: sid,
+				query,
+				limit: config.autoInlineK,
+				source,
+				skipInjected: true,
+				recallMaxTokens: config.recallMaxTokens,
+				windowDedupe: config.windowDedupe,
+				liveWindow,
+				dedupSim: config.crossRepoCosine,
+				crossRepo: true,
+				// F2: resolve the machine-wide index dir via the shared resolver so the
+				// cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
+				// unset. The env var still wins when set (getIndexDir checks it first);
+				// the default (~/.mega-compact-index) is the same DB mega-commands and the
+				// dashboard read, so injection counts stay consistent. Without this, a
+				// bare `process.env` read returns undefined → cross-repo hits re-inject in
+				// every new session (the global injected-set is never consulted).
+				globalIndexDir: getIndexDir(),
+			},
+			runtime.store,
+		);
+		runtime.dashboard.event("recall-crossrepo", {
+			source,
+			query: query.slice(0, 120),
+			injected: x.toInject.length,
+			sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
+		});
+		// Merge, dedup by checkpointId, respect the same token cap by reformatting.
+		const seen = new Set(
+			sameRepo.toInject.map((h) => h.checkpoint.checkpointId),
+		);
+		const merged = [...sameRepo.toInject];
+		for (const h of x.toInject) {
+			if (!seen.has(h.checkpoint.checkpointId)) {
+				merged.push(h);
+				seen.add(h.checkpoint.checkpointId);
+			}
+		}
+		const block = merged.length ? formatRecallBlock(merged) : "";
+		if (merged.length > 0) {
+			let sumTokens = 0;
+			for (const h of merged) sumTokens += h.checkpoint.tokenEstimate;
+			runtime.rt.recallInjections += merged.length;
+			runtime.rt.cacheHitTokens += sumTokens;
+			incRecallInjected(merged.length, runtime.currentStateDir);
+			incCacheHitTokens(sumTokens, runtime.currentStateDir);
+		}
+		return {
+			toInject: merged,
+			report: merged.map(
+				(h) =>
+					`  • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`,
+			),
+			block,
+			empty: merged.length === 0,
+		};
+	} catch {
+		return sameRepo; // cross-repo failure → same-repo only (non-fatal)
+	}
 }
 
 /**
@@ -180,18 +242,19 @@ export async function doRecallAsync(
  * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
  */
 function extractLiveWindow(ctx: ExtensionContext): string[] {
-  try {
-    const entries = ctx.sessionManager.getEntries();
-    const texts: string[] = [];
-    for (const e of entries) {
-      for (const m of sessionEntryToContextMessages(e)) {
-        const c = (m as { content?: unknown }).content;
-        if (typeof c === "string") texts.push(c);
-        else if (Array.isArray(c)) texts.push(c.map((b: { text?: string }) => b.text ?? "").join(" "));
-      }
-    }
-    return texts;
-  } catch {
-    return [];
-  }
+	try {
+		const entries = ctx.sessionManager.getEntries();
+		const texts: string[] = [];
+		for (const e of entries) {
+			for (const m of sessionEntryToContextMessages(e)) {
+				const c = (m as { content?: unknown }).content;
+				if (typeof c === "string") texts.push(c);
+				else if (Array.isArray(c))
+					texts.push(c.map((b: { text?: string }) => b.text ?? "").join(" "));
+			}
+		}
+		return texts;
+	} catch {
+		return [];
+	}
 }