pi-mega-compact 0.7.2 → 0.7.4

package/extensions/mega-config.ts

@@ -75,6 +75,13 @@ export interface MegaConfig {
    *  trim + pi native auto-compaction instead (compact and continue). Kept for
    *  one release as rollback. */
   legacyDurableTrim: boolean;
+  /** S27: durable raw-transcript DB mirror (MEGACOMPACT_DB_MIRROR). When on,
+   *  raw message bytes + checkpoint-epoch bookkeeping are appended to the
+   *  SQLite store so a compacted window can be rehydrated locally instead of
+   *  from the pi runtime transcript. Default OFF — additive, no behavior
+   *  change until flipped on. legacyDurableTrim takes precedence (the legacy
+   *  v0.4.28 ctx.compact() path does not emit the S27 mirror hook). */
+  dbMirror: boolean;
   /** Cross-repo recall enabled (S17). Resume + /mega-recall --cross-repo can
    *  pull checkpoints from OTHER repos via the PGlite HNSW index. Default true. */
   crossRepoEnabled: boolean;
@@ -214,6 +221,7 @@ export function loadConfig(): MegaConfig {
     dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
     raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
     legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
+    dbMirror: envBool("MEGACOMPACT_DB_MIRROR", false),
     crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
     crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
     memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),

package/extensions/mega-events.test.ts

@@ -0,0 +1,55 @@
+/**
+ * Tests for mega-events extension — DB-mirror event wiring.
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { openStore, listCheckpointEpochs, countRawTranscript } from "../src/store/sqlite.js";
+
+function makeTmp(): string {
+  return mkdtempSync(join(tmpdir(), "mega-events-test-"));
+}
+
+describe("mega-events: DB-mirror integration", () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = makeTmp();
+    // openStore creates the tables as a side effect
+    openStore(dir);
+  });
+
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("openStore creates checkpoint_epochs and raw_transcript tables", () => {
+    const db = openStore(dir);
+    // Should not throw — tables exist
+    const epochs = listCheckpointEpochs(db);
+    assert.ok(Array.isArray(epochs));
+    const count = countRawTranscript(db);
+    assert.equal(count, 0);
+  });
+
+  it("DB-mirror flag defaults to false when env is unset", () => {
+    delete process.env.MEGACOMPACT_DB_MIRROR;
+    // Re-import to pick up env
+    // The extension checks env at load time, so just verify the env is absent
+    assert.equal(process.env.MEGACOMPACT_DB_MIRROR, undefined);
+  });
+
+  it("DB-mirror flag is enabled when env is '1'", () => {
+    process.env.MEGACOMPACT_DB_MIRROR = "1";
+    assert.equal(process.env.MEGACOMPACT_DB_MIRROR, "1");
+    delete process.env.MEGACOMPACT_DB_MIRROR;
+  });
+
+  it("DB-mirror flag is enabled when env is 'true'", () => {
+    process.env.MEGACOMPACT_DB_MIRROR = "true";
+    assert.equal(process.env.MEGACOMPACT_DB_MIRROR, "true");
+    delete process.env.MEGACOMPACT_DB_MIRROR;
+  });
+});

package/extensions/mega-events.ts

@@ -12,9 +12,12 @@ import type {
 	ExtensionContext,
 	ContextEvent,
 	SessionBeforeCompactEvent,
+	SessionCompactEvent,
 } from "@earendil-works/pi-coding-agent";
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import { normalizeSessionId } from "../src/store.js";
+import { openStore, appendRawTranscript, writeCheckpointEpoch, type CheckpointEpoch } from "../src/store/sqlite.js";
+import { epochIdFor } from "../src/mirror/epoch.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens, estimateBlockTokens } from "../src/tokens.js";
 import {
@@ -40,6 +43,39 @@ import {
 	memoryReviewCadence,
 	type MegaConfig,
 } from "./mega-config.js";
+import type { RawTranscriptRow } from "../src/store/sqlite.js";
+import { createHash } from "node:crypto";
+
+/**
+ * Convert a pi AgentMessage to a RawTranscriptRow for the DB mirror.
+ * content_bytes is canonical JSON (sorted keys) for deterministic hashing.
+ * Returns null if the message has no usable content.
+ */
+function toRawTranscriptRow(
+	msg: AgentMessage,
+	sessionId: string,
+	epochId: string,
+): RawTranscriptRow | null {
+	// Narrow to Message union (has content + timestamp).
+	const m = msg as { role?: string; content?: unknown; timestamp?: number; toolName?: string };
+	const content = m.content;
+	if (content == null || content === "") return null;
+	// Canonical form: sort object keys for deterministic hashing.
+	const contentBytes = typeof content === "string"
+		? content
+		: JSON.stringify(content, Object.keys(content as object).sort());
+	const contentHash = createHash("sha256").update(contentBytes).digest("hex");
+	return {
+		contentHash,
+		sessionId,
+		seq: 0, // assigned by appendRawTranscript (COALESCE(MAX(seq),0)+1)
+		role: m.role ?? "unknown",
+		contentBytes,
+		toolName: m.toolName ?? null,
+		messageTimestamp: m.timestamp ?? null,
+		checkpointEpoch: epochId,
+	};
+}
 
 /**
  * DIAG accessor for the headless test harness: the most recently constructed
@@ -271,7 +307,20 @@ export function registerEventHandlers(
 				// trim we ALWAYS nudge so the agent reliably restarts. Debounced 30s.
 				let didDurableTrim = false;
 				if (idle && overThreshold && now >= runtime.debounceUntil) {
-					if (!piCompactWouldNoop(ctx)) {
+					// COMPACT-DEDUP FIX: skip the manual durable-trim trigger when pi's
+					// NATIVE auto-compaction just fired (or is in-flight). pi emits
+					// agent_end BEFORE its own _checkCompaction (per its docstring:
+					// "Called after agent_end and before prompt submission"), so a
+					// synchronous `piCompactWouldNoop` branch check misses a native
+					// compaction that hasn't appended its entry yet — calling
+					// ctx.compact() then races with pi and throws "Already compacted"
+					// to the user. The `lastCompactAt` cooldown (updated by the
+					// session_compact listener for EVERY compaction, native or
+					// extension-supplied) closes that race window.
+					const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
+					if (sinceCompact < 10_000) {
+						runtime.diagAgentEndDurableSkipRecent++;
+					} else if (!piCompactWouldNoop(ctx)) {
 						runtime.debounceUntil = now + 2000;
 						runtime.diagAgentEndDurable++;
 						runtime.logger.info("agent-end-durable-trigger", {
@@ -280,7 +329,7 @@ export function registerEventHandlers(
 							thresholdTokens: config.thresholdTokens,
 							queued,
 						});
-						ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
+						ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
 						didDurableTrim = true;
 					}
 				}
@@ -370,6 +419,22 @@ export function registerEventHandlers(
 			estimateSessionTokens(view) ??
 			Math.round((pct / 100) * (usage?.contextWindow ?? 0));
 
+		// S27 DB-mirror: append ALL incoming messages to raw_transcript.
+		// Runs BEFORE fast-gate so every message is captured, even if we
+		// don't compact this turn. Append is idempotent (content_hash PK).
+		if (config.dbMirror) {
+			try {
+				const db = openStore(runtime.currentStateDir);
+				const epochId = epochIdFor(runtime.rt.sessionId);
+				for (const msg of messages) {
+					const raw = toRawTranscriptRow(msg, runtime.rt.sessionId, epochId);
+					if (raw) appendRawTranscript(db, raw);
+				}
+			} catch (e) {
+				runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
+			}
+		}
+
 		// FAST GATE: token-based (tier% of the window), not a static amount.
 		if (currentTokens < runtime.effectiveThreshold) {
 			runtime.diagCtxFastGate++;
@@ -401,6 +466,41 @@ export function registerEventHandlers(
 			return;
 		}
 
+		// S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
+		// This makes the cache key stable across identical compactions.
+		if (config.dbMirror) {
+			try {
+				const db = openStore(runtime.currentStateDir);
+				const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
+				const epoch: CheckpointEpoch = {
+					epochId: epochIdFor(cpId),
+					sessionId: runtime.rt.sessionId,
+					startedSeq: 0,
+					committedSeq: ran.result.compactedFrom,
+					checkpointId: cpId,
+					cutIndex: ran.result.compactedFrom,
+					summaryMessageText: ran.result.summary,
+					createdAt: Date.now(),
+				};
+				writeCheckpointEpoch(db, epoch);
+				// S27 Task 6: Fire-and-forget dedup pipeline.
+				// Deduplicates raw_transcript rows for the compacted range.
+				try {
+					const { dedupTranscript } = await import("../src/mirror/dedup.js");
+					dedupTranscript(
+						db,
+						runtime.rt.sessionId,
+						0,
+						ran.result.compactedFrom,
+					);
+				} catch (_dedupErr) {
+					// Fire-and-forget: dedup failure is non-fatal
+				}
+			} catch (e) {
+				runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
+			}
+		}
+
 		// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
 		// manual compact path aborts the in-flight turn — only used behind the flag.
 		// Read live from env (in addition to the load-time config) so the flag can be
@@ -411,8 +511,13 @@ export function registerEventHandlers(
 			process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" ||
 			process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
 		if (legacy) {
-			if (piCompactWouldNoop(ctx)) return;
-			ctx.compact({ customInstructions: undefined });
+			// COMPACT-DEDUP FIX: same race guard as the agent_end path. Skip when a
+			// NATIVE compaction just fired (avoids racing pi and surfacing a spurious
+			// "Already compacted" / "Nothing to compact" toast). Uses lastNativeCompactAt
+			// (NOT lastCompactAt, which runCompact also stamps for our own checkpoint).
+			const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
+			if (sinceCompact < 10_000 || piCompactWouldNoop(ctx)) return;
+			ctx.compact({ customInstructions: undefined }); // race-guarded by lastNativeCompactAt cooldown (ctx.compact returns void → not catchable; the cooldown prevents the call)
 			return;
 		}
 
@@ -551,6 +656,23 @@ export function registerEventHandlers(
 		},
 	);
 
+		// COMPACT-DEDUP FIX: track EVERY compaction (native + extension-supplied)
+		// so the agent_end durable-trim guard can skip a redundant ctx.compact()
+		// when pi just compacted. Without this, agent_end fires ctx.compact()
+		// synchronously AFTER pi's native auto-compaction appended a compaction
+		// entry but BEFORE our branch read sees it on the next tick — racing
+		// into a user-facing "Already compacted" throw. `lastCompactAt` is the
+		// race-closing signal: any compaction (manual/threshold/overflow, ours
+		// or pi's own) stamps it, and the agent_end guard skips for 10s.
+		pi.on("session_compact", async (_event: SessionCompactEvent, _ctx: ExtensionContext) => {
+			runtime.rt.lastNativeCompactAt = Date.now();
+			runtime.rt.lastCompactAt = Date.now();
+			runtime.logger.info("session-compacted", {
+				sessionId: runtime.rt.sessionId,
+				at: runtime.rt.lastCompactAt,
+			});
+		});
+
 	/**
 	 * Build a minimal fallback compaction so pi never runs its throwing compact().
 	 *