pi-mega-compact 0.7.2 → 0.7.4

package/dist/extensions/mega-config.js

@@ -135,6 +135,7 @@ export function loadConfig() {
         dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
         raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
         legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
+        dbMirror: envBool("MEGACOMPACT_DB_MIRROR", false),
         crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
         crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
         memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),

package/dist/extensions/mega-events.js

@@ -7,6 +7,8 @@
  * sync and delegates the heavy lifting to the pipeline + command modules.
  */
 import { normalizeSessionId } from "../src/store.js";
+import { openStore, appendRawTranscript, writeCheckpointEpoch } from "../src/store/sqlite.js";
+import { epochIdFor } from "../src/mirror/epoch.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens, estimateBlockTokens } from "../src/tokens.js";
 import { recentUserQuery, WIDGET_KEY, } from "./mega-runtime.js";
@@ -15,6 +17,34 @@ import { recallMemoriesAndInline } from "../src/recall.js";
 import { driveNativeCompaction, } from "./mega-compact-driver.js";
 import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
 import { pressureFromPct, memoryReviewCadence, } from "./mega-config.js";
+import { createHash } from "node:crypto";
+/**
+ * Convert a pi AgentMessage to a RawTranscriptRow for the DB mirror.
+ * content_bytes is canonical JSON (sorted keys) for deterministic hashing.
+ * Returns null if the message has no usable content.
+ */
+function toRawTranscriptRow(msg, sessionId, epochId) {
+    // Narrow to Message union (has content + timestamp).
+    const m = msg;
+    const content = m.content;
+    if (content == null || content === "")
+        return null;
+    // Canonical form: sort object keys for deterministic hashing.
+    const contentBytes = typeof content === "string"
+        ? content
+        : JSON.stringify(content, Object.keys(content).sort());
+    const contentHash = createHash("sha256").update(contentBytes).digest("hex");
+    return {
+        contentHash,
+        sessionId,
+        seq: 0, // assigned by appendRawTranscript (COALESCE(MAX(seq),0)+1)
+        role: m.role ?? "unknown",
+        contentBytes,
+        toolName: m.toolName ?? null,
+        messageTimestamp: m.timestamp ?? null,
+        checkpointEpoch: epochId,
+    };
+}
 /**
  * DIAG accessor for the headless test harness: the most recently constructed
  * MegaRuntime, so a test that loads the compiled extension via its default
@@ -223,7 +253,21 @@ export function registerEventHandlers(pi, runtime, config) {
                 // trim we ALWAYS nudge so the agent reliably restarts. Debounced 30s.
                 let didDurableTrim = false;
                 if (idle && overThreshold && now >= runtime.debounceUntil) {
-                    if (!piCompactWouldNoop(ctx)) {
+                    // COMPACT-DEDUP FIX: skip the manual durable-trim trigger when pi's
+                    // NATIVE auto-compaction just fired (or is in-flight). pi emits
+                    // agent_end BEFORE its own _checkCompaction (per its docstring:
+                    // "Called after agent_end and before prompt submission"), so a
+                    // synchronous `piCompactWouldNoop` branch check misses a native
+                    // compaction that hasn't appended its entry yet — calling
+                    // ctx.compact() then races with pi and throws "Already compacted"
+                    // to the user. The `lastCompactAt` cooldown (updated by the
+                    // session_compact listener for EVERY compaction, native or
+                    // extension-supplied) closes that race window.
+                    const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
+                    if (sinceCompact < 10_000) {
+                        runtime.diagAgentEndDurableSkipRecent++;
+                    }
+                    else if (!piCompactWouldNoop(ctx)) {
                         runtime.debounceUntil = now + 2000;
                         runtime.diagAgentEndDurable++;
                         runtime.logger.info("agent-end-durable-trigger", {
@@ -232,7 +276,7 @@ export function registerEventHandlers(pi, runtime, config) {
                             thresholdTokens: config.thresholdTokens,
                             queued,
                         });
-                        ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
+                        ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
                         didDurableTrim = true;
                     }
                 }
@@ -309,6 +353,23 @@ export function registerEventHandlers(pi, runtime, config) {
         const currentTokens = usage?.tokens ??
             estimateSessionTokens(view) ??
             Math.round((pct / 100) * (usage?.contextWindow ?? 0));
+        // S27 DB-mirror: append ALL incoming messages to raw_transcript.
+        // Runs BEFORE fast-gate so every message is captured, even if we
+        // don't compact this turn. Append is idempotent (content_hash PK).
+        if (config.dbMirror) {
+            try {
+                const db = openStore(runtime.currentStateDir);
+                const epochId = epochIdFor(runtime.rt.sessionId);
+                for (const msg of messages) {
+                    const raw = toRawTranscriptRow(msg, runtime.rt.sessionId, epochId);
+                    if (raw)
+                        appendRawTranscript(db, raw);
+                }
+            }
+            catch (e) {
+                runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
+            }
+        }
         // FAST GATE: token-based (tier% of the window), not a static amount.
         if (currentTokens < runtime.effectiveThreshold) {
             runtime.diagCtxFastGate++;
@@ -336,6 +397,37 @@ export function registerEventHandlers(pi, runtime, config) {
             runtime.diagCtxRunSkipped++;
             return;
         }
+        // S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
+        // This makes the cache key stable across identical compactions.
+        if (config.dbMirror) {
+            try {
+                const db = openStore(runtime.currentStateDir);
+                const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
+                const epoch = {
+                    epochId: epochIdFor(cpId),
+                    sessionId: runtime.rt.sessionId,
+                    startedSeq: 0,
+                    committedSeq: ran.result.compactedFrom,
+                    checkpointId: cpId,
+                    cutIndex: ran.result.compactedFrom,
+                    summaryMessageText: ran.result.summary,
+                    createdAt: Date.now(),
+                };
+                writeCheckpointEpoch(db, epoch);
+                // S27 Task 6: Fire-and-forget dedup pipeline.
+                // Deduplicates raw_transcript rows for the compacted range.
+                try {
+                    const { dedupTranscript } = await import("../src/mirror/dedup.js");
+                    dedupTranscript(db, runtime.rt.sessionId, 0, ran.result.compactedFrom);
+                }
+                catch (_dedupErr) {
+                    // Fire-and-forget: dedup failure is non-fatal
+                }
+            }
+            catch (e) {
+                runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
+            }
+        }
         // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
         // manual compact path aborts the in-flight turn — only used behind the flag.
         // Read live from env (in addition to the load-time config) so the flag can be
@@ -345,9 +437,14 @@ export function registerEventHandlers(pi, runtime, config) {
             process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" ||
             process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
         if (legacy) {
-            if (piCompactWouldNoop(ctx))
+            // COMPACT-DEDUP FIX: same race guard as the agent_end path. Skip when a
+            // NATIVE compaction just fired (avoids racing pi and surfacing a spurious
+            // "Already compacted" / "Nothing to compact" toast). Uses lastNativeCompactAt
+            // (NOT lastCompactAt, which runCompact also stamps for our own checkpoint).
+            const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
+            if (sinceCompact < 10_000 || piCompactWouldNoop(ctx))
                 return;
-            ctx.compact({ customInstructions: undefined });
+            ctx.compact({ customInstructions: undefined }); // race-guarded by lastNativeCompactAt cooldown (ctx.compact returns void → not catchable; the cooldown prevents the call)
             return;
         }
         // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
@@ -482,6 +579,22 @@ export function registerEventHandlers(pi, runtime, config) {
         // Absolute last resort: let pi run its own (may throw "Nothing to compact").
         return {};
     });
+    // COMPACT-DEDUP FIX: track EVERY compaction (native + extension-supplied)
+    // so the agent_end durable-trim guard can skip a redundant ctx.compact()
+    // when pi just compacted. Without this, agent_end fires ctx.compact()
+    // synchronously AFTER pi's native auto-compaction appended a compaction
+    // entry but BEFORE our branch read sees it on the next tick — racing
+    // into a user-facing "Already compacted" throw. `lastCompactAt` is the
+    // race-closing signal: any compaction (manual/threshold/overflow, ours
+    // or pi's own) stamps it, and the agent_end guard skips for 10s.
+    pi.on("session_compact", async (_event, _ctx) => {
+        runtime.rt.lastNativeCompactAt = Date.now();
+        runtime.rt.lastCompactAt = Date.now();
+        runtime.logger.info("session-compacted", {
+            sessionId: runtime.rt.sessionId,
+            at: runtime.rt.lastCompactAt,
+        });
+    });
     /**
      * Build a minimal fallback compaction so pi never runs its throwing compact().
      *

package/dist/extensions/mega-events.test.js

@@ -0,0 +1,47 @@
+/**
+ * Tests for mega-events extension — DB-mirror event wiring.
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { openStore, listCheckpointEpochs, countRawTranscript } from "../src/store/sqlite.js";
+function makeTmp() {
+    return mkdtempSync(join(tmpdir(), "mega-events-test-"));
+}
+describe("mega-events: DB-mirror integration", () => {
+    let dir;
+    beforeEach(() => {
+        dir = makeTmp();
+        // openStore creates the tables as a side effect
+        openStore(dir);
+    });
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("openStore creates checkpoint_epochs and raw_transcript tables", () => {
+        const db = openStore(dir);
+        // Should not throw — tables exist
+        const epochs = listCheckpointEpochs(db);
+        assert.ok(Array.isArray(epochs));
+        const count = countRawTranscript(db);
+        assert.equal(count, 0);
+    });
+    it("DB-mirror flag defaults to false when env is unset", () => {
+        delete process.env.MEGACOMPACT_DB_MIRROR;
+        // Re-import to pick up env
+        // The extension checks env at load time, so just verify the env is absent
+        assert.equal(process.env.MEGACOMPACT_DB_MIRROR, undefined);
+    });
+    it("DB-mirror flag is enabled when env is '1'", () => {
+        process.env.MEGACOMPACT_DB_MIRROR = "1";
+        assert.equal(process.env.MEGACOMPACT_DB_MIRROR, "1");
+        delete process.env.MEGACOMPACT_DB_MIRROR;
+    });
+    it("DB-mirror flag is enabled when env is 'true'", () => {
+        process.env.MEGACOMPACT_DB_MIRROR = "true";
+        assert.equal(process.env.MEGACOMPACT_DB_MIRROR, "true");
+        delete process.env.MEGACOMPACT_DB_MIRROR;
+    });
+});