pi-mega-compact 0.4.21 → 0.4.23

package/extensions/dashboard-server.ts

@@ -16,7 +16,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync }
 import { homedir } from "node:os";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
-import Database from "better-sqlite3";
+import { DatabaseSync } from "node:sqlite";
 
 // --- Multi-repo index (Phase 5b) ------------------------------------------------
 // The extension writes a machine-wide repo registry into a single SQLite DB
@@ -54,11 +54,11 @@ interface IndexRepo {
 function readIndex(): { updatedAt: string; summary: unknown; repos: unknown[] } | null {
   const indexPath = join(getIndexDir(), "index.sqlite");
   if (!existsSync(indexPath)) return null;
-  let db: Database.Database | undefined;
+  let db: DatabaseSync | undefined;
   try {
     // Read-only + immutable WAL so a concurrent writer's WAL never blocks us.
-    db = new Database(indexPath, { readonly: true, fileMustExist: true });
-    db.pragma("journal_mode = WAL");
+    db = new DatabaseSync(indexPath, { readOnly: true });
+    db.exec("PRAGMA journal_mode = WAL");
     const rows = db
       .prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC")
       .all() as Record<string, unknown>[];

package/extensions/mega-compact-driver.ts

@@ -0,0 +1,105 @@
+/**
+ * mega-compact-driver.ts — the durable-trim driver (Fix B).
+ *
+ * The read-path token-growth bug: the old design cancelled pi's native
+ * compaction (`{ cancel: true }`) and did its own ephemeral `context`-hook
+ * drop. That drop only affected the outgoing request — the on-disk transcript
+ * was never trimmed (the session manager is read-only for extensions). So on
+ * resume pi reloaded the FULL transcript and we ADDED a recall block on top →
+ * more tokens than before compaction.
+ *
+ * The fix: on `session_before_compact` we RUN the Trident pipeline to produce a
+ * genuinely compressed summary, then RETURN it as a `CompactionResult`. pi
+ * durably writes our summary into a `compactionSummary` entry AND truncates the
+ * transcript from `firstKeptEntryId`. After that, resume reloads the already-
+ * trimmed transcript (summary baked in) — no additive re-injection, no token
+ * growth.
+ *
+ * We reuse pi's `preparation.firstKeptEntryId` (pi already computed the cut
+ * honoring the anchor-floor + tool-pair guards — PREVENT-PI-002) rather than
+ * recomputing it, so we cannot hand pi a boundary that splits a tool pair.
+ */
+
+import type { SessionBeforeCompactEvent } from "@earendil-works/pi-coding-agent";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import { compactSession } from "../src/engine.js";
+import { toEngineMessages } from "../src/adapt.js";
+import { estimateBlockTokens, estimateSessionTokens } from "../src/tokens.js";
+import type { MegaRuntime } from "./mega-runtime.js";
+import type { MegaConfig } from "./mega-config.js";
+import { recallRaptorRootSummary } from "../src/dedup/raptor/index.js";
+
+export interface NativeCompactionResult {
+  /** Our trimmed summary + the pi entry to keep from (durable trim). */
+  compaction: {
+    summary: string;
+    firstKeptEntryId: string;
+    tokensBefore: number;
+    estimatedTokensAfter: number;
+  };
+}
+
+/**
+ * Build our durable compaction result from pi's pre-computed preparation.
+ *
+ * Returns undefined when there is nothing to summarize (pi will then run its
+ * own native compaction, or skip). Never throws for "empty" — best-effort.
+ */
+export function driveNativeCompaction(
+  event: SessionBeforeCompactEvent,
+  runtime: MegaRuntime,
+  config: MegaConfig,
+): NativeCompactionResult | undefined {
+  const prep = event.preparation;
+  if (!prep) return undefined;
+
+  const sid = runtime.rt.sessionId;
+  const messagesToSummarize: AgentMessage[] = prep.messagesToSummarize ?? [];
+  if (messagesToSummarize.length === 0) return undefined;
+
+  const engineView = toEngineMessages(messagesToSummarize);
+  // We don't drop anything here — pi keeps from prep.firstKeptEntryId. We only
+  // summarize the region pi is about to discard.
+  const keepFrom = engineView.length;
+
+  const result = compactSession(
+    {
+      sessionId: sid,
+      messages: engineView,
+      keepFrom,
+      timestamp: Date.now(),
+      useExtractiveSummary: true,
+    },
+    runtime.store,
+  );
+  if (result.skipped) return undefined;
+
+  // Prefer the RAPTOR root summary when the tree is built + enabled (Fix D):
+  // it is a session-level compressed summary, broader than one slice's. Fall
+  // back to the extractive topicSummary of this slice.
+  let summary = result.summary;
+  if (config.raptorEnabled) {
+    const root = recallRaptorRootSummary(sid, runtime.currentStateDir);
+    if (root) summary = root;
+  }
+
+  const tokensBefore = prep.tokensBefore ?? estimateSessionTokens(engineView);
+  const summaryTokens = estimateBlockTokens(summary);
+  // pi keeps the tail from firstKeptEntryId; our summary replaces the discarded
+  // region. Honest saved = discarded-region tokens − our summary tokens.
+  const savedTokens = Math.max(0, tokensBefore - summaryTokens);
+
+  runtime.rt.lastCompactedFrom = keepFrom;
+  runtime.rt.lastCompactedTokens = tokensBefore;
+  runtime.rt.tokensSaved += savedTokens;
+  runtime.rt.persistedThisSession = true;
+
+  return {
+    compaction: {
+      summary,
+      firstKeptEntryId: prep.firstKeptEntryId,
+      tokensBefore,
+      estimatedTokensAfter: summaryTokens,
+    },
+  };
+}

package/extensions/mega-compact.test.ts

@@ -45,6 +45,7 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
   let statusKey: string | undefined;
   let statusText: string | undefined;
   const notifies: string[] = [];
+  const compactCalls: any[] = [];
 
   // Minimal AgentMessage factory for the session we project into the extension.
   function msg(role: string, text: string, toolName?: string): AgentMessage {
@@ -107,7 +108,30 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
       hasPendingMessages: () => false,
       shutdown: () => {},
       getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
-      compact: () => {},
+      // Faithful mock: ctx.compact() starts pi's flow, which fires the
+      // session_before_compact handler (where WE supply the durable trim).
+      compact: (opts?: any) => {
+        compactCalls.push(opts);
+        if (handlers["session_before_compact"]) {
+          return handlers["session_before_compact"](
+            {
+              type: "session_before_compact",
+              reason: "threshold",
+              willRetry: false,
+              signal: undefined,
+              // pi computed the cut honoring anchor floor + tool-pair (PREVENT-PI-002);
+              // our handler reuses it as firstKeptEntryId.
+              preparation: {
+                firstKeptEntryId: "e2",
+                messagesToSummarize: session.slice(0, 2),
+                tokensBefore: 500,
+              },
+            } as any,
+            makeCtx(),
+          );
+        }
+        return undefined;
+      },
       getSystemPrompt: () => "system base",
       ...over,
     } as any;
@@ -143,14 +167,14 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
   mod.default(pi);
 
   return {
-    stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies,
+    stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies, compactCalls,
     fire: (ev: string, event: any, ctx: any) => handlers[ev](event, ctx),
     ctx: makeCtx,
     session,
   };
 }
 
-test("auto-trigger: past threshold persists a chkpt and drops context", async () => {
+test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
   const h = harness();
   const messages = h.session;
   const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
@@ -159,27 +183,50 @@ test("auto-trigger: past threshold persists a chkpt and drops context", async ()
   const { listCheckpoints } = await import("../src/store/sqlite.js");
   assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
   assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
-  // Context dropped (the compacted range was trimmed).
-  assert.ok(res && Array.isArray(res.messages), "context handler returns filtered messages");
-  assert.ok((res.messages as any[]).length < messages.length, "outgoing context shrank");
+  // The context handler no longer drops messages itself (that was ephemeral —
+  // the read-path token-growth bug). It triggers pi's compaction flow, which
+  // calls our session_before_compact handler to supply the DURABLE trim.
+  assert.equal(res, undefined, "context handler returns nothing (no local drop)");
+  assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim");
+  // The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
+  assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
 });
 
-test("session_before_compact cancels once we've persisted", async () => {
+test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
   const h = harness();
-  const ctx = h.ctx();
-  // First fire the auto-trigger so a checkpoint is persisted this session.
-  await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
-  // Now pi tries to compact natively — we must cancel (no double-compact).
-  const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "overflow", willRetry: true, preparation: {}, signal: undefined } as any, ctx);
-  assert.deepEqual(res, { cancel: true });
+  // pi fires session_before_compact with its own computed preparation.
+  const res = await h.fire(
+    "session_before_compact",
+    {
+      type: "session_before_compact",
+      reason: "overflow",
+      willRetry: true,
+      preparation: { firstKeptEntryId: "e2", messagesToSummarize: h.session.slice(0, 2), tokensBefore: 500 },
+      signal: undefined,
+    } as any,
+    h.ctx(),
+  );
+  assert.ok(res && res.compaction, "returns a compaction result");
+  assert.equal(res.compaction.firstKeptEntryId, "e2", "reuses pi's cut boundary (PREVENT-PI-002 safe)");
+  assert.ok(typeof res.compaction.summary === "string" && res.compaction.summary.length > 0, "our summary supplied");
+  assert.ok(res.compaction.tokensBefore >= 0, "tokensBefore reported");
 });
 
-test("session_before_compact does NOT cancel when nothing persisted", async () => {
+test("session_before_compact falls back to pi when nothing to summarize", async () => {
   const h = harness();
-  const ctx = h.ctx();
-  // Do NOT fire context first; this session has no checkpoint.
-  const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "threshold", willRetry: false, preparation: {}, signal: undefined } as any, ctx);
-  assert.deepEqual(res, {});
+  // Empty preparation → no messages to summarize → return {} so pi compacts natively.
+  const res = await h.fire(
+    "session_before_compact",
+    {
+      type: "session_before_compact",
+      reason: "threshold",
+      willRetry: false,
+      preparation: { firstKeptEntryId: "e0", messagesToSummarize: [], tokensBefore: 0 },
+      signal: undefined,
+    } as any,
+    h.ctx(),
+  );
+  assert.deepEqual(res, {}, "no compaction supplied → pi runs its own");
 });
 
 test("resume auto-inline stages recall into the system prompt", async () => {

package/extensions/mega-config.ts

@@ -34,10 +34,24 @@ export interface MegaConfig {
   fastGatePct: number;
   anchorUserMessages: number;
   preserveRecent: number;
+  /** High-pressure floor for preserveRecent — when context is near the limit
+   *  we compact deeper, but never below this (keeps recent turns for coherence). */
+  preserveRecentMin: number;
   auto: boolean;
   autoInline: boolean;
   autoInlineK: number;
   dedupSim: number;
+  /** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
+   *  the durable-trim summary source (root summary). */
+  raptorEnabled: boolean;
+  /** Token ceiling for the re-injected recall block (Fix C). Recall stops
+   *  adding checkpoints once the block would exceed this — bounds read-path
+   *  token cost so it can never net-inflate the window. */
+  recallMaxTokens: number;
+  /** Inline-dedupe recalled checkpoints against the live window (Fix C): drop
+   *  a hit whose summary is ≥ dedupSim similar to a live message — "dedupe on
+   *  inline/read" so we never re-inject context already resident. */
+  windowDedupe: boolean;
   debug: boolean;
 }
 
@@ -65,6 +79,13 @@ function resolveThreshold(): { tier: CompactTier | "custom"; thresholdTokens: nu
   return { tier, thresholdTokens: COMPACT_TIERS[tier] };
 }
 
+/**
+ * Pressure helpers for adaptive compression (Fix E) live in src/config.ts
+ * (pi-agnostic) so unit tests can import them without the pi runtime. Re-export
+ * here so the extension has one import surface.
+ */
+export { pressureFromPct, preserveRecentForPressure } from "../src/config.js";
+
 /** Build the resolved config from env + defaults. */
 export function loadConfig(): MegaConfig {
   const { tier, thresholdTokens } = resolveThreshold();
@@ -77,10 +98,14 @@ export function loadConfig(): MegaConfig {
     thresholdTokens,
     anchorUserMessages: envFlag("MEGACOMPACT_ANCHOR_USER_MESSAGES", 3),
     preserveRecent: envFlag("MEGACOMPACT_PRESERVE_RECENT", 4),
+    preserveRecentMin: envFlag("MEGACOMPACT_PRESERVE_RECENT_MIN", 2),
     auto: envBool("MEGACOMPACT_AUTO", true),
     autoInline: envBool("MEGACOMPACT_AUTO_INLINE", true),
     autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
     dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
+    raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
+    recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
+    windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
     debug: envBool("MEGACOMPACT_DEBUG", false),
   };
 }

package/extensions/mega-events.ts

@@ -11,10 +11,10 @@ import type { ExtensionAPI, ExtensionContext, ContextEvent, SessionBeforeCompact
 import { normalizeSessionId } from "../src/store.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens } from "../src/tokens.js";
-import { dropCompactedRange } from "../src/adapt.js";
 import { MegaRuntime, recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
 import { runCompact, doRecall } from "./mega-pipeline.js";
-import type { MegaConfig } from "./mega-config.js";
+import { driveNativeCompaction } from "./mega-compact-driver.js";
+import { pressureFromPct, type MegaConfig } from "./mega-config.js";
 
 /** Register all pi lifecycle event handlers. */
 export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, config: MegaConfig): void {
@@ -119,7 +119,15 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
     runtime.snapshot(ctx);
   });
 
-  // ---- Auto-trigger: fast-gate → confirm → Trident+persist → drop --------
+  // ---- Auto-trigger: own the decision, pi owns the durable write ----------
+  // OUR auto-trigger (over threshold + debounce): persist our Trident checkpoint,
+  // then start pi's compaction flow via ctx.compact(). That fires
+  // `session_before_compact`, where OUR handler returns our summary +
+  // firstKeptEntryId, and pi durably writes the trim to disk (appendCompaction).
+  // Result: auto-compact AND a durable trim — resume reloads the trimmed window,
+  // no full-reload + additive recall inflation (Fix B kills the token-growth bug).
+  // We do NOT drop messages here (that would be ephemeral; the read-only session
+  // manager can't trim disk, so the trim has to come through pi).
   pi.on("context", async (event: ContextEvent, ctx: ExtensionContext) => {
     if (!config.auto) return;
     const usage = ctx.getContextUsage();
@@ -133,15 +141,11 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
 
     const messages = event.messages;
     const view = runtime.engineView(messages);
-    // Prefer the runtime's real token estimate; fall back to our heuristic
-    // (and to a percent-of-window proxy when tokens is unknown).
     const currentTokens =
       usage?.tokens ?? estimateSessionTokens(view) ??
       Math.round((pct / 100) * (usage?.contextWindow ?? 0));
 
     // FAST GATE: token-based (tier threshold), not percentage-based.
-    // A 20% gate on a 2M window = 400k, which is way above the 50k low-tier
-    // threshold. Gate on the actual token count instead.
     if (currentTokens < config.thresholdTokens) return;
 
     const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
@@ -152,29 +156,44 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
     if (now < runtime.debounceUntil) return;
     runtime.debounceUntil = now + 2000;
 
-    const ran = runCompact(pi, runtime, config, ctx, messages);
+    // Adaptive compression (Fix E): scale compression strength + keepFrom depth
+    // with how close we are to the model context limit.
+    const pressure = pressureFromPct(pct);
+    const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
     if (ran.skipped) return;
 
-    // DROP the compacted range from the outgoing context, honoring the anchor
-    // floor + tool-pair boundary guards (PREVENT-PI-001/002).
-    const kept = dropCompactedRange(messages, ran.keepFrom!, config.anchorUserMessages);
-    if (kept.length < messages.length) {
-      return { messages: kept };
-    }
+    // Start pi's compaction flow so our session_before_compact handler can
+    // supply the durable trim (pi writes it to disk). We never use pi's summary.
+    ctx.compact({ customInstructions: undefined });
   });
 
-  // ---- Cancel native compaction once we've persisted our own -------------
-  pi.on("session_before_compact", async (_event: SessionBeforeCompactEvent, ctx: ExtensionContext) => {
+  // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
+  // We run the Trident pipeline to produce a compressed summary, then return
+  // it as a CompactionResult. pi writes the summary into a compactionSummary
+  // entry AND truncates the on-disk transcript from firstKeptEntryId. This is
+  // the durable fix for "tokens grow on read": the trim survives resume, so
+  // there is no full-reload + additive recall inflation.
+  pi.on("session_before_compact", async (event: SessionBeforeCompactEvent, ctx: ExtensionContext) => {
     runtime.resetRuntime(ctx.sessionManager.getSessionId());
-    if (runtime.rt.persistedThisSession) {
-      // We already persisted a checkpoint for this session (via the context
-      // hook drop) — cancel pi's own compaction to avoid double-compacting.
-      // Our context-hook drop already trimmed the window.
-      return { cancel: true };
+    if (!config.auto) return {}; // let pi run its own native compaction
+    try {
+      const result = driveNativeCompaction(event, runtime, config);
+      if (result) {
+        runtime.logger.info("native-compact", {
+          sessionId: runtime.rt.sessionId,
+          firstKeptEntryId: result.compaction.firstKeptEntryId,
+          tokensBefore: result.compaction.tokensBefore,
+          summaryTokens: result.compaction.estimatedTokensAfter,
+        });
+        return { compaction: result.compaction };
+      }
+    } catch (err) {
+      runtime.logger.error("native-compact-failed", {
+        sessionId: runtime.rt.sessionId,
+        error: String(err instanceof Error ? err.message : err),
+      });
     }
-    // We haven't persisted yet this session: let pi run its native compaction.
-    // (Our auto-trigger only fires again past the threshold, and will then
-    // capture a checkpoint next time around.)
+    // Fall back to pi's own native compaction if we can't supply one.
     return {};
   });
 }

package/extensions/mega-pipeline.ts

@@ -8,6 +8,7 @@
  */
 
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import { compactSession } from "../src/engine.js";
 import { recallAndInline } from "../src/recall.js";
@@ -18,7 +19,9 @@ import {
   C,
   MARKER_TYPE,
 } from "./mega-runtime.js";
-import { resolveRepoRoot, type MegaConfig } from "./mega-config.js";
+import { resolveRepoRoot, preserveRecentForPressure, type MegaConfig } from "./mega-config.js";
+import { runRaptor } from "../src/dedup/raptor/index.js";
+import { loadDedupConfig } from "../src/config/dedup.js";
 
 export type RunCompactResult =
   | { skipped: true }
@@ -31,7 +34,7 @@ export function runCompact(
   config: MegaConfig,
   ctx: ExtensionContext,
   messages: AgentMessage[],
-  opts: { keepFrom?: number; summary?: string } = {},
+  opts: { keepFrom?: number; summary?: string; compressionPressure?: number } = {},
 ): RunCompactResult {
   runtime.bindRepo(ctx.cwd);
   const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
@@ -39,7 +42,14 @@ export function runCompact(
   runtime.rt.sessionId = sid;
 
   const view = runtime.engineView(messages);
-  const keepFrom = opts.keepFrom ?? Math.max(0, view.length - config.preserveRecent);
+  // keepFrom deepens with context pressure (Fix E): under high pressure we
+  // compact more of the session, down to the preserveRecentMin floor.
+  const preserve = preserveRecentForPressure(
+    opts.compressionPressure ?? 0,
+    config.preserveRecent,
+    config.preserveRecentMin,
+  );
+  const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
   if (keepFrom <= 0) return { skipped: true };
 
   runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
@@ -51,6 +61,7 @@ export function runCompact(
       summary: opts.summary,
       timestamp: Date.now(),
       onTier: runtime.makeTierCallback(ctx),
+      compressionPressure: opts.compressionPressure,
     },
     runtime.store,
   );
@@ -122,6 +133,37 @@ export function runCompact(
     deduped: result.deduped,
   });
 
+  // Fix D: refresh the RAPTOR tree for this session so live recall (search) can
+  // serve high-level summaries. Best-effort + non-fatal: never block compaction.
+  // Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
+  if (config.raptorEnabled && !result.deduped) {
+    try {
+      const dd = loadDedupConfig();
+      const all = runtime.store.list(sid);
+      const leaves = all.map((cp) => ({
+        id: cp.checkpointId,
+        messages: [],
+        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+        embedding: cp.embedding,
+      }));
+      if (leaves.length >= 2) {
+        runRaptor(
+          leaves,
+          {
+            stateDir: runtime.currentStateDir,
+            sessionId: sid,
+            budgetMs: dd.RAPTOR_BUDGET_MS,
+            clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
+            consistencyThreshold: dd.RAPTOR_CONSISTENCY,
+            logger: runtime.logger,
+          },
+        );
+      }
+    } catch {
+      /* non-fatal: tree refresh never blocks a compaction */
+    }
+  }
+
   runtime.setStatus(
     ctx,
     runtime.rt.persistedThisSession
@@ -160,8 +202,22 @@ export function doRecall(
 ) {
   runtime.bindRepo(ctx.cwd);
   const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+  // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
+  // are already resident in the session, so recall never re-injects context the
+  // model can already see. Best-effort — an empty window just skips dedupe.
+  const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
   const result = recallAndInline(
-    { sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true },
+    {
+      sessionId: sid,
+      query,
+      limit: config.autoInlineK,
+      source,
+      skipInjected: true,
+      recallMaxTokens: config.recallMaxTokens,
+      windowDedupe: config.windowDedupe,
+      liveWindow,
+      dedupSim: config.dedupSim,
+    },
     runtime.store,
   );
   runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
@@ -175,3 +231,26 @@ export function doRecall(
   }
   return result;
 }
+
+/**
+ * Extract the live-window message texts from the session manager (Fix C),
+ * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
+ * error so recall falls back to unbounded (still correct, just no dedupe).
+ * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
+ */
+function extractLiveWindow(ctx: ExtensionContext): string[] {
+  try {
+    const entries = ctx.sessionManager.getEntries();
+    const texts: string[] = [];
+    for (const e of entries) {
+      for (const m of sessionEntryToContextMessages(e)) {
+        const c = (m as { content?: unknown }).content;
+        if (typeof c === "string") texts.push(c);
+        else if (Array.isArray(c)) texts.push(c.map((b: any) => b.text).join(" "));
+      }
+    }
+    return texts;
+  } catch {
+    return [];
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.4.21",
+  "version": "0.4.23",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",
@@ -20,7 +20,7 @@
     "trident"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=22.13"
   },
   "files": [
     "dist",
@@ -45,19 +45,18 @@
     "test": "npm run build && node --test \"dist/src/**/*.test.js\" \"dist/extensions/**/*.test.js\"",
     "guardrails": "python3 scripts/regression_check.py --all || node scripts/guardrails-scan.mjs",
     "precommit": "bash .claude/hooks/pre-commit.sh",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "postinstall": "npm rebuild @mongodb-js/zstd || true"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",
     "openclaw": ">=0.1.0"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.13",
-    "@types/node": "^20.0.0",
+    "@types/node": "^22.20.1",
     "typescript": "^5.4.0"
   },
   "dependencies": {
-    "@mongodb-js/zstd": "^7.0.0",
-    "better-sqlite3": "^12.11.1"
+    "@mongodb-js/zstd": "^7.0.0"
   }
 }

package/src/config/dedup.ts

@@ -66,7 +66,10 @@ export function loadDedupConfig(): DedupConfigShape {
     L0_ENABLED: envBool("MEGACOMPACT_L0_ENABLED", true),
     L1_ENABLED: envBool("MEGACOMPACT_L1_ENABLED", true),
     L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
-    RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", false), // shadow by default
+    // Fix D: RAPTOR promoted to live recall. Default ON; canary.ts sequences it
+    // last (L0→L1→L2→RAPTOR) and auto-disables on p95 breach, so promotion is
+    // safe. `RAPTOR_SHADOW_MODE=false` still gates serving during transition.
+    RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
     MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
     MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
     MARK_ONLY_L2: envBool("MEGACOMPACT_MARK_ONLY_L2", false),

package/src/config.ts

@@ -13,3 +13,29 @@ export const STATE_DIR_DEFAULT = join(homedir(), ".pi", "agent", "extensions", "
 
 /** Pi custom message / entry type used as the dedup sentinel. */
 export const MARKER_TYPE = "mega-compact-marker";
+
+/**
+ * Derive context-window pressure (0–1) from a usage percentage. Used to scale
+ * compression strength + keepFrom depth (Fix E): low pct = room to spare,
+ * high pct = near the limit. Deterministic; clamps to [0,1].
+ */
+export function pressureFromPct(pct: number | null | undefined): number {
+  if (pct == null || Number.isNaN(pct)) return 0;
+  return pct < 0 ? 0 : pct > 100 ? 1 : pct / 100;
+}
+
+/**
+ * Map pressure → how many recent messages to preserve verbatim. Under low
+ * pressure we keep `preserveRecent`; under high pressure we compact deeper,
+ * down to `preserveRecentMin`. Never splits a tool pair / anchor floor — the
+ * boundary guard (computeDropRange) enforces that downstream.
+ */
+export function preserveRecentForPressure(
+  pressure: number,
+  preserveRecent: number,
+  preserveRecentMin: number,
+): number {
+  const p = pressure < 0 ? 0 : pressure > 1 ? 1 : pressure;
+  const v = Math.round(preserveRecent - (preserveRecent - preserveRecentMin) * p);
+  return Math.max(preserveRecentMin, Math.min(preserveRecent, v));
+}