pi-mega-compact 0.4.21 → 0.4.24

package/extensions/mega-compact.test.ts

@@ -45,6 +45,7 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
   let statusKey: string | undefined;
   let statusText: string | undefined;
   const notifies: string[] = [];
+  const compactCalls: any[] = [];
 
   // Minimal AgentMessage factory for the session we project into the extension.
   function msg(role: string, text: string, toolName?: string): AgentMessage {
@@ -107,7 +108,30 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
       hasPendingMessages: () => false,
       shutdown: () => {},
       getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
-      compact: () => {},
+      // Faithful mock: ctx.compact() starts pi's flow, which fires the
+      // session_before_compact handler (where WE supply the durable trim).
+      compact: (opts?: any) => {
+        compactCalls.push(opts);
+        if (handlers["session_before_compact"]) {
+          return handlers["session_before_compact"](
+            {
+              type: "session_before_compact",
+              reason: "threshold",
+              willRetry: false,
+              signal: undefined,
+              // pi computed the cut honoring anchor floor + tool-pair (PREVENT-PI-002);
+              // our handler reuses it as firstKeptEntryId.
+              preparation: {
+                firstKeptEntryId: "e2",
+                messagesToSummarize: session.slice(0, 2),
+                tokensBefore: 500,
+              },
+            } as any,
+            makeCtx(),
+          );
+        }
+        return undefined;
+      },
       getSystemPrompt: () => "system base",
       ...over,
     } as any;
@@ -143,14 +167,14 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
   mod.default(pi);
 
   return {
-    stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies,
+    stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies, compactCalls,
     fire: (ev: string, event: any, ctx: any) => handlers[ev](event, ctx),
     ctx: makeCtx,
     session,
   };
 }
 
-test("auto-trigger: past threshold persists a chkpt and drops context", async () => {
+test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
   const h = harness();
   const messages = h.session;
   const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
@@ -159,27 +183,50 @@ test("auto-trigger: past threshold persists a chkpt and drops context", async ()
   const { listCheckpoints } = await import("../src/store/sqlite.js");
   assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
   assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
-  // Context dropped (the compacted range was trimmed).
-  assert.ok(res && Array.isArray(res.messages), "context handler returns filtered messages");
-  assert.ok((res.messages as any[]).length < messages.length, "outgoing context shrank");
+  // The context handler no longer drops messages itself (that was ephemeral —
+  // the read-path token-growth bug). It triggers pi's compaction flow, which
+  // calls our session_before_compact handler to supply the DURABLE trim.
+  assert.equal(res, undefined, "context handler returns nothing (no local drop)");
+  assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim");
+  // The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
+  assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
 });
 
-test("session_before_compact cancels once we've persisted", async () => {
+test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
   const h = harness();
-  const ctx = h.ctx();
-  // First fire the auto-trigger so a checkpoint is persisted this session.
-  await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
-  // Now pi tries to compact natively — we must cancel (no double-compact).
-  const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "overflow", willRetry: true, preparation: {}, signal: undefined } as any, ctx);
-  assert.deepEqual(res, { cancel: true });
+  // pi fires session_before_compact with its own computed preparation.
+  const res = await h.fire(
+    "session_before_compact",
+    {
+      type: "session_before_compact",
+      reason: "overflow",
+      willRetry: true,
+      preparation: { firstKeptEntryId: "e2", messagesToSummarize: h.session.slice(0, 2), tokensBefore: 500 },
+      signal: undefined,
+    } as any,
+    h.ctx(),
+  );
+  assert.ok(res && res.compaction, "returns a compaction result");
+  assert.equal(res.compaction.firstKeptEntryId, "e2", "reuses pi's cut boundary (PREVENT-PI-002 safe)");
+  assert.ok(typeof res.compaction.summary === "string" && res.compaction.summary.length > 0, "our summary supplied");
+  assert.ok(res.compaction.tokensBefore >= 0, "tokensBefore reported");
 });
 
-test("session_before_compact does NOT cancel when nothing persisted", async () => {
+test("session_before_compact falls back to pi when nothing to summarize", async () => {
   const h = harness();
-  const ctx = h.ctx();
-  // Do NOT fire context first; this session has no checkpoint.
-  const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "threshold", willRetry: false, preparation: {}, signal: undefined } as any, ctx);
-  assert.deepEqual(res, {});
+  // Empty preparation → no messages to summarize → return {} so pi compacts natively.
+  const res = await h.fire(
+    "session_before_compact",
+    {
+      type: "session_before_compact",
+      reason: "threshold",
+      willRetry: false,
+      preparation: { firstKeptEntryId: "e0", messagesToSummarize: [], tokensBefore: 0 },
+      signal: undefined,
+    } as any,
+    h.ctx(),
+  );
+  assert.deepEqual(res, {}, "no compaction supplied → pi runs its own");
 });
 
 test("resume auto-inline stages recall into the system prompt", async () => {

package/extensions/mega-config.ts

@@ -34,10 +34,24 @@ export interface MegaConfig {
   fastGatePct: number;
   anchorUserMessages: number;
   preserveRecent: number;
+  /** High-pressure floor for preserveRecent — when context is near the limit
+   *  we compact deeper, but never below this (keeps recent turns for coherence). */
+  preserveRecentMin: number;
   auto: boolean;
   autoInline: boolean;
   autoInlineK: number;
   dedupSim: number;
+  /** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
+   *  the durable-trim summary source (root summary). */
+  raptorEnabled: boolean;
+  /** Token ceiling for the re-injected recall block (Fix C). Recall stops
+   *  adding checkpoints once the block would exceed this — bounds read-path
+   *  token cost so it can never net-inflate the window. */
+  recallMaxTokens: number;
+  /** Inline-dedupe recalled checkpoints against the live window (Fix C): drop
+   *  a hit whose summary is ≥ dedupSim similar to a live message — "dedupe on
+   *  inline/read" so we never re-inject context already resident. */
+  windowDedupe: boolean;
   debug: boolean;
 }
 
@@ -65,6 +79,13 @@ function resolveThreshold(): { tier: CompactTier | "custom"; thresholdTokens: nu
   return { tier, thresholdTokens: COMPACT_TIERS[tier] };
 }
 
+/**
+ * Pressure helpers for adaptive compression (Fix E) live in src/config.ts
+ * (pi-agnostic) so unit tests can import them without the pi runtime. Re-export
+ * here so the extension has one import surface.
+ */
+export { pressureFromPct, preserveRecentForPressure } from "../src/config.js";
+
 /** Build the resolved config from env + defaults. */
 export function loadConfig(): MegaConfig {
   const { tier, thresholdTokens } = resolveThreshold();
@@ -77,10 +98,14 @@ export function loadConfig(): MegaConfig {
     thresholdTokens,
     anchorUserMessages: envFlag("MEGACOMPACT_ANCHOR_USER_MESSAGES", 3),
     preserveRecent: envFlag("MEGACOMPACT_PRESERVE_RECENT", 4),
+    preserveRecentMin: envFlag("MEGACOMPACT_PRESERVE_RECENT_MIN", 2),
     auto: envBool("MEGACOMPACT_AUTO", true),
     autoInline: envBool("MEGACOMPACT_AUTO_INLINE", true),
     autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
     dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
+    raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
+    recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
+    windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
     debug: envBool("MEGACOMPACT_DEBUG", false),
   };
 }

package/extensions/mega-dashboard-cmds.ts

@@ -9,7 +9,7 @@
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { join, dirname, sep } from "node:path";
 import { fileURLToPath } from "node:url";
-import { existsSync, writeFileSync, readFileSync, unlinkSync } from "node:fs";
+import { existsSync, writeFileSync, readFileSync, unlinkSync, openSync, closeSync } from "node:fs";
 import { spawn } from "node:child_process"; // guardrails-allow PREVENT-PI-004: spawns the optional, user-triggered localhost dashboard server only
 import { MegaRuntime } from "./mega-runtime.js";
 
@@ -209,11 +209,32 @@ export function registerDashboardCommands(pi: ExtensionAPI, runtime: MegaRuntime
         return;
       }
 
+      // Clear any stale marker so a fresh bind never collides with a lingering
+      // orphan, and truncate the launch log so the next error report shows only
+      // this attempt's output.
+      try { unlinkSync(portFile); } catch { /* ignore */ }
+      try { writeFileSync(launchLog, ""); } catch { /* ignore */ }
+
       const args = dashboardNeedsStrip ? ["--experimental-strip-types", runnerFile] : [runnerFile];
+      // Redirect the child's stderr to the launch log so that a CRASH BEFORE the
+      // runner's own __fail handler runs (e.g. an ESM module-load / parse error,
+      // or a missing entry) is still captured. With the old `stdio: "ignore"`
+      // these failures were completely silent and the "check logs" message
+      // pointed at an empty file. We open the fd in the parent and pass it to the
+      // child; once spawned we close our copy (the child keeps its own dup).
+      let stderrFd: number;
+      try {
+        stderrFd = openSync(launchLog, "a");
+      } catch {
+        stderrFd = -1; // fall back to ignored stderr
+      }
       const child = spawn(process.execPath, args, {
         detached: true,
-        stdio: "ignore",
+        stdio: ["ignore", "ignore", stderrFd >= 0 ? stderrFd : "ignore"],
       });
+      if (stderrFd >= 0) {
+        try { closeSync(stderrFd); } catch { /* ignore */ }
+      }
       child.unref();
 
       // Poll for a live server (port 9320–9329) instead of relying solely on the

package/extensions/mega-events.ts

@@ -11,10 +11,10 @@ import type { ExtensionAPI, ExtensionContext, ContextEvent, SessionBeforeCompact
 import { normalizeSessionId } from "../src/store.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens } from "../src/tokens.js";
-import { dropCompactedRange } from "../src/adapt.js";
 import { MegaRuntime, recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
 import { runCompact, doRecall } from "./mega-pipeline.js";
-import type { MegaConfig } from "./mega-config.js";
+import { driveNativeCompaction } from "./mega-compact-driver.js";
+import { pressureFromPct, type MegaConfig } from "./mega-config.js";
 
 /** Register all pi lifecycle event handlers. */
 export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, config: MegaConfig): void {
@@ -119,7 +119,15 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
     runtime.snapshot(ctx);
   });
 
-  // ---- Auto-trigger: fast-gate → confirm → Trident+persist → drop --------
+  // ---- Auto-trigger: own the decision, pi owns the durable write ----------
+  // OUR auto-trigger (over threshold + debounce): persist our Trident checkpoint,
+  // then start pi's compaction flow via ctx.compact(). That fires
+  // `session_before_compact`, where OUR handler returns our summary +
+  // firstKeptEntryId, and pi durably writes the trim to disk (appendCompaction).
+  // Result: auto-compact AND a durable trim — resume reloads the trimmed window,
+  // no full-reload + additive recall inflation (Fix B kills the token-growth bug).
+  // We do NOT drop messages here (that would be ephemeral; the read-only session
+  // manager can't trim disk, so the trim has to come through pi).
   pi.on("context", async (event: ContextEvent, ctx: ExtensionContext) => {
     if (!config.auto) return;
     const usage = ctx.getContextUsage();
@@ -133,15 +141,11 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
 
     const messages = event.messages;
     const view = runtime.engineView(messages);
-    // Prefer the runtime's real token estimate; fall back to our heuristic
-    // (and to a percent-of-window proxy when tokens is unknown).
     const currentTokens =
       usage?.tokens ?? estimateSessionTokens(view) ??
       Math.round((pct / 100) * (usage?.contextWindow ?? 0));
 
     // FAST GATE: token-based (tier threshold), not percentage-based.
-    // A 20% gate on a 2M window = 400k, which is way above the 50k low-tier
-    // threshold. Gate on the actual token count instead.
     if (currentTokens < config.thresholdTokens) return;
 
     const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
@@ -152,29 +156,44 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
     if (now < runtime.debounceUntil) return;
     runtime.debounceUntil = now + 2000;
 
-    const ran = runCompact(pi, runtime, config, ctx, messages);
+    // Adaptive compression (Fix E): scale compression strength + keepFrom depth
+    // with how close we are to the model context limit.
+    const pressure = pressureFromPct(pct);
+    const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
     if (ran.skipped) return;
 
-    // DROP the compacted range from the outgoing context, honoring the anchor
-    // floor + tool-pair boundary guards (PREVENT-PI-001/002).
-    const kept = dropCompactedRange(messages, ran.keepFrom!, config.anchorUserMessages);
-    if (kept.length < messages.length) {
-      return { messages: kept };
-    }
+    // Start pi's compaction flow so our session_before_compact handler can
+    // supply the durable trim (pi writes it to disk). We never use pi's summary.
+    ctx.compact({ customInstructions: undefined });
   });
 
-  // ---- Cancel native compaction once we've persisted our own -------------
-  pi.on("session_before_compact", async (_event: SessionBeforeCompactEvent, ctx: ExtensionContext) => {
+  // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
+  // We run the Trident pipeline to produce a compressed summary, then return
+  // it as a CompactionResult. pi writes the summary into a compactionSummary
+  // entry AND truncates the on-disk transcript from firstKeptEntryId. This is
+  // the durable fix for "tokens grow on read": the trim survives resume, so
+  // there is no full-reload + additive recall inflation.
+  pi.on("session_before_compact", async (event: SessionBeforeCompactEvent, ctx: ExtensionContext) => {
     runtime.resetRuntime(ctx.sessionManager.getSessionId());
-    if (runtime.rt.persistedThisSession) {
-      // We already persisted a checkpoint for this session (via the context
-      // hook drop) — cancel pi's own compaction to avoid double-compacting.
-      // Our context-hook drop already trimmed the window.
-      return { cancel: true };
+    if (!config.auto) return {}; // let pi run its own native compaction
+    try {
+      const result = driveNativeCompaction(event, runtime, config);
+      if (result) {
+        runtime.logger.info("native-compact", {
+          sessionId: runtime.rt.sessionId,
+          firstKeptEntryId: result.compaction.firstKeptEntryId,
+          tokensBefore: result.compaction.tokensBefore,
+          summaryTokens: result.compaction.estimatedTokensAfter,
+        });
+        return { compaction: result.compaction };
+      }
+    } catch (err) {
+      runtime.logger.error("native-compact-failed", {
+        sessionId: runtime.rt.sessionId,
+        error: String(err instanceof Error ? err.message : err),
+      });
     }
-    // We haven't persisted yet this session: let pi run its native compaction.
-    // (Our auto-trigger only fires again past the threshold, and will then
-    // capture a checkpoint next time around.)
+    // Fall back to pi's own native compaction if we can't supply one.
     return {};
   });
 }

package/extensions/mega-pipeline.ts

@@ -8,6 +8,7 @@
  */
 
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import { compactSession } from "../src/engine.js";
 import { recallAndInline } from "../src/recall.js";
@@ -18,7 +19,9 @@ import {
   C,
   MARKER_TYPE,
 } from "./mega-runtime.js";
-import { resolveRepoRoot, type MegaConfig } from "./mega-config.js";
+import { resolveRepoRoot, preserveRecentForPressure, type MegaConfig } from "./mega-config.js";
+import { runRaptor } from "../src/dedup/raptor/index.js";
+import { loadDedupConfig } from "../src/config/dedup.js";
 
 export type RunCompactResult =
   | { skipped: true }
@@ -31,7 +34,7 @@ export function runCompact(
   config: MegaConfig,
   ctx: ExtensionContext,
   messages: AgentMessage[],
-  opts: { keepFrom?: number; summary?: string } = {},
+  opts: { keepFrom?: number; summary?: string; compressionPressure?: number } = {},
 ): RunCompactResult {
   runtime.bindRepo(ctx.cwd);
   const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
@@ -39,7 +42,14 @@ export function runCompact(
   runtime.rt.sessionId = sid;
 
   const view = runtime.engineView(messages);
-  const keepFrom = opts.keepFrom ?? Math.max(0, view.length - config.preserveRecent);
+  // keepFrom deepens with context pressure (Fix E): under high pressure we
+  // compact more of the session, down to the preserveRecentMin floor.
+  const preserve = preserveRecentForPressure(
+    opts.compressionPressure ?? 0,
+    config.preserveRecent,
+    config.preserveRecentMin,
+  );
+  const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
   if (keepFrom <= 0) return { skipped: true };
 
   runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
@@ -51,6 +61,7 @@ export function runCompact(
       summary: opts.summary,
       timestamp: Date.now(),
       onTier: runtime.makeTierCallback(ctx),
+      compressionPressure: opts.compressionPressure,
     },
     runtime.store,
   );
@@ -122,6 +133,37 @@ export function runCompact(
     deduped: result.deduped,
   });
 
+  // Fix D: refresh the RAPTOR tree for this session so live recall (search) can
+  // serve high-level summaries. Best-effort + non-fatal: never block compaction.
+  // Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
+  if (config.raptorEnabled && !result.deduped) {
+    try {
+      const dd = loadDedupConfig();
+      const all = runtime.store.list(sid);
+      const leaves = all.map((cp) => ({
+        id: cp.checkpointId,
+        messages: [],
+        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+        embedding: cp.embedding,
+      }));
+      if (leaves.length >= 2) {
+        runRaptor(
+          leaves,
+          {
+            stateDir: runtime.currentStateDir,
+            sessionId: sid,
+            budgetMs: dd.RAPTOR_BUDGET_MS,
+            clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
+            consistencyThreshold: dd.RAPTOR_CONSISTENCY,
+            logger: runtime.logger,
+          },
+        );
+      }
+    } catch {
+      /* non-fatal: tree refresh never blocks a compaction */
+    }
+  }
+
   runtime.setStatus(
     ctx,
     runtime.rt.persistedThisSession
@@ -160,8 +202,22 @@ export function doRecall(
 ) {
   runtime.bindRepo(ctx.cwd);
   const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+  // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
+  // are already resident in the session, so recall never re-injects context the
+  // model can already see. Best-effort — an empty window just skips dedupe.
+  const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
   const result = recallAndInline(
-    { sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true },
+    {
+      sessionId: sid,
+      query,
+      limit: config.autoInlineK,
+      source,
+      skipInjected: true,
+      recallMaxTokens: config.recallMaxTokens,
+      windowDedupe: config.windowDedupe,
+      liveWindow,
+      dedupSim: config.dedupSim,
+    },
     runtime.store,
   );
   runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
@@ -175,3 +231,26 @@ export function doRecall(
   }
   return result;
 }
+
+/**
+ * Extract the live-window message texts from the session manager (Fix C),
+ * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
+ * error so recall falls back to unbounded (still correct, just no dedupe).
+ * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
+ */
+function extractLiveWindow(ctx: ExtensionContext): string[] {
+  try {
+    const entries = ctx.sessionManager.getEntries();
+    const texts: string[] = [];
+    for (const e of entries) {
+      for (const m of sessionEntryToContextMessages(e)) {
+        const c = (m as { content?: unknown }).content;
+        if (typeof c === "string") texts.push(c);
+        else if (Array.isArray(c)) texts.push(c.map((b: any) => b.text).join(" "));
+      }
+    }
+    return texts;
+  } catch {
+    return [];
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.4.21",
+  "version": "0.4.24",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",
@@ -20,7 +20,7 @@
     "trident"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=22.13"
   },
   "files": [
     "dist",
@@ -45,19 +45,18 @@
     "test": "npm run build && node --test \"dist/src/**/*.test.js\" \"dist/extensions/**/*.test.js\"",
     "guardrails": "python3 scripts/regression_check.py --all || node scripts/guardrails-scan.mjs",
     "precommit": "bash .claude/hooks/pre-commit.sh",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "postinstall": "npm rebuild @mongodb-js/zstd || true"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",
     "openclaw": ">=0.1.0"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.13",
-    "@types/node": "^20.0.0",
+    "@types/node": "^22.20.1",
     "typescript": "^5.4.0"
   },
   "dependencies": {
-    "@mongodb-js/zstd": "^7.0.0",
-    "better-sqlite3": "^12.11.1"
+    "@mongodb-js/zstd": "^7.0.0"
   }
 }

package/src/config/dedup.ts

@@ -66,7 +66,10 @@ export function loadDedupConfig(): DedupConfigShape {
     L0_ENABLED: envBool("MEGACOMPACT_L0_ENABLED", true),
     L1_ENABLED: envBool("MEGACOMPACT_L1_ENABLED", true),
     L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
-    RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", false), // shadow by default
+    // Fix D: RAPTOR promoted to live recall. Default ON; canary.ts sequences it
+    // last (L0→L1→L2→RAPTOR) and auto-disables on p95 breach, so promotion is
+    // safe. `RAPTOR_SHADOW_MODE=false` still gates serving during transition.
+    RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
     MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
     MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
     MARK_ONLY_L2: envBool("MEGACOMPACT_MARK_ONLY_L2", false),

package/src/config.ts

@@ -13,3 +13,29 @@ export const STATE_DIR_DEFAULT = join(homedir(), ".pi", "agent", "extensions", "
 
 /** Pi custom message / entry type used as the dedup sentinel. */
 export const MARKER_TYPE = "mega-compact-marker";
+
+/**
+ * Derive context-window pressure (0–1) from a usage percentage. Used to scale
+ * compression strength + keepFrom depth (Fix E): low pct = room to spare,
+ * high pct = near the limit. Deterministic; clamps to [0,1].
+ */
+export function pressureFromPct(pct: number | null | undefined): number {
+  if (pct == null || Number.isNaN(pct)) return 0;
+  return pct < 0 ? 0 : pct > 100 ? 1 : pct / 100;
+}
+
+/**
+ * Map pressure → how many recent messages to preserve verbatim. Under low
+ * pressure we keep `preserveRecent`; under high pressure we compact deeper,
+ * down to `preserveRecentMin`. Never splits a tool pair / anchor floor — the
+ * boundary guard (computeDropRange) enforces that downstream.
+ */
+export function preserveRecentForPressure(
+  pressure: number,
+  preserveRecent: number,
+  preserveRecentMin: number,
+): number {
+  const p = pressure < 0 ? 0 : pressure > 1 ? 1 : pressure;
+  const v = Math.round(preserveRecent - (preserveRecent - preserveRecentMin) * p);
+  return Math.max(preserveRecentMin, Math.min(preserveRecent, v));
+}

package/src/dedup/raptor/index.ts

@@ -89,10 +89,22 @@ export function recallRaptor(
   opts: { embedder?: Embedder; stateDir: string; k?: number; topM?: number },
 ): string[] {
   const embedder = opts.embedder ?? defaultEmbedder();
-  const nodes = listRaptorNodes(sessionId, opts.stateDir);
-  if (nodes.length === 0) return [];
-  // Rehydrate a minimal in-memory tree (parent links reconstructed from children).
-  const byId = new Map(nodes.map((n) => [n.id, n]));
+  const tree = rehydrateRaptorTree(sessionId, opts.stateDir);
+  if (!tree) return [];
+  return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+}
+
+/**
+ * Rehydrate a persisted RAPTOR tree from raptor_nodes (Fix D): rebuild the
+ * in-memory RaptorTree + parent links so vectorStore.search can serve it live.
+ * Returns null when no tree exists (caller falls back to the flat path).
+ */
+export function rehydrateRaptorTree(
+  sessionId: string,
+  stateDir: string,
+): RaptorTree | null {
+  const nodes = listRaptorNodes(sessionId, stateDir);
+  if (nodes.length === 0) return null;
   const tree: RaptorTree = {
     nodes: new Map(
       nodes.map((n) => [
@@ -109,10 +121,33 @@ export function recallRaptor(
         },
       ]),
     ),
-    rootId: nodes.reduce<typeof nodes[number] | null>((best, n) => (!best || n.level > (best?.level ?? -1) ? n : best), null)?.id ?? null,
+    rootId:
+      nodes.reduce<typeof nodes[number] | null>(
+        (best, n) => (!best || n.level > (best?.level ?? -1) ? n : best),
+        null,
+      )?.id ?? null,
     levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
     timedOut: false,
   };
-  void byId;
-  return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+  return tree;
+}
+
+/**
+ * Return the RAPTOR root summary for a session, if a tree has been built.
+ * Used by the durable-trim driver (Fix B/D) to supply pi a session-level
+ * compressed summary instead of one slice's extractive summary. Returns
+ * undefined when no tree exists yet (caller falls back to the slice summary).
+ */
+export function recallRaptorRootSummary(
+  sessionId: string,
+  stateDir: string,
+): string | undefined {
+  const nodes = listRaptorNodes(sessionId, stateDir);
+  if (nodes.length === 0) return undefined;
+  // Highest-level node = the root (covers all leaves).
+  const root = nodes.reduce<(typeof nodes)[number] | null>(
+    (best, n) => (!best || n.level > best.level ? n : best),
+    null,
+  );
+  return root?.summary || undefined;
 }