pi-mega-compact 0.4.21 → 0.4.23

package/dist/extensions/dashboard-server.js

@@ -15,7 +15,7 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync }
 import { homedir } from "node:os";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
-import Database from "better-sqlite3";
+import { DatabaseSync } from "node:sqlite";
 // --- Multi-repo index (Phase 5b) ------------------------------------------------
 // The extension writes a machine-wide repo registry into a single SQLite DB
 // (<indexDir>/index.sqlite) as the concurrency-safe write path; the dashboard
@@ -42,8 +42,8 @@ function readIndex() {
     let db;
     try {
         // Read-only + immutable WAL so a concurrent writer's WAL never blocks us.
-        db = new Database(indexPath, { readonly: true, fileMustExist: true });
-        db.pragma("journal_mode = WAL");
+        db = new DatabaseSync(indexPath, { readOnly: true });
+        db.exec("PRAGMA journal_mode = WAL");
         const rows = db
             .prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC")
             .all();

package/dist/extensions/mega-compact-driver.js

@@ -0,0 +1,79 @@
+/**
+ * mega-compact-driver.ts — the durable-trim driver (Fix B).
+ *
+ * The read-path token-growth bug: the old design cancelled pi's native
+ * compaction (`{ cancel: true }`) and did its own ephemeral `context`-hook
+ * drop. That drop only affected the outgoing request — the on-disk transcript
+ * was never trimmed (the session manager is read-only for extensions). So on
+ * resume pi reloaded the FULL transcript and we ADDED a recall block on top →
+ * more tokens than before compaction.
+ *
+ * The fix: on `session_before_compact` we RUN the Trident pipeline to produce a
+ * genuinely compressed summary, then RETURN it as a `CompactionResult`. pi
+ * durably writes our summary into a `compactionSummary` entry AND truncates the
+ * transcript from `firstKeptEntryId`. After that, resume reloads the already-
+ * trimmed transcript (summary baked in) — no additive re-injection, no token
+ * growth.
+ *
+ * We reuse pi's `preparation.firstKeptEntryId` (pi already computed the cut
+ * honoring the anchor-floor + tool-pair guards — PREVENT-PI-002) rather than
+ * recomputing it, so we cannot hand pi a boundary that splits a tool pair.
+ */
+import { compactSession } from "../src/engine.js";
+import { toEngineMessages } from "../src/adapt.js";
+import { estimateBlockTokens, estimateSessionTokens } from "../src/tokens.js";
+import { recallRaptorRootSummary } from "../src/dedup/raptor/index.js";
+/**
+ * Build our durable compaction result from pi's pre-computed preparation.
+ *
+ * Returns undefined when there is nothing to summarize (pi will then run its
+ * own native compaction, or skip). Never throws for "empty" — best-effort.
+ */
+export function driveNativeCompaction(event, runtime, config) {
+    const prep = event.preparation;
+    if (!prep)
+        return undefined;
+    const sid = runtime.rt.sessionId;
+    const messagesToSummarize = prep.messagesToSummarize ?? [];
+    if (messagesToSummarize.length === 0)
+        return undefined;
+    const engineView = toEngineMessages(messagesToSummarize);
+    // We don't drop anything here — pi keeps from prep.firstKeptEntryId. We only
+    // summarize the region pi is about to discard.
+    const keepFrom = engineView.length;
+    const result = compactSession({
+        sessionId: sid,
+        messages: engineView,
+        keepFrom,
+        timestamp: Date.now(),
+        useExtractiveSummary: true,
+    }, runtime.store);
+    if (result.skipped)
+        return undefined;
+    // Prefer the RAPTOR root summary when the tree is built + enabled (Fix D):
+    // it is a session-level compressed summary, broader than one slice's. Fall
+    // back to the extractive topicSummary of this slice.
+    let summary = result.summary;
+    if (config.raptorEnabled) {
+        const root = recallRaptorRootSummary(sid, runtime.currentStateDir);
+        if (root)
+            summary = root;
+    }
+    const tokensBefore = prep.tokensBefore ?? estimateSessionTokens(engineView);
+    const summaryTokens = estimateBlockTokens(summary);
+    // pi keeps the tail from firstKeptEntryId; our summary replaces the discarded
+    // region. Honest saved = discarded-region tokens − our summary tokens.
+    const savedTokens = Math.max(0, tokensBefore - summaryTokens);
+    runtime.rt.lastCompactedFrom = keepFrom;
+    runtime.rt.lastCompactedTokens = tokensBefore;
+    runtime.rt.tokensSaved += savedTokens;
+    runtime.rt.persistedThisSession = true;
+    return {
+        compaction: {
+            summary,
+            firstKeptEntryId: prep.firstKeptEntryId,
+            tokensBefore,
+            estimatedTokensAfter: summaryTokens,
+        },
+    };
+}

package/dist/extensions/mega-compact.test.js

@@ -42,6 +42,7 @@ function harness(opts = {}) {
     let statusKey;
     let statusText;
     const notifies = [];
+    const compactCalls = [];
     // Minimal AgentMessage factory for the session we project into the extension.
     function msg(role, text, toolName) {
         if (role === "assistant" && toolName) {
@@ -100,7 +101,27 @@ function harness(opts = {}) {
             hasPendingMessages: () => false,
             shutdown: () => { },
             getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
-            compact: () => { },
+            // Faithful mock: ctx.compact() starts pi's flow, which fires the
+            // session_before_compact handler (where WE supply the durable trim).
+            compact: (opts) => {
+                compactCalls.push(opts);
+                if (handlers["session_before_compact"]) {
+                    return handlers["session_before_compact"]({
+                        type: "session_before_compact",
+                        reason: "threshold",
+                        willRetry: false,
+                        signal: undefined,
+                        // pi computed the cut honoring anchor floor + tool-pair (PREVENT-PI-002);
+                        // our handler reuses it as firstKeptEntryId.
+                        preparation: {
+                            firstKeptEntryId: "e2",
+                            messagesToSummarize: session.slice(0, 2),
+                            tokensBefore: 500,
+                        },
+                    }, makeCtx());
+                }
+                return undefined;
+            },
             getSystemPrompt: () => "system base",
             ...over,
         };
@@ -133,13 +154,13 @@ function harness(opts = {}) {
     const mod = require("./mega-compact.js");
     mod.default(pi);
     return {
-        stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies,
+        stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies, compactCalls,
         fire: (ev, event, ctx) => handlers[ev](event, ctx),
         ctx: makeCtx,
         session,
     };
 }
-test("auto-trigger: past threshold persists a chkpt and drops context", async () => {
+test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
     const h = harness();
     const messages = h.session;
     const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
@@ -148,25 +169,40 @@ test("auto-trigger: past threshold persists a chkpt and drops context", async ()
     const { listCheckpoints } = await import("../src/store/sqlite.js");
     assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
     assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
-    // Context dropped (the compacted range was trimmed).
-    assert.ok(res && Array.isArray(res.messages), "context handler returns filtered messages");
-    assert.ok(res.messages.length < messages.length, "outgoing context shrank");
+    // The context handler no longer drops messages itself (that was ephemeral —
+    // the read-path token-growth bug). It triggers pi's compaction flow, which
+    // calls our session_before_compact handler to supply the DURABLE trim.
+    assert.equal(res, undefined, "context handler returns nothing (no local drop)");
+    assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim");
+    // The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
+    assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
 });
-test("session_before_compact cancels once we've persisted", async () => {
+test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
     const h = harness();
-    const ctx = h.ctx();
-    // First fire the auto-trigger so a checkpoint is persisted this session.
-    await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
-    // Now pi tries to compact natively — we must cancel (no double-compact).
-    const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "overflow", willRetry: true, preparation: {}, signal: undefined }, ctx);
-    assert.deepEqual(res, { cancel: true });
+    // pi fires session_before_compact with its own computed preparation.
+    const res = await h.fire("session_before_compact", {
+        type: "session_before_compact",
+        reason: "overflow",
+        willRetry: true,
+        preparation: { firstKeptEntryId: "e2", messagesToSummarize: h.session.slice(0, 2), tokensBefore: 500 },
+        signal: undefined,
+    }, h.ctx());
+    assert.ok(res && res.compaction, "returns a compaction result");
+    assert.equal(res.compaction.firstKeptEntryId, "e2", "reuses pi's cut boundary (PREVENT-PI-002 safe)");
+    assert.ok(typeof res.compaction.summary === "string" && res.compaction.summary.length > 0, "our summary supplied");
+    assert.ok(res.compaction.tokensBefore >= 0, "tokensBefore reported");
 });
-test("session_before_compact does NOT cancel when nothing persisted", async () => {
+test("session_before_compact falls back to pi when nothing to summarize", async () => {
     const h = harness();
-    const ctx = h.ctx();
-    // Do NOT fire context first; this session has no checkpoint.
-    const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "threshold", willRetry: false, preparation: {}, signal: undefined }, ctx);
-    assert.deepEqual(res, {});
+    // Empty preparation → no messages to summarize → return {} so pi compacts natively.
+    const res = await h.fire("session_before_compact", {
+        type: "session_before_compact",
+        reason: "threshold",
+        willRetry: false,
+        preparation: { firstKeptEntryId: "e0", messagesToSummarize: [], tokensBefore: 0 },
+        signal: undefined,
+    }, h.ctx());
+    assert.deepEqual(res, {}, "no compaction supplied → pi runs its own");
 });
 test("resume auto-inline stages recall into the system prompt", async () => {
     const h = harness();

package/dist/extensions/mega-config.js

@@ -46,6 +46,12 @@ function resolveThreshold() {
     const tier = (raw in COMPACT_TIERS ? raw : "low");
     return { tier, thresholdTokens: COMPACT_TIERS[tier] };
 }
+/**
+ * Pressure helpers for adaptive compression (Fix E) live in src/config.ts
+ * (pi-agnostic) so unit tests can import them without the pi runtime. Re-export
+ * here so the extension has one import surface.
+ */
+export { pressureFromPct, preserveRecentForPressure } from "../src/config.js";
 /** Build the resolved config from env + defaults. */
 export function loadConfig() {
     const { tier, thresholdTokens } = resolveThreshold();
@@ -58,10 +64,14 @@ export function loadConfig() {
         thresholdTokens,
         anchorUserMessages: envFlag("MEGACOMPACT_ANCHOR_USER_MESSAGES", 3),
         preserveRecent: envFlag("MEGACOMPACT_PRESERVE_RECENT", 4),
+        preserveRecentMin: envFlag("MEGACOMPACT_PRESERVE_RECENT_MIN", 2),
         auto: envBool("MEGACOMPACT_AUTO", true),
         autoInline: envBool("MEGACOMPACT_AUTO_INLINE", true),
         autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
         dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
+        raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
+        recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
+        windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
         debug: envBool("MEGACOMPACT_DEBUG", false),
     };
 }

package/dist/extensions/mega-events.js

@@ -9,9 +9,10 @@
 import { normalizeSessionId } from "../src/store.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens } from "../src/tokens.js";
-import { dropCompactedRange } from "../src/adapt.js";
 import { recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
 import { runCompact, doRecall } from "./mega-pipeline.js";
+import { driveNativeCompaction } from "./mega-compact-driver.js";
+import { pressureFromPct } from "./mega-config.js";
 /** Register all pi lifecycle event handlers. */
 export function registerEventHandlers(pi, runtime, config) {
     // ---- Session lifecycle (state reset points) -------------------------------
@@ -108,7 +109,15 @@ export function registerEventHandlers(pi, runtime, config) {
         runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
         runtime.snapshot(ctx);
     });
-    // ---- Auto-trigger: fast-gate → confirm → Trident+persist → drop --------
+    // ---- Auto-trigger: own the decision, pi owns the durable write ----------
+    // OUR auto-trigger (over threshold + debounce): persist our Trident checkpoint,
+    // then start pi's compaction flow via ctx.compact(). That fires
+    // `session_before_compact`, where OUR handler returns our summary +
+    // firstKeptEntryId, and pi durably writes the trim to disk (appendCompaction).
+    // Result: auto-compact AND a durable trim — resume reloads the trimmed window,
+    // no full-reload + additive recall inflation (Fix B kills the token-growth bug).
+    // We do NOT drop messages here (that would be ephemeral; the read-only session
+    // manager can't trim disk, so the trim has to come through pi).
     pi.on("context", async (event, ctx) => {
         if (!config.auto)
             return;
@@ -123,13 +132,9 @@ export function registerEventHandlers(pi, runtime, config) {
             return;
         const messages = event.messages;
         const view = runtime.engineView(messages);
-        // Prefer the runtime's real token estimate; fall back to our heuristic
-        // (and to a percent-of-window proxy when tokens is unknown).
         const currentTokens = usage?.tokens ?? estimateSessionTokens(view) ??
             Math.round((pct / 100) * (usage?.contextWindow ?? 0));
         // FAST GATE: token-based (tier threshold), not percentage-based.
-        // A 20% gate on a 2M window = 400k, which is way above the 50k low-tier
-        // threshold. Gate on the actual token count instead.
         if (currentTokens < config.thresholdTokens)
             return;
         const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
@@ -140,28 +145,45 @@ export function registerEventHandlers(pi, runtime, config) {
         if (now < runtime.debounceUntil)
             return;
         runtime.debounceUntil = now + 2000;
-        const ran = runCompact(pi, runtime, config, ctx, messages);
+        // Adaptive compression (Fix E): scale compression strength + keepFrom depth
+        // with how close we are to the model context limit.
+        const pressure = pressureFromPct(pct);
+        const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
         if (ran.skipped)
             return;
-        // DROP the compacted range from the outgoing context, honoring the anchor
-        // floor + tool-pair boundary guards (PREVENT-PI-001/002).
-        const kept = dropCompactedRange(messages, ran.keepFrom, config.anchorUserMessages);
-        if (kept.length < messages.length) {
-            return { messages: kept };
-        }
+        // Start pi's compaction flow so our session_before_compact handler can
+        // supply the durable trim (pi writes it to disk). We never use pi's summary.
+        ctx.compact({ customInstructions: undefined });
     });
-    // ---- Cancel native compaction once we've persisted our own -------------
-    pi.on("session_before_compact", async (_event, ctx) => {
+    // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
+    // We run the Trident pipeline to produce a compressed summary, then return
+    // it as a CompactionResult. pi writes the summary into a compactionSummary
+    // entry AND truncates the on-disk transcript from firstKeptEntryId. This is
+    // the durable fix for "tokens grow on read": the trim survives resume, so
+    // there is no full-reload + additive recall inflation.
+    pi.on("session_before_compact", async (event, ctx) => {
         runtime.resetRuntime(ctx.sessionManager.getSessionId());
-        if (runtime.rt.persistedThisSession) {
-            // We already persisted a checkpoint for this session (via the context
-            // hook drop) — cancel pi's own compaction to avoid double-compacting.
-            // Our context-hook drop already trimmed the window.
-            return { cancel: true };
+        if (!config.auto)
+            return {}; // let pi run its own native compaction
+        try {
+            const result = driveNativeCompaction(event, runtime, config);
+            if (result) {
+                runtime.logger.info("native-compact", {
+                    sessionId: runtime.rt.sessionId,
+                    firstKeptEntryId: result.compaction.firstKeptEntryId,
+                    tokensBefore: result.compaction.tokensBefore,
+                    summaryTokens: result.compaction.estimatedTokensAfter,
+                });
+                return { compaction: result.compaction };
+            }
+        }
+        catch (err) {
+            runtime.logger.error("native-compact-failed", {
+                sessionId: runtime.rt.sessionId,
+                error: String(err instanceof Error ? err.message : err),
+            });
         }
-        // We haven't persisted yet this session: let pi run its native compaction.
-        // (Our auto-trigger only fires again past the threshold, and will then
-        // capture a checkpoint next time around.)
+        // Fall back to pi's own native compaction if we can't supply one.
         return {};
     });
 }

package/dist/extensions/mega-pipeline.js

@@ -6,12 +6,15 @@
  * the shared MegaRuntime (token accounting, ticker, status, events) and are
  * driven by the event + command handlers in mega-events.ts / mega-commands.ts.
  */
+import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
 import { compactSession } from "../src/engine.js";
 import { recallAndInline } from "../src/recall.js";
 import { normalizeSessionId } from "../src/store.js";
 import { touchSession, logDaily } from "../src/store/sqlite.js";
 import { C, MARKER_TYPE, } from "./mega-runtime.js";
-import { resolveRepoRoot } from "./mega-config.js";
+import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
+import { runRaptor } from "../src/dedup/raptor/index.js";
+import { loadDedupConfig } from "../src/config/dedup.js";
 /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
 export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
     runtime.bindRepo(ctx.cwd);
@@ -19,7 +22,10 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
     runtime.resetRuntime(sid);
     runtime.rt.sessionId = sid;
     const view = runtime.engineView(messages);
-    const keepFrom = opts.keepFrom ?? Math.max(0, view.length - config.preserveRecent);
+    // keepFrom deepens with context pressure (Fix E): under high pressure we
+    // compact more of the session, down to the preserveRecentMin floor.
+    const preserve = preserveRecentForPressure(opts.compressionPressure ?? 0, config.preserveRecent, config.preserveRecentMin);
+    const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
     if (keepFrom <= 0)
         return { skipped: true };
     runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
@@ -30,6 +36,7 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
         summary: opts.summary,
         timestamp: Date.now(),
         onTier: runtime.makeTierCallback(ctx),
+        compressionPressure: opts.compressionPressure,
     }, runtime.store);
     runtime.pulsing = false;
     if (result.skipped)
@@ -96,6 +103,34 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
         tokenEstimate: result.tokenEstimate,
         deduped: result.deduped,
     });
+    // Fix D: refresh the RAPTOR tree for this session so live recall (search) can
+    // serve high-level summaries. Best-effort + non-fatal: never block compaction.
+    // Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
+    if (config.raptorEnabled && !result.deduped) {
+        try {
+            const dd = loadDedupConfig();
+            const all = runtime.store.list(sid);
+            const leaves = all.map((cp) => ({
+                id: cp.checkpointId,
+                messages: [],
+                sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+                embedding: cp.embedding,
+            }));
+            if (leaves.length >= 2) {
+                runRaptor(leaves, {
+                    stateDir: runtime.currentStateDir,
+                    sessionId: sid,
+                    budgetMs: dd.RAPTOR_BUDGET_MS,
+                    clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
+                    consistencyThreshold: dd.RAPTOR_CONSISTENCY,
+                    logger: runtime.logger,
+                });
+            }
+        }
+        catch {
+            /* non-fatal: tree refresh never blocks a compaction */
+        }
+    }
     runtime.setStatus(ctx, runtime.rt.persistedThisSession
         ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
         : `mega-compact: ready`);
@@ -124,7 +159,21 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
 export function doRecall(runtime, config, ctx, query, source) {
     runtime.bindRepo(ctx.cwd);
     const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-    const result = recallAndInline({ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true }, runtime.store);
+    // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
+    // are already resident in the session, so recall never re-injects context the
+    // model can already see. Best-effort — an empty window just skips dedupe.
+    const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
+    const result = recallAndInline({
+        sessionId: sid,
+        query,
+        limit: config.autoInlineK,
+        source,
+        skipInjected: true,
+        recallMaxTokens: config.recallMaxTokens,
+        windowDedupe: config.windowDedupe,
+        liveWindow,
+        dedupSim: config.dedupSim,
+    }, runtime.store);
     runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
     if (!result.empty && result.toInject.length > 0) {
         const top = result.toInject[0];
@@ -136,3 +185,28 @@ export function doRecall(runtime, config, ctx, query, source) {
     }
     return result;
 }
+/**
+ * Extract the live-window message texts from the session manager (Fix C),
+ * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
+ * error so recall falls back to unbounded (still correct, just no dedupe).
+ * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
+ */
+function extractLiveWindow(ctx) {
+    try {
+        const entries = ctx.sessionManager.getEntries();
+        const texts = [];
+        for (const e of entries) {
+            for (const m of sessionEntryToContextMessages(e)) {
+                const c = m.content;
+                if (typeof c === "string")
+                    texts.push(c);
+                else if (Array.isArray(c))
+                    texts.push(c.map((b) => b.text).join(" "));
+            }
+        }
+        return texts;
+    }
+    catch {
+        return [];
+    }
+}

package/dist/src/config/dedup.js

@@ -33,7 +33,10 @@ export function loadDedupConfig() {
         L0_ENABLED: envBool("MEGACOMPACT_L0_ENABLED", true),
         L1_ENABLED: envBool("MEGACOMPACT_L1_ENABLED", true),
         L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
-        RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", false), // shadow by default
+        // Fix D: RAPTOR promoted to live recall. Default ON; canary.ts sequences it
+        // last (L0→L1→L2→RAPTOR) and auto-disables on p95 breach, so promotion is
+        // safe. `RAPTOR_SHADOW_MODE=false` still gates serving during transition.
+        RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
         MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
         MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
         MARK_ONLY_L2: envBool("MEGACOMPACT_MARK_ONLY_L2", false),

package/dist/src/config.js

@@ -10,3 +10,24 @@ import { homedir } from "node:os";
 export const STATE_DIR_DEFAULT = join(homedir(), ".pi", "agent", "extensions", "pi-mega-compact");
 /** Pi custom message / entry type used as the dedup sentinel. */
 export const MARKER_TYPE = "mega-compact-marker";
+/**
+ * Derive context-window pressure (0–1) from a usage percentage. Used to scale
+ * compression strength + keepFrom depth (Fix E): low pct = room to spare,
+ * high pct = near the limit. Deterministic; clamps to [0,1].
+ */
+export function pressureFromPct(pct) {
+    if (pct == null || Number.isNaN(pct))
+        return 0;
+    return pct < 0 ? 0 : pct > 100 ? 1 : pct / 100;
+}
+/**
+ * Map pressure → how many recent messages to preserve verbatim. Under low
+ * pressure we keep `preserveRecent`; under high pressure we compact deeper,
+ * down to `preserveRecentMin`. Never splits a tool pair / anchor floor — the
+ * boundary guard (computeDropRange) enforces that downstream.
+ */
+export function preserveRecentForPressure(pressure, preserveRecent, preserveRecentMin) {
+    const p = pressure < 0 ? 0 : pressure > 1 ? 1 : pressure;
+    const v = Math.round(preserveRecent - (preserveRecent - preserveRecentMin) * p);
+    return Math.max(preserveRecentMin, Math.min(preserveRecent, v));
+}

package/dist/src/dedup/raptor/index.js

@@ -66,11 +66,20 @@ export function runRaptor(leaves, opts) {
  */
 export function recallRaptor(query, sessionId, opts) {
     const embedder = opts.embedder ?? defaultEmbedder();
-    const nodes = listRaptorNodes(sessionId, opts.stateDir);
-    if (nodes.length === 0)
+    const tree = rehydrateRaptorTree(sessionId, opts.stateDir);
+    if (!tree)
         return [];
-    // Rehydrate a minimal in-memory tree (parent links reconstructed from children).
-    const byId = new Map(nodes.map((n) => [n.id, n]));
+    return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+}
+/**
+ * Rehydrate a persisted RAPTOR tree from raptor_nodes (Fix D): rebuild the
+ * in-memory RaptorTree + parent links so vectorStore.search can serve it live.
+ * Returns null when no tree exists (caller falls back to the flat path).
+ */
+export function rehydrateRaptorTree(sessionId, stateDir) {
+    const nodes = listRaptorNodes(sessionId, stateDir);
+    if (nodes.length === 0)
+        return null;
     const tree = {
         nodes: new Map(nodes.map((n) => [
             n.id,
@@ -89,6 +98,19 @@ export function recallRaptor(query, sessionId, opts) {
         levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
         timedOut: false,
     };
-    void byId;
-    return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+    return tree;
+}
+/**
+ * Return the RAPTOR root summary for a session, if a tree has been built.
+ * Used by the durable-trim driver (Fix B/D) to supply pi a session-level
+ * compressed summary instead of one slice's extractive summary. Returns
+ * undefined when no tree exists yet (caller falls back to the slice summary).
+ */
+export function recallRaptorRootSummary(sessionId, stateDir) {
+    const nodes = listRaptorNodes(sessionId, stateDir);
+    if (nodes.length === 0)
+        return undefined;
+    // Highest-level node = the root (covers all leaves).
+    const root = nodes.reduce((best, n) => (!best || n.level > best.level ? n : best), null);
+    return root?.summary || undefined;
 }

package/dist/src/dedup/raptor/promote.test.js

@@ -0,0 +1,69 @@
+/**
+ * promote.test.ts — Fix D: RAPTOR tree served by vectorStore.search.
+ *
+ * Asserts that, when a RAPTOR tree has been built + persisted for a session,
+ * VectorStore.search returns the tree's staged-expansion hits (broader, O(log n)
+ * coverage) merged with the flat hits — so the dormant tree becomes the live
+ * recall surface. No network: default extractive summarizer + trigram embedder.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../../vectorStore.js";
+import { runRaptor } from "./index.js";
+import { compactSession } from "../../engine.js";
+import { Logger } from "../../log.js";
+import { loadDedupConfig } from "../../config/dedup.js";
+import { listRaptorNodes } from "../../store/sqlite.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-promote-"));
+let counter = 0;
+function raptorConfig() {
+    return { ...loadDedupConfig(), RAPTOR_ENABLED: true };
+}
+function msg(text, toolName) {
+    return toolName ? { role: "assistant", text, toolName, input: text, output: text } : { role: "user", text };
+}
+const SESS = "sess_promote";
+test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)", () => {
+    const stateDir = join(baseTmp, `run-${counter++}`);
+    const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
+    // Persist several distinct checkpoints.
+    for (let i = 1; i <= 5; i++) {
+        compactSession({ sessionId: SESS, messages: [msg(`topic alpha wire ${i} and bootstrap sequence`), msg(`ok ${i}`, "Edit")], keepFrom: 2, timestamp: i }, s);
+    }
+    // No tree yet → flat search only, returns hits, no RAPTOR coverage.
+    assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree initially");
+    const flat = s.search(SESS, "alpha wire bootstrap", 3);
+    assert.ok(flat.length > 0, "flat search returns hits");
+    // Build + persist a RAPTOR tree for the session (mirrors runCompact refresh).
+    const all = s.list(SESS);
+    const leaves = all.map((cp) => ({
+        id: cp.checkpointId,
+        messages: [],
+        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+        embedding: cp.embedding,
+    }));
+    const tree = runRaptor(leaves, { stateDir, sessionId: SESS, logger: new Logger() });
+    assert.ok(tree && listRaptorNodes(SESS, stateDir).length > 0, "tree persisted");
+    // With the tree live + RAPTOR_ENABLED, search still returns hits and now
+    // exercises the RAPTOR-served path without regression.
+    const withTree = s.search(SESS, "alpha wire bootstrap", 3);
+    assert.ok(withTree.length > 0, "search returns hits with RAPTOR promoted");
+    // Every returned hit is a real checkpoint in the session.
+    for (const h of withTree) {
+        assert.ok(all.some((cp) => cp.checkpointId === h.checkpoint.checkpointId), "hit is a real checkpoint");
+    }
+});
+test("Fix D: search still works for a session with <2 leaves (no tree)", () => {
+    const stateDir = join(baseTmp, `run-${counter++}`);
+    const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
+    compactSession({ sessionId: SESS, messages: [msg("only one topic here"), msg("ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    const r = s.search(SESS, "only one topic", 3);
+    assert.ok(r.length > 0, "single-checkpoint search still works (no tree)");
+    assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree built for <2 leaves");
+});
+test("cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});

package/dist/src/engine.js

@@ -104,6 +104,7 @@ export function compactSession(input, store = getDefaultStore()) {
         originalTokenEstimate,
         timestamp: input.timestamp ?? 0,
         onTier: input.onTier,
+        compressionPressure: input.compressionPressure,
     });
     return {
         skipped: false,