pi-mega-compact 0.4.21 → 0.4.24

package/dist/extensions/mega-pipeline.js

@@ -6,12 +6,15 @@
  * the shared MegaRuntime (token accounting, ticker, status, events) and are
  * driven by the event + command handlers in mega-events.ts / mega-commands.ts.
  */
+import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
 import { compactSession } from "../src/engine.js";
 import { recallAndInline } from "../src/recall.js";
 import { normalizeSessionId } from "../src/store.js";
 import { touchSession, logDaily } from "../src/store/sqlite.js";
 import { C, MARKER_TYPE, } from "./mega-runtime.js";
-import { resolveRepoRoot } from "./mega-config.js";
+import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
+import { runRaptor } from "../src/dedup/raptor/index.js";
+import { loadDedupConfig } from "../src/config/dedup.js";
 /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
 export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
     runtime.bindRepo(ctx.cwd);
@@ -19,7 +22,10 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
     runtime.resetRuntime(sid);
     runtime.rt.sessionId = sid;
     const view = runtime.engineView(messages);
-    const keepFrom = opts.keepFrom ?? Math.max(0, view.length - config.preserveRecent);
+    // keepFrom deepens with context pressure (Fix E): under high pressure we
+    // compact more of the session, down to the preserveRecentMin floor.
+    const preserve = preserveRecentForPressure(opts.compressionPressure ?? 0, config.preserveRecent, config.preserveRecentMin);
+    const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
     if (keepFrom <= 0)
         return { skipped: true };
     runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
@@ -30,6 +36,7 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
         summary: opts.summary,
         timestamp: Date.now(),
         onTier: runtime.makeTierCallback(ctx),
+        compressionPressure: opts.compressionPressure,
     }, runtime.store);
     runtime.pulsing = false;
     if (result.skipped)
@@ -96,6 +103,34 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
         tokenEstimate: result.tokenEstimate,
         deduped: result.deduped,
     });
+    // Fix D: refresh the RAPTOR tree for this session so live recall (search) can
+    // serve high-level summaries. Best-effort + non-fatal: never block compaction.
+    // Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
+    if (config.raptorEnabled && !result.deduped) {
+        try {
+            const dd = loadDedupConfig();
+            const all = runtime.store.list(sid);
+            const leaves = all.map((cp) => ({
+                id: cp.checkpointId,
+                messages: [],
+                sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+                embedding: cp.embedding,
+            }));
+            if (leaves.length >= 2) {
+                runRaptor(leaves, {
+                    stateDir: runtime.currentStateDir,
+                    sessionId: sid,
+                    budgetMs: dd.RAPTOR_BUDGET_MS,
+                    clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
+                    consistencyThreshold: dd.RAPTOR_CONSISTENCY,
+                    logger: runtime.logger,
+                });
+            }
+        }
+        catch {
+            /* non-fatal: tree refresh never blocks a compaction */
+        }
+    }
     runtime.setStatus(ctx, runtime.rt.persistedThisSession
         ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
         : `mega-compact: ready`);
@@ -124,7 +159,21 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
 export function doRecall(runtime, config, ctx, query, source) {
     runtime.bindRepo(ctx.cwd);
     const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-    const result = recallAndInline({ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true }, runtime.store);
+    // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
+    // are already resident in the session, so recall never re-injects context the
+    // model can already see. Best-effort — an empty window just skips dedupe.
+    const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
+    const result = recallAndInline({
+        sessionId: sid,
+        query,
+        limit: config.autoInlineK,
+        source,
+        skipInjected: true,
+        recallMaxTokens: config.recallMaxTokens,
+        windowDedupe: config.windowDedupe,
+        liveWindow,
+        dedupSim: config.dedupSim,
+    }, runtime.store);
     runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
     if (!result.empty && result.toInject.length > 0) {
         const top = result.toInject[0];
@@ -136,3 +185,28 @@ export function doRecall(runtime, config, ctx, query, source) {
     }
     return result;
 }
+/**
+ * Extract the live-window message texts from the session manager (Fix C),
+ * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
+ * error so recall falls back to unbounded (still correct, just no dedupe).
+ * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
+ */
+function extractLiveWindow(ctx) {
+    try {
+        const entries = ctx.sessionManager.getEntries();
+        const texts = [];
+        for (const e of entries) {
+            for (const m of sessionEntryToContextMessages(e)) {
+                const c = m.content;
+                if (typeof c === "string")
+                    texts.push(c);
+                else if (Array.isArray(c))
+                    texts.push(c.map((b) => b.text).join(" "));
+            }
+        }
+        return texts;
+    }
+    catch {
+        return [];
+    }
+}

package/dist/src/config/dedup.js

@@ -33,7 +33,10 @@ export function loadDedupConfig() {
         L0_ENABLED: envBool("MEGACOMPACT_L0_ENABLED", true),
         L1_ENABLED: envBool("MEGACOMPACT_L1_ENABLED", true),
         L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
-        RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", false), // shadow by default
+        // Fix D: RAPTOR promoted to live recall. Default ON; canary.ts sequences it
+        // last (L0→L1→L2→RAPTOR) and auto-disables on p95 breach, so promotion is
+        // safe. `RAPTOR_SHADOW_MODE=false` still gates serving during transition.
+        RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
         MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
         MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
         MARK_ONLY_L2: envBool("MEGACOMPACT_MARK_ONLY_L2", false),

package/dist/src/config.js

@@ -10,3 +10,24 @@ import { homedir } from "node:os";
 export const STATE_DIR_DEFAULT = join(homedir(), ".pi", "agent", "extensions", "pi-mega-compact");
 /** Pi custom message / entry type used as the dedup sentinel. */
 export const MARKER_TYPE = "mega-compact-marker";
+/**
+ * Derive context-window pressure (0–1) from a usage percentage. Used to scale
+ * compression strength + keepFrom depth (Fix E): low pct = room to spare,
+ * high pct = near the limit. Deterministic; clamps to [0,1].
+ */
+export function pressureFromPct(pct) {
+    if (pct == null || Number.isNaN(pct))
+        return 0;
+    return pct < 0 ? 0 : pct > 100 ? 1 : pct / 100;
+}
+/**
+ * Map pressure → how many recent messages to preserve verbatim. Under low
+ * pressure we keep `preserveRecent`; under high pressure we compact deeper,
+ * down to `preserveRecentMin`. Never splits a tool pair / anchor floor — the
+ * boundary guard (computeDropRange) enforces that downstream.
+ */
+export function preserveRecentForPressure(pressure, preserveRecent, preserveRecentMin) {
+    const p = pressure < 0 ? 0 : pressure > 1 ? 1 : pressure;
+    const v = Math.round(preserveRecent - (preserveRecent - preserveRecentMin) * p);
+    return Math.max(preserveRecentMin, Math.min(preserveRecent, v));
+}

package/dist/src/dedup/raptor/index.js

@@ -66,11 +66,20 @@ export function runRaptor(leaves, opts) {
  */
 export function recallRaptor(query, sessionId, opts) {
     const embedder = opts.embedder ?? defaultEmbedder();
-    const nodes = listRaptorNodes(sessionId, opts.stateDir);
-    if (nodes.length === 0)
+    const tree = rehydrateRaptorTree(sessionId, opts.stateDir);
+    if (!tree)
         return [];
-    // Rehydrate a minimal in-memory tree (parent links reconstructed from children).
-    const byId = new Map(nodes.map((n) => [n.id, n]));
+    return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+}
+/**
+ * Rehydrate a persisted RAPTOR tree from raptor_nodes (Fix D): rebuild the
+ * in-memory RaptorTree + parent links so vectorStore.search can serve it live.
+ * Returns null when no tree exists (caller falls back to the flat path).
+ */
+export function rehydrateRaptorTree(sessionId, stateDir) {
+    const nodes = listRaptorNodes(sessionId, stateDir);
+    if (nodes.length === 0)
+        return null;
     const tree = {
         nodes: new Map(nodes.map((n) => [
             n.id,
@@ -89,6 +98,19 @@ export function recallRaptor(query, sessionId, opts) {
         levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
         timedOut: false,
     };
-    void byId;
-    return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+    return tree;
+}
+/**
+ * Return the RAPTOR root summary for a session, if a tree has been built.
+ * Used by the durable-trim driver (Fix B/D) to supply pi a session-level
+ * compressed summary instead of one slice's extractive summary. Returns
+ * undefined when no tree exists yet (caller falls back to the slice summary).
+ */
+export function recallRaptorRootSummary(sessionId, stateDir) {
+    const nodes = listRaptorNodes(sessionId, stateDir);
+    if (nodes.length === 0)
+        return undefined;
+    // Highest-level node = the root (covers all leaves).
+    const root = nodes.reduce((best, n) => (!best || n.level > best.level ? n : best), null);
+    return root?.summary || undefined;
 }

package/dist/src/dedup/raptor/promote.test.js

@@ -0,0 +1,69 @@
+/**
+ * promote.test.ts — Fix D: RAPTOR tree served by vectorStore.search.
+ *
+ * Asserts that, when a RAPTOR tree has been built + persisted for a session,
+ * VectorStore.search returns the tree's staged-expansion hits (broader, O(log n)
+ * coverage) merged with the flat hits — so the dormant tree becomes the live
+ * recall surface. No network: default extractive summarizer + trigram embedder.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../../vectorStore.js";
+import { runRaptor } from "./index.js";
+import { compactSession } from "../../engine.js";
+import { Logger } from "../../log.js";
+import { loadDedupConfig } from "../../config/dedup.js";
+import { listRaptorNodes } from "../../store/sqlite.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-promote-"));
+let counter = 0;
+function raptorConfig() {
+    return { ...loadDedupConfig(), RAPTOR_ENABLED: true };
+}
+function msg(text, toolName) {
+    return toolName ? { role: "assistant", text, toolName, input: text, output: text } : { role: "user", text };
+}
+const SESS = "sess_promote";
+test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)", () => {
+    const stateDir = join(baseTmp, `run-${counter++}`);
+    const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
+    // Persist several distinct checkpoints.
+    for (let i = 1; i <= 5; i++) {
+        compactSession({ sessionId: SESS, messages: [msg(`topic alpha wire ${i} and bootstrap sequence`), msg(`ok ${i}`, "Edit")], keepFrom: 2, timestamp: i }, s);
+    }
+    // No tree yet → flat search only, returns hits, no RAPTOR coverage.
+    assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree initially");
+    const flat = s.search(SESS, "alpha wire bootstrap", 3);
+    assert.ok(flat.length > 0, "flat search returns hits");
+    // Build + persist a RAPTOR tree for the session (mirrors runCompact refresh).
+    const all = s.list(SESS);
+    const leaves = all.map((cp) => ({
+        id: cp.checkpointId,
+        messages: [],
+        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+        embedding: cp.embedding,
+    }));
+    const tree = runRaptor(leaves, { stateDir, sessionId: SESS, logger: new Logger() });
+    assert.ok(tree && listRaptorNodes(SESS, stateDir).length > 0, "tree persisted");
+    // With the tree live + RAPTOR_ENABLED, search still returns hits and now
+    // exercises the RAPTOR-served path without regression.
+    const withTree = s.search(SESS, "alpha wire bootstrap", 3);
+    assert.ok(withTree.length > 0, "search returns hits with RAPTOR promoted");
+    // Every returned hit is a real checkpoint in the session.
+    for (const h of withTree) {
+        assert.ok(all.some((cp) => cp.checkpointId === h.checkpoint.checkpointId), "hit is a real checkpoint");
+    }
+});
+test("Fix D: search still works for a session with <2 leaves (no tree)", () => {
+    const stateDir = join(baseTmp, `run-${counter++}`);
+    const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
+    compactSession({ sessionId: SESS, messages: [msg("only one topic here"), msg("ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    const r = s.search(SESS, "only one topic", 3);
+    assert.ok(r.length > 0, "single-checkpoint search still works (no tree)");
+    assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree built for <2 leaves");
+});
+test("cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});

package/dist/src/engine.js

@@ -104,6 +104,7 @@ export function compactSession(input, store = getDefaultStore()) {
         originalTokenEstimate,
         timestamp: input.timestamp ?? 0,
         onTier: input.onTier,
+        compressionPressure: input.compressionPressure,
     });
     return {
         skipped: false,

package/dist/src/recall.js

@@ -14,6 +14,8 @@
  * extension decides where it lands.
  */
 import { recall as searchRecall } from "./engine.js";
+import { estimateBlockTokens } from "./tokens.js";
+import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
 /** Wrap a recall block so the model reads it as restored compacted context. */
 export function formatRecallBlock(hits) {
     if (hits.length === 0)
@@ -38,18 +40,42 @@ export function formatRecallBlock(hits) {
 export function recallAndInline(opts, store) {
     const limit = opts.limit ?? 3;
     const skip = opts.skipInjected ?? true;
+    const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)
+    const doWindowDedupe = opts.windowDedupe ?? false;
+    const dedupSim = opts.dedupSim ?? 0.9;
     const { hits } = searchRecall({ sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false }, store);
-    // Shared dedup: drop checkpoints already injected this session, then mark the
-    // survivors so repeated triggers are free. (Cosine near-dup collapse already
-    // happened inside store.search.)
+    // Precompute live-window embeddings once for inline dedupe (Fix C). Trigram
+    // embedder is local + cheap; never a network call (PREVENT-PI-004).
+    let liveEmbeddings = [];
+    if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+        const embedder = defaultEmbedder();
+        liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+    }
+    // Shared dedup + bounded/inline block assembly. We build the block
+    // incrementally so the token cap can stop mid-stream (Fix C).
     const toInject = [];
+    const parts = [];
+    let blockTokens = 0;
     for (const h of hits) {
         if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
             continue;
+        // Inline dedupe: skip a hit already resident in the live window (Fix C).
+        if (doWindowDedupe && liveEmbeddings.length > 0) {
+            const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+            if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
+                continue;
+        }
+        const part = formatRecallBlock([h]);
+        const partTokens = estimateBlockTokens(part);
+        // Token cap: never push a chunk that would overrun the ceiling.
+        if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
+            break;
+        parts.push(part);
         toInject.push(h);
+        blockTokens += partTokens;
         store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
     }
-    const block = formatRecallBlock(toInject);
+    const block = parts.join("\n");
     const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
     return {
         toInject,

package/dist/src/recall.test.js

@@ -45,6 +45,34 @@ test("recallAndInline empty when store has nothing for query", () => {
     assert.equal(r.empty, true);
     assert.equal(r.block, "");
 });
+test("Fix C: recallMaxTokens caps the injected block", () => {
+    const s = store();
+    // Three distinct checkpoints so we can observe the cap bite mid-stream.
+    compactSession({ sessionId: SESS, messages: [msg("user", "alpha module wiring and bootstrap sequence"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "beta module config and env resolution"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "gamma module shutdown and cleanup hooks"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 3 }, s);
+    // A ceiling of 100 tokens fits the first checkpoint (~82) but stops before the
+    // second (~163 cumulative) — proving the cap bites mid-stream.
+    const r = recallAndInline({ sessionId: SESS, query: "module wiring config shutdown", limit: 5, source: "command", recallMaxTokens: 100, skipInjected: false }, s);
+    assert.ok(r.toInject.length >= 1, "at least one injected under the cap");
+    assert.ok(r.toInject.length < 3, "cap prevented all three from injecting");
+    assert.ok(r.block.length > 0, "block non-empty");
+});
+test("Fix C: inline dedupe drops a hit already resident in the live window", () => {
+    const s = store();
+    const resident = "alpha module wiring and bootstrap sequence";
+    compactSession({ sessionId: SESS, messages: [msg("user", resident), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "omega module telemetry and tracing spans"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+    // Baseline: with dedupe OFF, both checkpoints are candidates.
+    const rNoDedup = recallAndInline({ sessionId: SESS, query: "module wiring telemetry", limit: 5, source: "command", skipInjected: false }, s);
+    // The live window contains the exact summary of the first checkpoint — as it
+    // would be if a prior recall already injected it. Inline dedupe must drop it
+    // (strictly fewer injected than the no-dedupe baseline).
+    const residentSummary = rNoDedup.toInject[0].checkpoint.summary;
+    const rDedup = recallAndInline({ sessionId: SESS, query: "module wiring telemetry", limit: 5, source: "command", skipInjected: false, windowDedupe: true, liveWindow: [residentSummary], dedupSim: 0.9 }, s);
+    assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
+    assert.ok(rDedup.toInject.length < rNoDedup.toInject.length, "inline dedupe dropped a resident hit");
+});
 test("cleanup", () => {
     rmSync(baseTmp, { recursive: true, force: true });
 });

package/dist/src/store/backfill.js

@@ -19,7 +19,7 @@ import { openStore } from "./sqlite.js";
 import { computeContentDigest } from "../dedup/digest.js";
 import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
 import { lshBands } from "../dedup/l1-lsh.js";
-import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree } from "./sqlite.js";
+import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree, withTx } from "./sqlite.js";
 import { buildRaptorTree } from "../dedup/raptor/tree.js";
 import { defaultEmbedder } from "../embedder.js";
 import { getStateDir } from "../store.js";
@@ -64,7 +64,7 @@ export function backfillContentHashes(stateDir = getStateDir()) {
     let processed = 0;
     let lastSid = start.lastSid;
     let lastId = start.lastId;
-    const tx = db.transaction((rows) => {
+    function applyRows(rows) {
         const lookup = db.prepare("SELECT id FROM context_chunks WHERE session_id = ? AND content_hash = ? AND content_hash2 = ? AND id != ? LIMIT 1");
         const update = db.prepare(`UPDATE context_chunks
        SET content_hash=?, content_hash2=?, content_hash_version=?, normalized_text=?,
@@ -87,9 +87,9 @@ export function backfillContentHashes(stateDir = getStateDir()) {
             lastId = row.id;
             processed++;
         }
-    });
+    }
     if (pending.length > 0) {
-        tx(pending);
+        withTx(db, () => applyRows(pending));
         db.prepare("INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved) VALUES('content_hashes',?,?,?,?) ON CONFLICT(name) DO UPDATE SET last_session_id=excluded.last_session_id, last_id=excluded.last_id, updated=excluded.updated, duplicates_resolved=excluded.duplicates_resolved").run(lastSid, lastId, updated, duplicatesResolved);
     }
     if (THROTTLE_MS > 0) {
@@ -135,7 +135,7 @@ export function backfillPhase(phase, sessionId, stateDir, opts = {}) {
     let cursor = lastId ?? undefined;
     for (let i = Math.max(0, startIndex); i < all.length; i += batchSize) {
         const batch = all.slice(i, i + batchSize);
-        const tx = db.transaction(() => {
+        withTx(db, () => {
             for (const cp of batch) {
                 const sig = minhashSignature(cp.normalizedText ?? cp.summary ?? "");
                 if (sig.length === NUM_HASHES) {
@@ -148,7 +148,6 @@ export function backfillPhase(phase, sessionId, stateDir, opts = {}) {
                 processed++;
             }
         });
-        tx();
         savePhaseCursor(db, phase, cursor ?? null, processed);
         batches++;
         if (THROTTLE_MS > 0) {

package/dist/src/store/compression.js

@@ -5,7 +5,7 @@
  *
  *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
  *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
- *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *     node:sqlite replaced PGlite precisely to avoid an async cascade).
  *
  *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
  *     Optional, used for DR-export / large-blob paths where an await is fine.
@@ -25,7 +25,13 @@
  * decompressSmart detects the magic first, so all three eras roundtrip together.
  */
 import { gzipSync, gunzipSync, brotliCompressSync, brotliDecompressSync, constants as zlibConstants, } from "node:zlib";
-import zstd from "@mongodb-js/zstd";
+// zstd is loaded lazily (see compressZstdWithLevel / decompressZstd). It is an
+// OPTIONAL async DR-export dependency: its native addon (`zstd.node`) is not in
+// the npm tarball and may be absent on a clean/allowScripts-blocked install, so
+// a static import here would crash the whole extension at load time. Lazy
+// import keeps the extension loadable even when the binary is missing; the DR
+// path throws a clear error only if it is actually used. (Fix A.)
+// import zstd from "@mongodb-js/zstd";
 // --- Versioned format markers ----------------------------------------------
 const MAGIC_HI = 0xec;
 const MAGIC_LO = 0x01; // format version 1
@@ -45,6 +51,12 @@ const SIZE_MEDIUM = 32768;
 function header(ver, tag) {
     return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
 }
+/** Clamp a value to the [0, 1] range (pressure bands). */
+function clamp01(n) {
+    if (Number.isNaN(n))
+        return 0;
+    return n < 0 ? 0 : n > 1 ? 1 : n;
+}
 /**
  * Compress synchronously using the best zlib tier for the payload size.
  *
@@ -54,21 +66,36 @@ function header(ver, tag) {
  *   4KB–32KB → gzip level 6 (tag 0x02)
  *   > 32 KB  → brotli 4     (tag 0x05)
  *
- * Writes the versioned header so readers disambiguate from legacy blobs.
+ * `pressure` (0–1, optional) escalates the brotli quality for the large tier
+ * when the session is near its context limit — the "variable compression as we
+ * approach the limit" design (Fix E). Low/undefined pressure keeps brotli-4;
+ * high pressure pushes toward brotli-11. Stays fully synchronous (brotli-11 is
+ * sync via brotliCompressSync) so the sync `add()` contract is preserved; zstd
+ * is reserved for the async DR-export path only. Same versioned header/tags for
+ * every pressure, so decompressSmart is unaffected.
  */
-export function compressSmart(data) {
+export function compressSmart(data, pressure = 0) {
+    const p = clamp01(pressure);
     const len = data.length;
     if (len < SIZE_TINY) {
         return Buffer.concat([header(1, TAG_RAW), data]);
     }
     if (len < SIZE_SMALL) {
-        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+        // Small tier: escalate gzip level 1 → 9 with context pressure (Fix E) so
+        // the "variable compression as we approach the limit" dial bites for
+        // short sessions too, not just the >32KB brotli tier.
+        const level = Math.max(1, Math.min(9, Math.round(1 + 8 * p)));
+        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level })]);
     }
     if (len < SIZE_MEDIUM) {
-        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+        // Medium tier: escalate gzip level 6 → 9 with context pressure (Fix E).
+        const level = Math.max(6, Math.min(9, Math.round(6 + 3 * p)));
+        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level })]);
     }
+    // Large tier: escalate brotli quality 4 → 11 with context pressure (Fix E).
+    const quality = Math.max(4, Math.min(11, Math.round(4 + 7 * p)));
     const compressed = brotliCompressSync(data, {
-        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: quality },
     });
     return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
 }
@@ -141,6 +168,17 @@ export function decompressSmart(buf) {
 const ZSTD_MAGIC_HI = 0x5a; // 'Z'
 const ZSTD_MAGIC_LO = 0x53; // 'S'
 async function compressZstdWithLevel(data, level) {
+    // Lazy import: the native addon may be absent (clean/allowScripts install).
+    // Throws a clear, actionable error instead of a load-time crash.
+    let zstd;
+    try {
+        zstd = await import("@mongodb-js/zstd");
+    }
+    catch {
+        throw new Error("zstd is not available — the @mongodb-js/zstd native addon (zstd.node) " +
+            "was not built. Run the extension's native install step (or allow npm " +
+            "install scripts) to enable DR-export compression.");
+    }
     const compressed = await zstd.compress(data, level);
     return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
 }
@@ -163,6 +201,8 @@ export async function decompressZstd(buf) {
     if (!isZstd(buf)) {
         throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
     }
+    // Lazy import (see compressZstdWithLevel for rationale).
+    const zstd = await import("@mongodb-js/zstd");
     return zstd.decompress(buf.subarray(2));
 }
 /**

package/dist/src/store/compression.test.js

@@ -65,3 +65,51 @@ test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
     assert.equal(auto.isZstd, true, "flagged as zstd");
     assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
 });
+test("module loads without a top-level zstd import (Fix A: no load crash)", async () => {
+    // The extension must load even when the @mongodb-js/zstd native addon is
+    // absent (clean/allowScripts-blocked install). The dynamic import() lives
+    // inside the helpers, so importing this module must never throw.
+    const mod = await import("./compression.js");
+    assert.equal(typeof mod.compressSmart, "function", "compressSmart exported");
+    assert.equal(typeof mod.compressZstd, "function", "compressZstd exported");
+    // The real invariant: no STATIC `import ... from "@mongodb-js/zstd"` at the
+    // top level (that's what crashed the whole extension). zstd must be loaded
+    // lazily inside the helpers only. Check the source text.
+    const { readFileSync } = await import("node:fs");
+    const { join } = await import("node:path");
+    // Tests run with cwd at repo root (`node --test`), so resolve the source.
+    const src = readFileSync(join(process.cwd(), "src/store/compression.ts"), "utf-8");
+    const staticImport = /^import\s+.+\s+from\s+["']@mongodb-js\/zstd["'];?$/m;
+    assert.equal(staticImport.test(src), false, "no static top-level import of @mongodb-js/zstd (would crash load if binary absent)");
+    assert.ok(src.includes('await import("@mongodb-js/zstd")'), "zstd is loaded lazily via dynamic import() inside the helpers");
+});
+test("compressSmart escalates brotli quality with pressure (Fix E)", () => {
+    // Large (>32KB) payloads hit the brotli tier; higher pressure → brotli-11
+    // → smaller output than the default brotli-4, and still decodes.
+    const words = Array.from({ length: 6000 }, (_, i) => "word" + ((i * 2654435761) % 9973));
+    const big = Buffer.from(words.join(" "));
+    const low = compressSmart(big, 0);
+    const high = compressSmart(big, 1);
+    assert.equal(isVersioned(low), true, "versioned header preserved at p=0");
+    assert.equal(isVersioned(high), true, "versioned header preserved at p=1");
+    assert.ok(high.length < low.length, "high pressure compresses smaller");
+    assert.deepEqual(decompressSmart(low), big, "p=0 roundtrip");
+    assert.deepEqual(decompressSmart(high), big, "p=1 roundtrip");
+    // Small payloads ignore pressure (gzip tier) but still roundtrip.
+    const small = buf("hello world ", 300);
+    assert.deepEqual(decompressSmart(compressSmart(small, 1)), small, "small ignores pressure");
+    // pressure out of range is clamped (no throw, still versioned + decodable).
+    assert.deepEqual(decompressSmart(compressSmart(big, 5)), big, "over-pressure clamped");
+    assert.deepEqual(decompressSmart(compressSmart(big, -1)), big, "under-pressure clamped");
+});
+test("pressureFromPct + preserveRecentForPressure scale with context (Fix E)", async () => {
+    const { pressureFromPct, preserveRecentForPressure } = await import("../config.js");
+    assert.equal(pressureFromPct(50), 0.5, "pct→pressure");
+    assert.equal(pressureFromPct(null), 0, "null pct → 0");
+    assert.equal(pressureFromPct(150), 1, "pct clamped");
+    // low pressure keeps preserveRecent; high pressure compacts deeper (min floor).
+    assert.equal(preserveRecentForPressure(0, 4, 2), 4, "p=0 → preserveRecent");
+    assert.equal(preserveRecentForPressure(1, 4, 2), 2, "p=1 → preserveRecentMin");
+    assert.equal(preserveRecentForPressure(0.5, 4, 2), 3, "p=0.5 → interpolates");
+    assert.ok(preserveRecentForPressure(1, 4, 2) >= 2, "never below floor");
+});