pi-mega-compact 0.4.21 → 0.4.23

package/src/dedup/raptor/index.ts

@@ -89,10 +89,22 @@ export function recallRaptor(
   opts: { embedder?: Embedder; stateDir: string; k?: number; topM?: number },
 ): string[] {
   const embedder = opts.embedder ?? defaultEmbedder();
-  const nodes = listRaptorNodes(sessionId, opts.stateDir);
-  if (nodes.length === 0) return [];
-  // Rehydrate a minimal in-memory tree (parent links reconstructed from children).
-  const byId = new Map(nodes.map((n) => [n.id, n]));
+  const tree = rehydrateRaptorTree(sessionId, opts.stateDir);
+  if (!tree) return [];
+  return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+}
+
+/**
+ * Rehydrate a persisted RAPTOR tree from raptor_nodes (Fix D): rebuild the
+ * in-memory RaptorTree + parent links so vectorStore.search can serve it live.
+ * Returns null when no tree exists (caller falls back to the flat path).
+ */
+export function rehydrateRaptorTree(
+  sessionId: string,
+  stateDir: string,
+): RaptorTree | null {
+  const nodes = listRaptorNodes(sessionId, stateDir);
+  if (nodes.length === 0) return null;
   const tree: RaptorTree = {
     nodes: new Map(
       nodes.map((n) => [
@@ -109,10 +121,33 @@ export function recallRaptor(
         },
       ]),
     ),
-    rootId: nodes.reduce<typeof nodes[number] | null>((best, n) => (!best || n.level > (best?.level ?? -1) ? n : best), null)?.id ?? null,
+    rootId:
+      nodes.reduce<typeof nodes[number] | null>(
+        (best, n) => (!best || n.level > (best?.level ?? -1) ? n : best),
+        null,
+      )?.id ?? null,
     levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
     timedOut: false,
   };
-  void byId;
-  return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+  return tree;
+}
+
+/**
+ * Return the RAPTOR root summary for a session, if a tree has been built.
+ * Used by the durable-trim driver (Fix B/D) to supply pi a session-level
+ * compressed summary instead of one slice's extractive summary. Returns
+ * undefined when no tree exists yet (caller falls back to the slice summary).
+ */
+export function recallRaptorRootSummary(
+  sessionId: string,
+  stateDir: string,
+): string | undefined {
+  const nodes = listRaptorNodes(sessionId, stateDir);
+  if (nodes.length === 0) return undefined;
+  // Highest-level node = the root (covers all leaves).
+  const root = nodes.reduce<(typeof nodes)[number] | null>(
+    (best, n) => (!best || n.level > best.level ? n : best),
+    null,
+  );
+  return root?.summary || undefined;
 }

package/src/dedup/raptor/promote.test.ts

@@ -0,0 +1,82 @@
+/**
+ * promote.test.ts — Fix D: RAPTOR tree served by vectorStore.search.
+ *
+ * Asserts that, when a RAPTOR tree has been built + persisted for a session,
+ * VectorStore.search returns the tree's staged-expansion hits (broader, O(log n)
+ * coverage) merged with the flat hits — so the dormant tree becomes the live
+ * recall surface. No network: default extractive summarizer + trigram embedder.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../../vectorStore.js";
+import { runRaptor } from "./index.js";
+import { compactSession } from "../../engine.js";
+import { Logger } from "../../log.js";
+import { loadDedupConfig } from "../../config/dedup.js";
+import { listRaptorNodes } from "../../store/sqlite.js";
+import type { EngineMessage } from "../../types.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-promote-"));
+let counter = 0;
+function raptorConfig() {
+  return { ...loadDedupConfig(), RAPTOR_ENABLED: true };
+}
+function msg(text: string, toolName?: string): EngineMessage {
+  return toolName ? { role: "assistant", text, toolName, input: text, output: text } : { role: "user", text };
+}
+const SESS = "sess_promote";
+
+test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)", () => {
+  const stateDir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
+
+  // Persist several distinct checkpoints.
+  for (let i = 1; i <= 5; i++) {
+    compactSession(
+      { sessionId: SESS, messages: [msg(`topic alpha wire ${i} and bootstrap sequence`), msg(`ok ${i}`, "Edit")], keepFrom: 2, timestamp: i },
+      s,
+    );
+  }
+
+  // No tree yet → flat search only, returns hits, no RAPTOR coverage.
+  assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree initially");
+  const flat = s.search(SESS, "alpha wire bootstrap", 3);
+  assert.ok(flat.length > 0, "flat search returns hits");
+
+  // Build + persist a RAPTOR tree for the session (mirrors runCompact refresh).
+  const all = s.list(SESS);
+  const leaves = all.map((cp) => ({
+    id: cp.checkpointId,
+    messages: [],
+    sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+    embedding: cp.embedding,
+  }));
+  const tree = runRaptor(leaves, { stateDir, sessionId: SESS, logger: new Logger() });
+  assert.ok(tree && listRaptorNodes(SESS, stateDir).length > 0, "tree persisted");
+
+  // With the tree live + RAPTOR_ENABLED, search still returns hits and now
+  // exercises the RAPTOR-served path without regression.
+  const withTree = s.search(SESS, "alpha wire bootstrap", 3);
+  assert.ok(withTree.length > 0, "search returns hits with RAPTOR promoted");
+  // Every returned hit is a real checkpoint in the session.
+  for (const h of withTree) {
+    assert.ok(all.some((cp) => cp.checkpointId === h.checkpoint.checkpointId), "hit is a real checkpoint");
+  }
+});
+
+test("Fix D: search still works for a session with <2 leaves (no tree)", () => {
+  const stateDir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
+  compactSession({ sessionId: SESS, messages: [msg("only one topic here"), msg("ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+  const r = s.search(SESS, "only one topic", 3);
+  assert.ok(r.length > 0, "single-checkpoint search still works (no tree)");
+  assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree built for <2 leaves");
+});
+
+test("cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/engine.ts

@@ -38,6 +38,10 @@ export interface CompactInput {
   timestamp?: number;
   /** When true (default), use extractive summary instead of raw concatenation. */
   useExtractiveSummary?: boolean;
+  /** Context-window pressure (0–1): how close the session is to the model
+   *  limit. Drives adaptive compression strength in the stored checkpoint
+   *  (Fix E). 0/undefined = room to spare; 1 = at the limit. */
+  compressionPressure?: number;
   /** Sync progress callback fired by the store as each dedup tier is evaluated
    *  (L0→L1→L2→new). Lets the UI render a live "L0 ✓ → L1 ✓ → L2 0.91 → stored"
    *  progress line during compaction. Never awaited; must be side-effect-free-ish
@@ -174,6 +178,7 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
     originalTokenEstimate,
     timestamp: input.timestamp ?? 0,
     onTier: input.onTier,
+    compressionPressure: input.compressionPressure,
   });
 
   return {

package/src/recall.test.ts

@@ -54,6 +54,50 @@ test("recallAndInline empty when store has nothing for query", () => {
   assert.equal(r.block, "");
 });
 
+test("Fix C: recallMaxTokens caps the injected block", () => {
+  const s = store();
+  // Three distinct checkpoints so we can observe the cap bite mid-stream.
+  compactSession({ sessionId: SESS, messages: [msg("user", "alpha module wiring and bootstrap sequence"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+  compactSession({ sessionId: SESS, messages: [msg("user", "beta module config and env resolution"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+  compactSession({ sessionId: SESS, messages: [msg("user", "gamma module shutdown and cleanup hooks"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 3 }, s);
+
+  // A ceiling of 100 tokens fits the first checkpoint (~82) but stops before the
+  // second (~163 cumulative) — proving the cap bites mid-stream.
+  const r = recallAndInline(
+    { sessionId: SESS, query: "module wiring config shutdown", limit: 5, source: "command", recallMaxTokens: 100, skipInjected: false },
+    s as any,
+  );
+  assert.ok(r.toInject.length >= 1, "at least one injected under the cap");
+  assert.ok(r.toInject.length < 3, "cap prevented all three from injecting");
+  assert.ok(r.block.length > 0, "block non-empty");
+});
+
+test("Fix C: inline dedupe drops a hit already resident in the live window", () => {
+  const s = store();
+  const resident = "alpha module wiring and bootstrap sequence";
+  compactSession({ sessionId: SESS, messages: [msg("user", resident), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+  compactSession({ sessionId: SESS, messages: [msg("user", "omega module telemetry and tracing spans"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+
+  // Baseline: with dedupe OFF, both checkpoints are candidates.
+  const rNoDedup = recallAndInline(
+    { sessionId: SESS, query: "module wiring telemetry", limit: 5, source: "command", skipInjected: false },
+    s as any,
+  );
+  // The live window contains the exact summary of the first checkpoint — as it
+  // would be if a prior recall already injected it. Inline dedupe must drop it
+  // (strictly fewer injected than the no-dedupe baseline).
+  const residentSummary = rNoDedup.toInject[0].checkpoint.summary;
+  const rDedup = recallAndInline(
+    { sessionId: SESS, query: "module wiring telemetry", limit: 5, source: "command", skipInjected: false, windowDedupe: true, liveWindow: [residentSummary], dedupSim: 0.9 },
+    s as any,
+  );
+  assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
+  assert.ok(
+    rDedup.toInject.length < rNoDedup.toInject.length,
+    "inline dedupe dropped a resident hit",
+  );
+});
+
 test("cleanup", () => {
   rmSync(baseTmp, { recursive: true, force: true });
 });

package/src/recall.ts

@@ -16,6 +16,8 @@
 
 import { recall as searchRecall } from "./engine.js";
 import type { SearchHit, VectorStore } from "./vectorStore.js";
+import { estimateBlockTokens } from "./tokens.js";
+import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
 
 export type RecallSource = "resume" | "command" | "sentinel";
 
@@ -26,6 +28,16 @@ export interface RecallInjectOptions {
   source: RecallSource;
   /** Skip checkpoints already injected this session (recall dedup). */
   skipInjected?: boolean;
+  /** Token ceiling for the re-injected block (Fix C). Recall stops adding once
+   *  the block would exceed this, so the read path can never net-inflate. */
+  recallMaxTokens?: number;
+  /** Inline-dedupe hits against the live window (Fix C): drop a hit whose
+   *  summary is ≥ `dedupSim` similar to a live message. */
+  windowDedupe?: boolean;
+  /** Live window text (from the session manager) used for inline dedupe. */
+  liveWindow?: string[];
+  /** Similarity threshold for inline dedupe (defaults to 0.9). */
+  dedupSim?: number;
 }
 
 export interface RecallInjectResult {
@@ -70,23 +82,50 @@ export function recallAndInline(
 ): RecallInjectResult {
   const limit = opts.limit ?? 3;
   const skip = opts.skipInjected ?? true;
+  const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)
+  const doWindowDedupe = opts.windowDedupe ?? false;
+  const dedupSim = opts.dedupSim ?? 0.9;
 
   const { hits } = searchRecall(
     { sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false },
     store as VectorStore,
   );
 
-  // Shared dedup: drop checkpoints already injected this session, then mark the
-  // survivors so repeated triggers are free. (Cosine near-dup collapse already
-  // happened inside store.search.)
+  // Precompute live-window embeddings once for inline dedupe (Fix C). Trigram
+  // embedder is local + cheap; never a network call (PREVENT-PI-004).
+  let liveEmbeddings: number[][] = [];
+  if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+    const embedder = defaultEmbedder();
+    liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+  }
+
+  // Shared dedup + bounded/inline block assembly. We build the block
+  // incrementally so the token cap can stop mid-stream (Fix C).
   const toInject: SearchHit[] = [];
+  const parts: string[] = [];
+  let blockTokens = 0;
+
   for (const h of hits) {
     if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
+
+    // Inline dedupe: skip a hit already resident in the live window (Fix C).
+    if (doWindowDedupe && liveEmbeddings.length > 0) {
+      const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+      if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
+    }
+
+    const part = formatRecallBlock([h]);
+    const partTokens = estimateBlockTokens(part);
+    // Token cap: never push a chunk that would overrun the ceiling.
+    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+
+    parts.push(part);
     toInject.push(h);
+    blockTokens += partTokens;
     store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
   }
 
-  const block = formatRecallBlock(toInject);
+  const block = parts.join("\n");
   const report = toInject.map(
     (h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
   );

package/src/store/backfill.ts

@@ -16,12 +16,12 @@
  * SQLite is the source of truth; this touches no network (PREVENT-PI-004).
  */
 
-import type { Database } from "better-sqlite3";
+import type { DatabaseSync } from "node:sqlite";
 import { openStore } from "./sqlite.js";
 import { computeContentDigest } from "../dedup/digest.js";
 import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
 import { lshBands } from "../dedup/l1-lsh.js";
-import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree } from "./sqlite.js";
+import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree, withTx } from "./sqlite.js";
 import { buildRaptorTree, type Leaf } from "../dedup/raptor/tree.js";
 import type { Embedder } from "../embedder.js";
 import { defaultEmbedder } from "../embedder.js";
@@ -45,7 +45,7 @@ interface PhaseProgressRow {
   processed: number;
 }
 
-function ensureProgressTable(db: Database): void {
+function ensureProgressTable(db: DatabaseSync): void {
   db.exec(`
     CREATE TABLE IF NOT EXISTS backfill_progress (
       name TEXT PRIMARY KEY,
@@ -57,7 +57,7 @@ function ensureProgressTable(db: Database): void {
   `);
 }
 
-function progress(db: Database): { lastSid: string | null; lastId: string | null; updated: number; dups: number } {
+function progress(db: DatabaseSync): { lastSid: string | null; lastId: string | null; updated: number; dups: number } {
   const row = db
     .prepare("SELECT last_session_id, last_id, updated, duplicates_resolved FROM backfill_progress WHERE name='content_hashes'")
     .get() as { last_session_id: string | null; last_id: string | null; updated: number; duplicates_resolved: number } | undefined;
@@ -91,7 +91,7 @@ export function backfillContentHashes(stateDir: string = getStateDir()): Backfil
   let lastSid = start.lastSid;
   let lastId = start.lastId;
 
-  const tx = db.transaction((rows: { id: string; session_id: string; summary: string }[]) => {
+  function applyRows(rows: { id: string; session_id: string; summary: string }[]): void {
     const lookup = db.prepare(
       "SELECT id FROM context_chunks WHERE session_id = ? AND content_hash = ? AND content_hash2 = ? AND id != ? LIMIT 1",
     );
@@ -123,10 +123,10 @@ export function backfillContentHashes(stateDir: string = getStateDir()): Backfil
       lastId = row.id;
       processed++;
     }
-  });
+  }
 
   if (pending.length > 0) {
-    tx(pending);
+    withTx(db, () => applyRows(pending));
     db.prepare(
       "INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved) VALUES('content_hashes',?,?,?,?) ON CONFLICT(name) DO UPDATE SET last_session_id=excluded.last_session_id, last_id=excluded.last_id, updated=excluded.updated, duplicates_resolved=excluded.duplicates_resolved",
     ).run(lastSid, lastId, updated, duplicatesResolved);
@@ -150,7 +150,7 @@ export function isBackfillComplete(stateDir: string = getStateDir()): boolean {
 
 // ---- Sprint 14: L1 / L2 / RAPTOR phase backfill (resumable) ---------------
 
-function phaseCursor(db: Database, phase: BackfillPhase): { lastId: string | null; processed: number } {
+function phaseCursor(db: DatabaseSync, phase: BackfillPhase): { lastId: string | null; processed: number } {
   ensureProgressTable(db);
   const row = db
     .prepare("SELECT last_id, updated AS processed FROM backfill_progress WHERE name = ?")
@@ -158,7 +158,7 @@ function phaseCursor(db: Database, phase: BackfillPhase): { lastId: string | nul
   return { lastId: row?.last_id ?? null, processed: row?.processed ?? 0 };
 }
 
-function savePhaseCursor(db: Database, phase: BackfillPhase, lastId: string | null, processed: number): void {
+function savePhaseCursor(db: DatabaseSync, phase: BackfillPhase, lastId: string | null, processed: number): void {
   db.prepare(
     `INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved)
      VALUES(?, NULL, ?, ?, 0)
@@ -201,7 +201,7 @@ export function backfillPhase(
 
   for (let i = Math.max(0, startIndex); i < all.length; i += batchSize) {
     const batch = all.slice(i, i + batchSize);
-    const tx = db.transaction(() => {
+    withTx(db, () => {
       for (const cp of batch) {
         const sig = minhashSignature(cp.normalizedText ?? cp.summary ?? "");
         if (sig.length === NUM_HASHES) {
@@ -217,7 +217,6 @@ export function backfillPhase(
         processed++;
       }
     });
-    tx();
     savePhaseCursor(db, phase, cursor ?? null, processed);
     batches++;
     if (THROTTLE_MS > 0) { const end = Date.now() + THROTTLE_MS; while (Date.now() < end) { /* throttle */ } }

package/src/store/compression.test.ts

@@ -81,3 +81,61 @@ test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
   assert.equal(auto.isZstd, true, "flagged as zstd");
   assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
 });
+
+test("module loads without a top-level zstd import (Fix A: no load crash)", async () => {
+  // The extension must load even when the @mongodb-js/zstd native addon is
+  // absent (clean/allowScripts-blocked install). The dynamic import() lives
+  // inside the helpers, so importing this module must never throw.
+  const mod = await import("./compression.js");
+  assert.equal(typeof mod.compressSmart, "function", "compressSmart exported");
+  assert.equal(typeof mod.compressZstd, "function", "compressZstd exported");
+  // The real invariant: no STATIC `import ... from "@mongodb-js/zstd"` at the
+  // top level (that's what crashed the whole extension). zstd must be loaded
+  // lazily inside the helpers only. Check the source text.
+  const { readFileSync } = await import("node:fs");
+  const { join } = await import("node:path");
+  // Tests run with cwd at repo root (`node --test`), so resolve the source.
+  const src = readFileSync(join(process.cwd(), "src/store/compression.ts"), "utf-8");
+  const staticImport = /^import\s+.+\s+from\s+["']@mongodb-js\/zstd["'];?$/m;
+  assert.equal(
+    staticImport.test(src),
+    false,
+    "no static top-level import of @mongodb-js/zstd (would crash load if binary absent)",
+  );
+  assert.ok(
+    src.includes('await import("@mongodb-js/zstd")'),
+    "zstd is loaded lazily via dynamic import() inside the helpers",
+  );
+});
+
+test("compressSmart escalates brotli quality with pressure (Fix E)", () => {
+  // Large (>32KB) payloads hit the brotli tier; higher pressure → brotli-11
+  // → smaller output than the default brotli-4, and still decodes.
+  const words = Array.from({ length: 6000 }, (_, i) => "word" + ((i * 2654435761) % 9973));
+  const big = Buffer.from(words.join(" "));
+  const low = compressSmart(big, 0);
+  const high = compressSmart(big, 1);
+  assert.equal(isVersioned(low), true, "versioned header preserved at p=0");
+  assert.equal(isVersioned(high), true, "versioned header preserved at p=1");
+  assert.ok(high.length < low.length, "high pressure compresses smaller");
+  assert.deepEqual(decompressSmart(low), big, "p=0 roundtrip");
+  assert.deepEqual(decompressSmart(high), big, "p=1 roundtrip");
+  // Small payloads ignore pressure (gzip tier) but still roundtrip.
+  const small = buf("hello world ", 300);
+  assert.deepEqual(decompressSmart(compressSmart(small, 1)), small, "small ignores pressure");
+  // pressure out of range is clamped (no throw, still versioned + decodable).
+  assert.deepEqual(decompressSmart(compressSmart(big, 5)), big, "over-pressure clamped");
+  assert.deepEqual(decompressSmart(compressSmart(big, -1)), big, "under-pressure clamped");
+});
+
+test("pressureFromPct + preserveRecentForPressure scale with context (Fix E)", async () => {
+  const { pressureFromPct, preserveRecentForPressure } = await import("../config.js");
+  assert.equal(pressureFromPct(50), 0.5, "pct→pressure");
+  assert.equal(pressureFromPct(null), 0, "null pct → 0");
+  assert.equal(pressureFromPct(150), 1, "pct clamped");
+  // low pressure keeps preserveRecent; high pressure compacts deeper (min floor).
+  assert.equal(preserveRecentForPressure(0, 4, 2), 4, "p=0 → preserveRecent");
+  assert.equal(preserveRecentForPressure(1, 4, 2), 2, "p=1 → preserveRecentMin");
+  assert.equal(preserveRecentForPressure(0.5, 4, 2), 3, "p=0.5 → interpolates");
+  assert.ok(preserveRecentForPressure(1, 4, 2) >= 2, "never below floor");
+});

package/src/store/compression.ts

@@ -5,7 +5,7 @@
  *
  *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
  *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
- *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *     node:sqlite replaced PGlite precisely to avoid an async cascade).
  *
  *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
  *     Optional, used for DR-export / large-blob paths where an await is fine.
@@ -32,7 +32,13 @@ import {
   brotliDecompressSync,
   constants as zlibConstants,
 } from "node:zlib";
-import zstd from "@mongodb-js/zstd";
+// zstd is loaded lazily (see compressZstdWithLevel / decompressZstd). It is an
+// OPTIONAL async DR-export dependency: its native addon (`zstd.node`) is not in
+// the npm tarball and may be absent on a clean/allowScripts-blocked install, so
+// a static import here would crash the whole extension at load time. Lazy
+// import keeps the extension loadable even when the binary is missing; the DR
+// path throws a clear error only if it is actually used. (Fix A.)
+// import zstd from "@mongodb-js/zstd";
 
 // --- Versioned format markers ----------------------------------------------
 const MAGIC_HI = 0xec;
@@ -59,6 +65,12 @@ function header(ver: number, tag: number): Buffer {
   return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
 }
 
+/** Clamp a value to the [0, 1] range (pressure bands). */
+function clamp01(n: number): number {
+  if (Number.isNaN(n)) return 0;
+  return n < 0 ? 0 : n > 1 ? 1 : n;
+}
+
 /**
  * Compress synchronously using the best zlib tier for the payload size.
  *
@@ -68,21 +80,36 @@ function header(ver: number, tag: number): Buffer {
  *   4KB–32KB → gzip level 6 (tag 0x02)
  *   > 32 KB  → brotli 4     (tag 0x05)
  *
- * Writes the versioned header so readers disambiguate from legacy blobs.
+ * `pressure` (0–1, optional) escalates the brotli quality for the large tier
+ * when the session is near its context limit — the "variable compression as we
+ * approach the limit" design (Fix E). Low/undefined pressure keeps brotli-4;
+ * high pressure pushes toward brotli-11. Stays fully synchronous (brotli-11 is
+ * sync via brotliCompressSync) so the sync `add()` contract is preserved; zstd
+ * is reserved for the async DR-export path only. Same versioned header/tags for
+ * every pressure, so decompressSmart is unaffected.
  */
-export function compressSmart(data: Buffer): Buffer {
+export function compressSmart(data: Buffer, pressure = 0): Buffer {
+  const p = clamp01(pressure);
   const len = data.length;
   if (len < SIZE_TINY) {
     return Buffer.concat([header(1, TAG_RAW), data]);
   }
   if (len < SIZE_SMALL) {
-    return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+    // Small tier: escalate gzip level 1 → 9 with context pressure (Fix E) so
+    // the "variable compression as we approach the limit" dial bites for
+    // short sessions too, not just the >32KB brotli tier.
+    const level = Math.max(1, Math.min(9, Math.round(1 + 8 * p)));
+    return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level })]);
   }
   if (len < SIZE_MEDIUM) {
-    return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+    // Medium tier: escalate gzip level 6 → 9 with context pressure (Fix E).
+    const level = Math.max(6, Math.min(9, Math.round(6 + 3 * p)));
+    return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level })]);
   }
+  // Large tier: escalate brotli quality 4 → 11 with context pressure (Fix E).
+  const quality = Math.max(4, Math.min(11, Math.round(4 + 7 * p)));
   const compressed = brotliCompressSync(data, {
-    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: quality },
   });
   return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
 }
@@ -164,6 +191,18 @@ const ZSTD_MAGIC_HI = 0x5a; // 'Z'
 const ZSTD_MAGIC_LO = 0x53; // 'S'
 
 async function compressZstdWithLevel(data: Buffer, level: number): Promise<Buffer> {
+  // Lazy import: the native addon may be absent (clean/allowScripts install).
+  // Throws a clear, actionable error instead of a load-time crash.
+  let zstd: typeof import("@mongodb-js/zstd");
+  try {
+    zstd = await import("@mongodb-js/zstd");
+  } catch {
+    throw new Error(
+      "zstd is not available — the @mongodb-js/zstd native addon (zstd.node) " +
+        "was not built. Run the extension's native install step (or allow npm " +
+        "install scripts) to enable DR-export compression.",
+    );
+  }
   const compressed = await zstd.compress(data, level);
   return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
 }
@@ -189,6 +228,8 @@ export async function decompressZstd(buf: Buffer): Promise<Buffer> {
   if (!isZstd(buf)) {
     throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
   }
+  // Lazy import (see compressZstdWithLevel for rationale).
+  const zstd = await import("@mongodb-js/zstd");
   return zstd.decompress(buf.subarray(2));
 }