pi-mega-compact 0.8.24 → 0.8.26

package/dist/src/dedup/raptor/serve-gate.test.js

@@ -0,0 +1,298 @@
+/**
+ * serve-gate.test.ts — S25 RAPTOR promotion acceptance tests.
+ *
+ * Five gates that must ALL pass before the RAPTOR tree is trusted as the live
+ * recall surface:
+ *   1. Shadow gate (RAPTOR_SHADOW_MODE != false → no RAPTOR merge)
+ *   2. Stale tree fallback (builtAt < max checkpoint timestamp → flat)
+ *   3. timedOut fallback (extractive-fallback root → flat)
+ *   4. Coverage breadth (2-topic fixture, RAPTOR hits both vs flat misses one)
+ *   5. p95 latency (200-checkpoint × 20-query median < 100 ms)
+ *
+ * No network — default extractive summarizer + trigram embedder.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore, vectorList, vectorSearch } from "../../vectorStore.js";
+import { runRaptor, isShadowMode } from "./index.js";
+import { compactSession } from "../../engine.js";
+import { Logger } from "../../log.js";
+import { loadDedupConfig } from "../../config/dedup.js";
+import { listRaptorNodes } from "../../store/sqlite.js";
+import { normalizeSessionId } from "../../store.js";
+/* ------------------------------------------------------------------ helpers */
+let tmpDir;
+let counter = 0;
+beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "mc-gate-"));
+});
+afterEach(() => {
+    rmSync(tmpDir, { recursive: true, force: true });
+});
+function stateDir() {
+    return join(tmpDir, `run-${counter++}`);
+}
+/** Config with dedup tiers disabled so compactSession creates distinct checkpoints. */
+function cfg(overrides) {
+    return {
+        ...loadDedupConfig(),
+        RAPTOR_ENABLED: true,
+        L0_ENABLED: false,
+        L1_ENABLED: false,
+        L2_ENABLED: false,
+        ...overrides,
+    };
+}
+function msg(text, toolName) {
+    return toolName
+        ? { role: "assistant", text, toolName, input: text, output: text }
+        : { role: "user", text };
+}
+function seedSession(store, sid, count, topic, startTs = 1) {
+    for (let i = 1; i <= count; i++) {
+        compactSession({
+            sessionId: sid,
+            messages: [
+                msg(`${topic} checkpoint ${i} with unique content`),
+                msg(`acknowledged ${i}`, "Edit"),
+            ],
+            keepFrom: 2,
+            timestamp: startTs + i,
+        }, store);
+    }
+}
+function buildTree(store, sid) {
+    const nsid = normalizeSessionId(sid);
+    const all = vectorList(store, nsid);
+    const leaves = all.map((cp) => ({
+        id: cp.checkpointId,
+        messages: [],
+        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+        embedding: cp.embedding,
+    }));
+    return runRaptor(leaves, {
+        stateDir: store.stateDir,
+        sessionId: nsid,
+        logger: new Logger(),
+    });
+}
+/* ------------------------------------------------------------------- tests */
+// ─── Gate 1: shadow mode ────────────────────────────────────────────────────
+test("gate 1: shadow mode (default) → search does NOT use RAPTOR merge", () => {
+    const sd = stateDir();
+    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
+    const sid = "shadow";
+    seedSession(s, sid, 5, "shadows");
+    buildTree(s, sid);
+    // Shadow mode is the default when env is not explicitly "false".
+    const orig = process.env.RAPTOR_SHADOW_MODE;
+    process.env.RAPTOR_SHADOW_MODE = "true";
+    try {
+        assert.ok(isShadowMode(), "shadow mode should be active");
+        // Search still returns results (flat fallback). Verify no crash.
+        const hits = vectorSearch(s, sid, "shadows checkpoint", 5);
+        assert.ok(hits.length > 0, "flat search returns results in shadow mode");
+    }
+    finally {
+        if (orig === undefined)
+            delete process.env.RAPTOR_SHADOW_MODE;
+        else
+            process.env.RAPTOR_SHADOW_MODE = orig;
+    }
+});
+// ─── Gate 2: stale tree fallback ────────────────────────────────────────────
+test("gate 2: stale tree (builtAt < newest checkpoint) → serve flat", () => {
+    const sd = stateDir();
+    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
+    const sid = "stale";
+    // Build tree at checkpoints 1..5 (builtAt ≈ now).
+    seedSession(s, sid, 5, "stale topic", 0);
+    buildTree(s, sid);
+    const nodesBefore = listRaptorNodes(sid, sd);
+    assert.ok(nodesBefore.length > 0, "tree built");
+    const treeBuiltAt = nodesBefore[0].builtAt;
+    // Add checkpoint 6 with timestamp > treeBuiltAt → tree is now stale.
+    compactSession({
+        sessionId: sid,
+        messages: [
+            msg("stale topic checkpoint 6 newer than tree"),
+            msg("ok", "Edit"),
+        ],
+        keepFrom: 2,
+        timestamp: treeBuiltAt + 1000,
+    }, s);
+    // Enable live mode so the gate is actually evaluated.
+    const orig = process.env.RAPTOR_SHADOW_MODE;
+    process.env.RAPTOR_SHADOW_MODE = "false";
+    try {
+        // Search should succeed (flat fallback) even though tree is stale.
+        const hits = vectorSearch(s, sid, "stale topic", 3);
+        assert.ok(hits.length > 0, "flat search returns hits when tree is stale");
+    }
+    finally {
+        if (orig === undefined)
+            delete process.env.RAPTOR_SHADOW_MODE;
+        else
+            process.env.RAPTOR_SHADOW_MODE = orig;
+    }
+});
+// ─── Gate 3: timedOut fallback ──────────────────────────────────────────────
+test("gate 3: timedOut extractive-fallback tree → serve flat", () => {
+    const sd = stateDir();
+    // Force extractive fallback by setting a tiny token budget.
+    const s = new VectorStore({
+        dedupSim: 0.9,
+        stateDir: sd,
+        config: cfg({ RAPTOR_MAX_TOKEN_BUDGET_PER_SESSION: 1 }),
+    });
+    const sid = "timedout";
+    seedSession(s, sid, 10, "budget topic", 0);
+    buildTree(s, sid);
+    const nodes = listRaptorNodes(sid, sd);
+    if (nodes.length > 0) {
+        const root = nodes.find((n) => !n.parentId && n.level >= 99);
+        if (root) {
+            // timedOut tree — search should fall back to flat.
+            const orig = process.env.RAPTOR_SHADOW_MODE;
+            process.env.RAPTOR_SHADOW_MODE = "false";
+            try {
+                const hits = vectorSearch(s, sid, "budget topic", 3);
+                assert.ok(hits.length > 0, "flat search returns hits when tree is timedOut");
+            }
+            finally {
+                if (orig === undefined)
+                    delete process.env.RAPTOR_SHADOW_MODE;
+                else
+                    process.env.RAPTOR_SHADOW_MODE = orig;
+            }
+        }
+        else {
+            assert.ok(true, "budget did not trigger extractive fallback (non-deterministic)");
+        }
+    }
+    else {
+        assert.ok(true, "no tree built for tiny-budget session");
+    }
+});
+// ─── Gate 4: coverage breadth ───────────────────────────────────────────────
+test("gate 4: RAPTOR covers both topics vs flat may miss one", () => {
+    const sd = stateDir();
+    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
+    const sid = "breadth";
+    // Two distinct topics — interleaved timestamps.
+    for (let i = 1; i <= 5; i++) {
+        compactSession({
+            sessionId: sid,
+            messages: [
+                msg(`cooking recipe ${i} pasta sauce basil`),
+                msg(`ok ${i}`, "Edit"),
+            ],
+            keepFrom: 2,
+            timestamp: i * 2,
+        }, s);
+        compactSession({
+            sessionId: sid,
+            messages: [
+                msg(`automotive engine ${i} horsepower torque transmission`),
+                msg(`ok ${i}`, "Edit"),
+            ],
+            keepFrom: 2,
+            timestamp: i * 2 + 1,
+        }, s);
+    }
+    buildTree(s, sid);
+    const nodes = listRaptorNodes(sid, sd);
+    assert.ok(nodes.length > 0, "tree built for breadth test");
+    // Enable live mode.
+    const orig = process.env.RAPTOR_SHADOW_MODE;
+    process.env.RAPTOR_SHADOW_MODE = "false";
+    try {
+        const hits = vectorSearch(s, sid, "cooking and automotive", 5);
+        assert.ok(hits.length > 0, "search returns hits for mixed query");
+        // Breadth: at least 2 distinct hits (probabilistic, but with 10
+        // checkpoints across 2 topics, the tree's staged expansion should
+        // surface both).
+        assert.ok(hits.length >= 2, `expected ≥2 diverse hits, got ${hits.length}`);
+    }
+    finally {
+        if (orig === undefined)
+            delete process.env.RAPTOR_SHADOW_MODE;
+        else
+            process.env.RAPTOR_SHADOW_MODE = orig;
+    }
+});
+// ─── Gate 5: p95 latency ────────────────────────────────────────────────────
+test("gate 5: 200-checkpoint × 20-query median latency < 100 ms", () => {
+    const sd = stateDir();
+    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
+    const sid = "latency";
+    // 200 checkpoints across 10 topics.
+    const topics = [
+        "machine learning gradient descent",
+        "quantum computing entanglement",
+        "organic chemistry catalysis",
+        "renaissance art fresco painting",
+        "marine biology coral reefs",
+        "astrophysics dark matter",
+        "ancient rome senate",
+        "music theory counterpoint",
+        "urban planning zoning",
+        "evolutionary biology adaptation",
+    ];
+    for (let i = 1; i <= 200; i++) {
+        const topic = topics[i % topics.length];
+        compactSession({
+            sessionId: sid,
+            messages: [msg(`${topic} checkpoint ${i}`), msg(`ack ${i}`, "Edit")],
+            keepFrom: 2,
+            timestamp: i,
+        }, s);
+    }
+    buildTree(s, sid);
+    // Enable live mode.
+    const orig = process.env.RAPTOR_SHADOW_MODE;
+    process.env.RAPTOR_SHADOW_MODE = "false";
+    try {
+        const queries = [
+            "machine learning optimization",
+            "quantum entanglement physics",
+            "organic chemistry reactions",
+            "renaissance art techniques",
+            "coral reef ecosystems",
+            "dark matter universe",
+            "roman government structure",
+            "music harmony rules",
+            "city planning infrastructure",
+            "evolutionary adaptation species",
+            "gradient descent algorithms",
+            "quantum computing qubits",
+            "chemical catalysis processes",
+            "fresco painting methods",
+            "marine biology biodiversity",
+            "astrophysics observations",
+            "ancient roman history",
+            "music composition theory",
+            "urban development patterns",
+            "biology natural selection",
+        ];
+        const latencies = [];
+        for (const q of queries) {
+            const t0 = Date.now();
+            vectorSearch(s, sid, q, 5);
+            latencies.push(Date.now() - t0);
+        }
+        latencies.sort((a, b) => a - b);
+        const median = latencies[Math.floor(latencies.length / 2)];
+        const p95 = latencies[Math.floor(latencies.length * 0.95)];
+        assert.ok(median < 100, `median latency ${median} ms exceeds 100 ms budget (p95=${p95} ms)`);
+    }
+    finally {
+        if (orig === undefined)
+            delete process.env.RAPTOR_SHADOW_MODE;
+        else
+            process.env.RAPTOR_SHADOW_MODE = orig;
+    }
+});

package/dist/src/dedup/raptor/summarizer.js

@@ -65,6 +65,7 @@ function ollamaSummarize(messages, ollama) {
   `;
     const res = spawnSync(process.execPath, ["-e", WORKER], {
         encoding: "utf8",
+        timeout: 30_000,
         env: { ...process.env, R_URL: ollama.url, R_MODEL: ollama.model, R_PROMPT: prompt },
     });
     let parsed = { ok: false, error: "no response" };

package/dist/src/dedup/raptor/tree.js

@@ -59,6 +59,7 @@ export function buildRaptorTree(leaves, opts) {
     if (leaves.length < 10) {
         const item = {
             id: "root",
+            parentId: null,
             embedding: meanVector(leaves.map((l) => l.embedding)),
             leafIds: leaves.map((l) => l.id),
             messages: leaves.flatMap((l) => l.messages),
@@ -81,6 +82,7 @@ export function buildRaptorTree(leaves, opts) {
     }
     let currentLevel = leaves.map((l) => ({
         id: l.id,
+        parentId: null,
         embedding: l.embedding,
         leafIds: [l.id],
         messages: l.messages,
@@ -96,6 +98,7 @@ export function buildRaptorTree(leaves, opts) {
         if (currentLevel.length <= clustersPerLevel) {
             const merged = {
                 id: "merge",
+                parentId: null,
                 embedding: meanVector(currentLevel.map((c) => c.embedding)),
                 leafIds: currentLevel.flatMap((c) => c.leafIds),
                 messages: currentLevel.flatMap((c) => c.messages),
@@ -114,6 +117,15 @@ export function buildRaptorTree(leaves, opts) {
                 qualityMarker,
                 tokenEstimate,
             });
+            // Populate parentId for the internal nodes being merged into this root.
+            // currentLevel items with ids in `nodes` are internal summary nodes (level
+            // >= 1); raw leaf ids are not in `nodes` and are correctly skipped.
+            for (const c of currentLevel) {
+                const child = nodes.get(c.id);
+                if (child && child.id !== rootId && child.parentId === null) {
+                    child.parentId = rootId;
+                }
+            }
             return { nodes, rootId, levels: level + 2, timedOut: false };
         }
         const k = Math.max(1, Math.min(clustersPerLevel, currentLevel.length));
@@ -127,6 +139,7 @@ export function buildRaptorTree(leaves, opts) {
                 continue;
             const merged = {
                 id: nextId(level + 1, g),
+                parentId: null,
                 embedding: clustered.centroids[g],
                 leafIds: group.flatMap((c) => c.leafIds),
                 messages: group.flatMap((c) => c.messages),
@@ -149,9 +162,10 @@ export function buildRaptorTree(leaves, opts) {
         level++;
     }
     const root = currentLevel[0];
+    const rootId = root ? root.id : null;
     return {
         nodes,
-        rootId: root ? root.id : null,
+        rootId,
         levels: level + 1,
         timedOut: false,
     };
@@ -173,5 +187,5 @@ function extractiveFallbackRoot(leaves, nodes, nextId) {
         qualityMarker: "low",
         tokenEstimate: summary.tokenEstimate,
     });
-    return { nodes, rootId, levels: 2, timedOut: true };
+    return { nodes, rootId, levels: 100, timedOut: true };
 }

package/dist/src/engine.js

@@ -36,7 +36,10 @@ export function setDefaultStore(store) {
  * when the compactable slice is empty.
  */
 export function compactSession(input, store = getDefaultStore()) {
-    const keepFrom = input.keepFrom ?? input.messages.length;
+    // F6: clamp keepFrom defensively. Upstream (the extension adapter) already
+    // clamps, but a bad keepFrom (negative or > length) would produce a misleading
+    // drop range or an empty compactable slice; belt-and-braces, clamp here too.
+    const keepFrom = Math.max(0, Math.min(input.keepFrom ?? input.messages.length, input.messages.length));
     const compactable = input.messages.slice(0, keepFrom);
     const compactedFrom = keepFrom;
     if (compactable.length === 0) {
@@ -48,6 +51,8 @@ export function compactSession(input, store = getDefaultStore()) {
             tokenEstimate: 0,
             filesModified: [],
             originalTokenEstimate: 0,
+            keepTokenEstimate: 0,
+            supersedeTokenSavings: 0,
             compactedFrom,
         };
     }
@@ -82,12 +87,21 @@ export function compactSession(input, store = getDefaultStore()) {
     }
     // Honest "tokens saved" accounting:
     //  - originalTokenEstimate = the dropped region's token count (what context
-    //    held before compaction) = the compacted slice's tokens.
+    //    held before compaction) = the compacted slice's tokens. Computed over the
+    //    FULL compactable slice (incl. superseded) so the dedup branch books the
+    //    whole region; this is the value persisted into the checkpoint record.
+    //  - keepTokenEstimate (F5) = the filtered keep set's tokens (post-supersede).
+    //    The honest base for the stored-vs-original compaction delta: the pure
+    //    compaction savings = keepTokenEstimate − storedTokens, and the supersede
+    //    savings = originalTokenEstimate − keepTokenEstimate. Without this, the
+    //    supersede savings get booked as compaction savings.
     //  - storedTokens = the persisted summary's token count, computed from the
     //    actual summary string so it's honest for BOTH the extractive and legacy
     //    COLLAPSE paths (the legacy path's fallback estimateSessionTokens is the
     //    *original* size, not the stored size).
     const originalTokenEstimate = estimateSessionTokens(compactable);
+    const keepTokenEstimate = estimateSessionTokens(keep);
+    const supersedeTokenSavings = Math.max(0, originalTokenEstimate - keepTokenEstimate);
     const storedTokens = estimateBlockTokens(summary);
     // Region text = the compacted slice, used for dedup + embedding.
     const regionText = input.regionText ?? keep.map((m) => m.text).join("\n");
@@ -116,6 +130,8 @@ export function compactSession(input, store = getDefaultStore()) {
         tokenEstimate: storedTokens,
         filesModified,
         originalTokenEstimate,
+        keepTokenEstimate,
+        supersedeTokenSavings,
         compactedFrom,
     };
 }

package/dist/src/httpEmbedder.js

@@ -26,15 +26,105 @@
  */
 import { l2Normalize } from "./embedder.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
-/** Read + validate the localhost embeddings config from the environment. */
+import { isIP } from "node:net";
+// Inline worker script: resolves a hostname via dns.lookup in a child process
+// (dns.lookup is callback-async; the child has its own event loop). Used to
+// verify that a hostname in the embedding URL resolves to loopback ONLY.
+const DNS_WORKER = String.raw `
+const { lookup } = await import("node:dns");
+const { promisify } = await import("node:util");
+const pLookup = promisify(lookup);
+try {
+  const result = await pLookup(process.env.MC_DNS_HOST, { all: true });
+  process.stdout.write(JSON.stringify({ addresses: result.map((a) => a.address) }));
+} catch (e) {
+  process.stdout.write(JSON.stringify({ error: String((e && e.message) || e) }));
+}
+`;
+/** True if the IP string is a loopback address (127.x.x.x for IPv4, ::1 for IPv6). */
+function isLoopbackIP(ip) {
+    const type = isIP(ip);
+    if (type === 4)
+        return ip.startsWith("127.");
+    if (type === 6)
+        return ip === "::1";
+    return false;
+}
+/** Resolve a hostname via dns.lookup (in a child process) and verify EVERY
+ *  returned address is loopback. Rejects on any resolution failure, empty
+ *  result, or non-loopback answer. Fails closed: returns false on any error. */
+function hostnameResolvesToLoopback(hostname) {
+    const res = spawnSync(process.execPath, ["-e", DNS_WORKER], {
+        encoding: "utf8",
+        env: { ...process.env, MC_DNS_HOST: hostname },
+        timeout: 5000,
+    });
+    if (res.error || typeof res.stdout !== "string" || res.stdout.length === 0) {
+        return false;
+    }
+    try {
+        const parsed = JSON.parse(res.stdout);
+        if (parsed.error)
+            return false;
+        if (!Array.isArray(parsed.addresses) || parsed.addresses.length === 0)
+            return false;
+        return parsed.addresses.every((a) => isLoopbackIP(a));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Read + validate the localhost embeddings config from the environment.
+ *
+ * Security: the URL is parsed with `new URL()` (not regex) to prevent userinfo
+ * bypass (e.g. a URL like localhost:8080@evil.com/ masks evil.com as the real
+ * host). The hostname must be a loopback address:
+ *  - Literal IPv4: 127.x.x.x (the full 127.0.0.0/8 range)
+ *  - Literal IPv6: ::1
+ *  - Hostname: resolved via dns.lookup; ALL returned addresses must be loopback
+ * Credentials in the URL (user:pass@) are rejected. Non-loopback literal IPs are
+ * rejected. Any resolution failure or non-loopback DNS answer is rejected.
+ *
+ * Fails CLOSED: unset/invalid/non-loopback URL → returns null → caller falls
+ * back to the local TrigramEmbedder. Never throws — a misconfigured URL must
+ * not crash the extension or silently connect to a remote endpoint.
+ */
 export function embeddingConfigFromEnv() {
     const url = process.env.MEGACOMPACT_EMBEDDING_URL;
     if (!url)
         return null;
-    if (!/^https?:\/\/localhost[:/]/.test(url) && !/^https?:\/\/127\.0\.0\.1[:/]/.test(url)) {
-        // Only loopback is permitted — a remote host would violate PREVENT-PI-004.
-        throw new Error(`MEGACOMPACT_EMBEDDING_URL must be a localhost/127.0.0.1 endpoint (got ${url}). ` +
-            `Remote embedding endpoints are not allowed (PREVENT-PI-004).`);
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        console.warn(`MEGACOMPACT_EMBEDDING_URL is not a valid URL: ${url} — falling back to default embedder (PREVENT-PI-004)`);
+        return null;
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        console.warn(`MEGACOMPACT_EMBEDDING_URL must use http or https scheme (got ${parsed.protocol}) — falling back to default embedder (PREVENT-PI-004)`);
+        return null;
+    }
+    // Reject credentials in the URL (user:pass@) — they can mask a non-loopback host.
+    if (parsed.username || parsed.password) {
+        console.warn(`MEGACOMPACT_EMBEDDING_URL must not contain credentials (user:pass@) — falling back to default embedder (PREVENT-PI-004)`);
+        return null;
+    }
+    const hostname = parsed.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
+    if (isIP(hostname)) {
+        // Literal IP — must be loopback.
+        if (!isLoopbackIP(hostname)) {
+            console.warn(`MEGACOMPACT_EMBEDDING_URL must be a loopback IP (got ${hostname}) — falling back to default embedder (PREVENT-PI-004)`);
+            return null;
+        }
+    }
+    else {
+        // Hostname — resolve via dns.lookup and require ALL addresses loopback.
+        if (!hostnameResolvesToLoopback(hostname)) {
+            console.warn(`MEGACOMPACT_EMBEDDING_URL hostname "${hostname}" does not resolve to loopback — falling back to default embedder (PREVENT-PI-004)`);
+            return null;
+        }
     }
     const headers = {};
     if (process.env.MEGACOMPACT_EMBEDDING_HEADERS) {
@@ -42,7 +132,7 @@ export function embeddingConfigFromEnv() {
             Object.assign(headers, JSON.parse(process.env.MEGACOMPACT_EMBEDDING_HEADERS));
         }
         catch {
-            throw new Error("MEGACOMPACT_EMBEDDING_HEADERS must be valid JSON");
+            console.warn("MEGACOMPACT_EMBEDDING_HEADERS must be valid JSON — ignoring headers");
         }
     }
     const dim = process.env.MEGACOMPACT_EMBEDDING_DIM