pi-mega-compact 0.11.3 → 0.11.5

package/dist/src/dedup/raptor/serve-gate.test.js

@@ -1,298 +0,0 @@
-/**
- * serve-gate.test.ts — S25 RAPTOR promotion acceptance tests.
- *
- * Five gates that must ALL pass before the RAPTOR tree is trusted as the live
- * recall surface:
- *   1. Shadow gate (RAPTOR_SHADOW_MODE != false → no RAPTOR merge)
- *   2. Stale tree fallback (builtAt < max checkpoint timestamp → flat)
- *   3. timedOut fallback (extractive-fallback root → flat)
- *   4. Coverage breadth (2-topic fixture, RAPTOR hits both vs flat misses one)
- *   5. p95 latency (200-checkpoint × 20-query median < 100 ms)
- *
- * No network — default extractive summarizer + trigram embedder.
- */
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { VectorStore, vectorList, vectorSearch } from "../../vectorStore.js";
-import { runRaptor, isShadowMode } from "./index.js";
-import { compactSession } from "../../engine.js";
-import { Logger } from "../../log.js";
-import { loadDedupConfig } from "../../config/dedup.js";
-import { listRaptorNodes } from "../../store/sqlite.js";
-import { normalizeSessionId } from "../../store.js";
-/* ------------------------------------------------------------------ helpers */
-let tmpDir;
-let counter = 0;
-beforeEach(() => {
-    tmpDir = mkdtempSync(join(tmpdir(), "mc-gate-"));
-});
-afterEach(() => {
-    rmSync(tmpDir, { recursive: true, force: true });
-});
-function stateDir() {
-    return join(tmpDir, `run-${counter++}`);
-}
-/** Config with dedup tiers disabled so compactSession creates distinct checkpoints. */
-function cfg(overrides) {
-    return {
-        ...loadDedupConfig(),
-        RAPTOR_ENABLED: true,
-        L0_ENABLED: false,
-        L1_ENABLED: false,
-        L2_ENABLED: false,
-        ...overrides,
-    };
-}
-function msg(text, toolName) {
-    return toolName
-        ? { role: "assistant", text, toolName, input: text, output: text }
-        : { role: "user", text };
-}
-function seedSession(store, sid, count, topic, startTs = 1) {
-    for (let i = 1; i <= count; i++) {
-        compactSession({
-            sessionId: sid,
-            messages: [
-                msg(`${topic} checkpoint ${i} with unique content`),
-                msg(`acknowledged ${i}`, "Edit"),
-            ],
-            keepFrom: 2,
-            timestamp: startTs + i,
-        }, store);
-    }
-}
-function buildTree(store, sid) {
-    const nsid = normalizeSessionId(sid);
-    const all = vectorList(store, nsid);
-    const leaves = all.map((cp) => ({
-        id: cp.checkpointId,
-        messages: [],
-        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-        embedding: cp.embedding,
-    }));
-    return runRaptor(leaves, {
-        stateDir: store.stateDir,
-        sessionId: nsid,
-        logger: new Logger(),
-    });
-}
-/* ------------------------------------------------------------------- tests */
-// ─── Gate 1: shadow mode ────────────────────────────────────────────────────
-test("gate 1: shadow mode (default) → search does NOT use RAPTOR merge", () => {
-    const sd = stateDir();
-    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
-    const sid = "shadow";
-    seedSession(s, sid, 5, "shadows");
-    buildTree(s, sid);
-    // Shadow mode is the default when env is not explicitly "false".
-    const orig = process.env.RAPTOR_SHADOW_MODE;
-    process.env.RAPTOR_SHADOW_MODE = "true";
-    try {
-        assert.ok(isShadowMode(), "shadow mode should be active");
-        // Search still returns results (flat fallback). Verify no crash.
-        const hits = vectorSearch(s, sid, "shadows checkpoint", 5);
-        assert.ok(hits.length > 0, "flat search returns results in shadow mode");
-    }
-    finally {
-        if (orig === undefined)
-            delete process.env.RAPTOR_SHADOW_MODE;
-        else
-            process.env.RAPTOR_SHADOW_MODE = orig;
-    }
-});
-// ─── Gate 2: stale tree fallback ────────────────────────────────────────────
-test("gate 2: stale tree (builtAt < newest checkpoint) → serve flat", () => {
-    const sd = stateDir();
-    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
-    const sid = "stale";
-    // Build tree at checkpoints 1..5 (builtAt ≈ now).
-    seedSession(s, sid, 5, "stale topic", 0);
-    buildTree(s, sid);
-    const nodesBefore = listRaptorNodes(sid, sd);
-    assert.ok(nodesBefore.length > 0, "tree built");
-    const treeBuiltAt = nodesBefore[0].builtAt;
-    // Add checkpoint 6 with timestamp > treeBuiltAt → tree is now stale.
-    compactSession({
-        sessionId: sid,
-        messages: [
-            msg("stale topic checkpoint 6 newer than tree"),
-            msg("ok", "Edit"),
-        ],
-        keepFrom: 2,
-        timestamp: treeBuiltAt + 1000,
-    }, s);
-    // Enable live mode so the gate is actually evaluated.
-    const orig = process.env.RAPTOR_SHADOW_MODE;
-    process.env.RAPTOR_SHADOW_MODE = "false";
-    try {
-        // Search should succeed (flat fallback) even though tree is stale.
-        const hits = vectorSearch(s, sid, "stale topic", 3);
-        assert.ok(hits.length > 0, "flat search returns hits when tree is stale");
-    }
-    finally {
-        if (orig === undefined)
-            delete process.env.RAPTOR_SHADOW_MODE;
-        else
-            process.env.RAPTOR_SHADOW_MODE = orig;
-    }
-});
-// ─── Gate 3: timedOut fallback ──────────────────────────────────────────────
-test("gate 3: timedOut extractive-fallback tree → serve flat", () => {
-    const sd = stateDir();
-    // Force extractive fallback by setting a tiny token budget.
-    const s = new VectorStore({
-        dedupSim: 0.9,
-        stateDir: sd,
-        config: cfg({ RAPTOR_MAX_TOKEN_BUDGET_PER_SESSION: 1 }),
-    });
-    const sid = "timedout";
-    seedSession(s, sid, 10, "budget topic", 0);
-    buildTree(s, sid);
-    const nodes = listRaptorNodes(sid, sd);
-    if (nodes.length > 0) {
-        const root = nodes.find((n) => !n.parentId && n.level >= 99);
-        if (root) {
-            // timedOut tree — search should fall back to flat.
-            const orig = process.env.RAPTOR_SHADOW_MODE;
-            process.env.RAPTOR_SHADOW_MODE = "false";
-            try {
-                const hits = vectorSearch(s, sid, "budget topic", 3);
-                assert.ok(hits.length > 0, "flat search returns hits when tree is timedOut");
-            }
-            finally {
-                if (orig === undefined)
-                    delete process.env.RAPTOR_SHADOW_MODE;
-                else
-                    process.env.RAPTOR_SHADOW_MODE = orig;
-            }
-        }
-        else {
-            assert.ok(true, "budget did not trigger extractive fallback (non-deterministic)");
-        }
-    }
-    else {
-        assert.ok(true, "no tree built for tiny-budget session");
-    }
-});
-// ─── Gate 4: coverage breadth ───────────────────────────────────────────────
-test("gate 4: RAPTOR covers both topics vs flat may miss one", () => {
-    const sd = stateDir();
-    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
-    const sid = "breadth";
-    // Two distinct topics — interleaved timestamps.
-    for (let i = 1; i <= 5; i++) {
-        compactSession({
-            sessionId: sid,
-            messages: [
-                msg(`cooking recipe ${i} pasta sauce basil`),
-                msg(`ok ${i}`, "Edit"),
-            ],
-            keepFrom: 2,
-            timestamp: i * 2,
-        }, s);
-        compactSession({
-            sessionId: sid,
-            messages: [
-                msg(`automotive engine ${i} horsepower torque transmission`),
-                msg(`ok ${i}`, "Edit"),
-            ],
-            keepFrom: 2,
-            timestamp: i * 2 + 1,
-        }, s);
-    }
-    buildTree(s, sid);
-    const nodes = listRaptorNodes(sid, sd);
-    assert.ok(nodes.length > 0, "tree built for breadth test");
-    // Enable live mode.
-    const orig = process.env.RAPTOR_SHADOW_MODE;
-    process.env.RAPTOR_SHADOW_MODE = "false";
-    try {
-        const hits = vectorSearch(s, sid, "cooking and automotive", 5);
-        assert.ok(hits.length > 0, "search returns hits for mixed query");
-        // Breadth: at least 2 distinct hits (probabilistic, but with 10
-        // checkpoints across 2 topics, the tree's staged expansion should
-        // surface both).
-        assert.ok(hits.length >= 2, `expected ≥2 diverse hits, got ${hits.length}`);
-    }
-    finally {
-        if (orig === undefined)
-            delete process.env.RAPTOR_SHADOW_MODE;
-        else
-            process.env.RAPTOR_SHADOW_MODE = orig;
-    }
-});
-// ─── Gate 5: p95 latency ────────────────────────────────────────────────────
-test("gate 5: 200-checkpoint × 20-query median latency < 100 ms", () => {
-    const sd = stateDir();
-    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
-    const sid = "latency";
-    // 200 checkpoints across 10 topics.
-    const topics = [
-        "machine learning gradient descent",
-        "quantum computing entanglement",
-        "organic chemistry catalysis",
-        "renaissance art fresco painting",
-        "marine biology coral reefs",
-        "astrophysics dark matter",
-        "ancient rome senate",
-        "music theory counterpoint",
-        "urban planning zoning",
-        "evolutionary biology adaptation",
-    ];
-    for (let i = 1; i <= 200; i++) {
-        const topic = topics[i % topics.length];
-        compactSession({
-            sessionId: sid,
-            messages: [msg(`${topic} checkpoint ${i}`), msg(`ack ${i}`, "Edit")],
-            keepFrom: 2,
-            timestamp: i,
-        }, s);
-    }
-    buildTree(s, sid);
-    // Enable live mode.
-    const orig = process.env.RAPTOR_SHADOW_MODE;
-    process.env.RAPTOR_SHADOW_MODE = "false";
-    try {
-        const queries = [
-            "machine learning optimization",
-            "quantum entanglement physics",
-            "organic chemistry reactions",
-            "renaissance art techniques",
-            "coral reef ecosystems",
-            "dark matter universe",
-            "roman government structure",
-            "music harmony rules",
-            "city planning infrastructure",
-            "evolutionary adaptation species",
-            "gradient descent algorithms",
-            "quantum computing qubits",
-            "chemical catalysis processes",
-            "fresco painting methods",
-            "marine biology biodiversity",
-            "astrophysics observations",
-            "ancient roman history",
-            "music composition theory",
-            "urban development patterns",
-            "biology natural selection",
-        ];
-        const latencies = [];
-        for (const q of queries) {
-            const t0 = Date.now();
-            vectorSearch(s, sid, q, 5);
-            latencies.push(Date.now() - t0);
-        }
-        latencies.sort((a, b) => a - b);
-        const median = latencies[Math.floor(latencies.length / 2)];
-        const p95 = latencies[Math.floor(latencies.length * 0.95)];
-        assert.ok(median < 100, `median latency ${median} ms exceeds 100 ms budget (p95=${p95} ms)`);
-    }
-    finally {
-        if (orig === undefined)
-            delete process.env.RAPTOR_SHADOW_MODE;
-        else
-            process.env.RAPTOR_SHADOW_MODE = orig;
-    }
-});

package/dist/src/dedup/sprint12.test.js

@@ -1,219 +0,0 @@
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { VectorStore, L2_ENABLED, vectorSemDedup, vectorList, vectorSearch } from "../vectorStore.js";
-import { mmrRerank } from "./mmr.js";
-import { topK } from "./topk.js";
-import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
-import { upsertCheckpoint } from "../store/sqlite.js";
-const baseTmp = mkdtempSync(join(tmpdir(), "mc-s12-"));
-let counter = 0;
-function store(opts = {}) {
-    const dir = join(baseTmp, `run-${counter++}`);
-    return new VectorStore({ stateDir: dir, ...opts });
-}
-// --- MMR diversity ---------------------------------------------------------
-test("mmrRerank diversifies: a cluster yields distinct-relevance results", () => {
-    const e = defaultEmbedder();
-    // Three near-identical vectors + one distinct.
-    const v = e.embed("the compiler optimized the parser hot loop");
-    const v2 = e.embed("the compiler optimized the parser hot loops"); // near-dup of v
-    const v3 = e.embed("the compiler optimized the parser hot loop now"); // near-dup of v
-    const vDistinct = e.embed("the database added a covering index for queries");
-    const items = [
-        { item: "a", vector: v, relevance: 0.9 },
-        { item: "b", vector: v2, relevance: 0.88 },
-        { item: "c", vector: v3, relevance: 0.87 },
-        { item: "d", vector: vDistinct, relevance: 0.5 },
-    ];
-    const ranked = mmrRerank(items, 2, 0.5);
-    assert.equal(ranked.length, 2);
-    // The near-dup cluster (a) and the distinct one (d) should both survive.
-    assert.ok(ranked.includes("a"));
-    assert.ok(ranked.includes("d"));
-    assert.ok(!ranked.includes("b") || !ranked.includes("c"));
-});
-test("mmrRerank with lambda=1 is pure relevance ranking", () => {
-    const items = [
-        { item: "low", vector: [1, 0, 0], relevance: 0.1 },
-        { item: "high", vector: [0, 1, 0], relevance: 0.9 },
-    ];
-    const ranked = mmrRerank(items, 2, 1);
-    assert.deepEqual(ranked, ["high", "low"]);
-});
-// --- Heap top-k ------------------------------------------------------------
-test("topK matches brute-force full sort on a fixture", () => {
-    const items = Array.from({ length: 1000 }, (_, i) => ({ item: i, score: Math.sin(i) * 100 + (i % 7) }));
-    for (const k of [1, 3, 10, 50]) {
-        const heap = topK(items, k).map((s) => s.item).sort((a, b) => b - a);
-        const brute = [...items].sort((a, b) => b.score - a.score).slice(0, k).map((s) => s.item).sort((a, b) => b - a);
-        assert.deepEqual(heap, brute, `topK(${k}) should match brute force`);
-    }
-});
-test("topK with k >= n returns all (descending by score)", () => {
-    const items = [{ item: "x", score: 1 }, { item: "y", score: 2 }];
-    assert.deepEqual(topK(items, 5).map((s) => s.item), ["y", "x"]);
-});
-// --- Empty-vector guard ----------------------------------------------------
-test("cosineSimilarity guards empty vector → 0 (no NaN)", () => {
-    assert.equal(cosineSimilarity([], [1, 2, 3]), 0);
-    assert.equal(cosineSimilarity([0, 0, 0], [1, 2, 3]), 0);
-    assert.ok(!Number.isNaN(cosineSimilarity([], [])));
-});
-// --- L2_ENABLED flag -------------------------------------------------------
-test("L2_ENABLED defaults true; search still returns hits", () => {
-    assert.equal(L2_ENABLED, true);
-    const s = store();
-    s.add({ sessionId: "sess_l2", summary: "investigated the parser", regionText: "investigated src/parser.ts and added a tokenizer", timestamp: 1 });
-    const hits = vectorSearch(s, "sess_l2", "src/parser.ts tokenizer", 3);
-    assert.ok(hits.length >= 1);
-});
-test("L2_ENABLED=false skips semantic tier but L0/L1 still work", () => {
-    const s = store({ l2Enabled: false });
-    const r1 = s.add({ sessionId: "sess_l2off", summary: "x", regionText: "the auth module validates the session token", timestamp: 1 });
-    const r2 = s.add({ sessionId: "sess_l2off", summary: "x", regionText: "the auth module validates the session token", timestamp: 2 });
-    assert.equal(r2.deduped, true); // L0 catches exact
-    assert.equal(r1.deduped, false);
-});
-// --- SemDeDup --------------------------------------------------------------
-// Seed a legacy session directly (bypassing add-time dedup tiers) so SemDeDup
-// has a redundant pair to prune — its real job is offline cleanup of data that
-// predates / escaped the online cascade.
-function seed(dir, sessionId, rows) {
-    const e = defaultEmbedder();
-    for (const r of rows) {
-        const cp = {
-            checkpointId: r.id,
-            sessionId,
-            summary: r.text,
-            keyDecisions: [],
-            nextSteps: [],
-            filesModified: [],
-            tokenEstimate: r.tok,
-            regionHash: `r-${r.id}`,
-            embedding: e.embed(r.text),
-            timestamp: 1,
-        };
-        upsertCheckpoint(cp, dir);
-    }
-}
-test("semDedup marks redundant near-identical rows 'removed' and search excludes them", () => {
-    const dir = join(baseTmp, `run-${counter++}`);
-    const s = new VectorStore({ stateDir: dir });
-    seed(dir, "sess_sd", [
-        { id: "chkpt_001", text: "the cache stores parsed ast nodes for fast lookup", tok: 100 },
-        { id: "chkpt_002", text: "the cache stores parsed ast nodes for fast lookup and reuse", tok: 900 },
-    ]);
-    const removed = vectorSemDedup(s, "sess_sd", 0.85);
-    assert.equal(removed, 1);
-    const st = vectorList(s, "sess_sd");
-    const dropped = st.find((c) => c.dedupStatus === "removed");
-    assert.ok(dropped);
-    assert.equal(dropped.checkpointId, "chkpt_001"); // lower tokenEstimate removed
-    // Search excludes the removed row (only one active remains).
-    const hits = vectorSearch(s, "sess_sd", "cache parsed ast nodes", 5);
-    assert.equal(hits.length, 1);
-});
-test("semDedup is idempotent (re-run removes nothing new)", () => {
-    const dir = join(baseTmp, `run-${counter++}`);
-    const s = new VectorStore({ stateDir: dir });
-    seed(dir, "sess_sd2", [
-        { id: "chkpt_001", text: "identical region text for the dedup job now", tok: 100 },
-        { id: "chkpt_002", text: "identical region text for the dedup job right now", tok: 200 },
-    ]);
-    const first = vectorSemDedup(s, "sess_sd2", 0.85);
-    const second = vectorSemDedup(s, "sess_sd2", 0.85);
-    assert.equal(first, 1);
-    assert.equal(second, 0);
-});
-// --- HttpEmbedder (BYO localhost backend) ----------------------------------
-// Hermetic: a self-test server returns a deterministic embedding. The server
-// runs in an INDEPENDENT child process (its own event loop) so that when
-// HttpEmbedder.embed() blocks the *parent* via spawnSync, the server can still
-// accept the connection — hosting it in-process would deadlock.
-import { spawn } from "node:child_process";
-const ECHO_SERVER = String.raw `
-import { createServer } from "node:http";
-const s = createServer((req, res) => {
-  let b = "";
-  req.on("data", (c) => (b += c));
-  req.on("end", () => {
-    const input = JSON.parse(b).input || [""];
-    const text = (input[0] || "").toLowerCase().replace(/\s+/g, " ");
-    const vec = new Array(8).fill(0);
-    for (const w of text.split(" ")) {
-      let h = 0x811c9dc5;
-      for (let i = 0; i < w.length; i++) { h ^= w.charCodeAt(i); h = Math.imul(h, 0x01000193); }
-      vec[h % 8] += 1;
-    }
-    res.setHeader("content-type", "application/json");
-    res.end(JSON.stringify({ data: [{ embedding: vec }] }));
-  });
-});
-s.listen(0, "127.0.0.1", () => process.stdout.write(String(s.address().port)));
-`;
-// A simpler shape-only server: always returns { data: [[0.1,0.2,0.3]] }.
-const DATA_ARR_SERVER = String.raw `
-import { createServer } from "node:http";
-const s = createServer((_req, res) => {
-  res.setHeader("content-type", "application/json");
-  res.end(JSON.stringify({ data: [[0.1, 0.2, 0.3]] }));
-});
-s.listen(0, "127.0.0.1", () => process.stdout.write(String(s.address().port)));
-`;
-/** Spawn an independent echo server; resolves with its url once the port is up. */
-function startEchoServer(shape) {
-    const script = shape === "data-arr" ? DATA_ARR_SERVER : ECHO_SERVER;
-    const proc = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "ignore"] });
-    return new Promise((resolve, reject) => {
-        let buf = "";
-        proc.stdout.on("data", (d) => {
-            buf += d.toString();
-            const m = buf.match(/(\d+)/);
-            if (m)
-                resolve({ url: `http://127.0.0.1:${m[1]}`, proc });
-        });
-        proc.on("error", reject);
-        setTimeout(() => reject(new Error("echo server did not start")), 5000);
-    });
-}
-test("HttpEmbedder parses OpenAI-style response and resolves dim", async () => {
-    const { url, proc } = await startEchoServer();
-    try {
-        const { HttpEmbedder } = await import("../httpEmbedder.js");
-        const emb = new HttpEmbedder({ url });
-        assert.equal(emb.dim, 0); // unknown until first embed
-        const v = emb.embed("the parser optimized the hot loop");
-        assert.equal(v.length, 8); // echo server returns 8-dim
-        assert.equal(emb.dim, 8); // resolved after first call
-        const v2 = emb.embed("the parser optimized the hot loop");
-        assert.deepEqual(v2, v); // deterministic
-    }
-    finally {
-        proc.kill();
-    }
-});
-test("HttpEmbedder tolerant parser: accepts { data: [[...]] } shape", async () => {
-    const { url, proc } = await startEchoServer("data-arr");
-    try {
-        const { HttpEmbedder } = await import("../httpEmbedder.js");
-        const emb = new HttpEmbedder({ url });
-        // embed() L2-normalizes; verify the { data: [[...]] } shape parsed (dim 3)
-        // and the returned vector is the unit-normalized form of [0.1, 0.2, 0.3].
-        const v = emb.embed("x");
-        assert.equal(v.length, 3);
-        const norm = Math.sqrt(0.1 ** 2 + 0.2 ** 2 + 0.3 ** 2);
-        assert.ok(Math.abs(v[0] - 0.1 / norm) < 1e-9);
-        assert.ok(Math.abs(v[1] - 0.2 / norm) < 1e-9);
-        assert.ok(Math.abs(v[2] - 0.3 / norm) < 1e-9);
-    }
-    finally {
-        proc.kill();
-    }
-});
-// --- cleanup ---------------------------------------------------------------
-test("Sprint 12 cleanup", () => {
-    rmSync(baseTmp, { recursive: true, force: true });
-});