pi-mega-compact 0.11.4 → 0.11.6

package/dist/src/dedup/raptor/buildHistory.test.js

@@ -1,292 +0,0 @@
-/**
- * buildHistory.test.ts — S42D RAPTOR build history + freshness tests.
- *
- * No network. Real stores with temp state dirs.
- */
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { openStore } from "../../store/sqlite/utils.js";
-import { insertBuildHistory, getLatestBuild, listBuildHistory, computeCoherenceScore, isRaptorTreeFresh, clearBuildHistory, } from "./buildHistory.js";
-import { runRaptor } from "./index.js";
-import { VectorStore, vectorList } from "../../vectorStore.js";
-import { compactSession } from "../../engine.js";
-import { loadDedupConfig } from "../../config/dedup.js";
-import { normalizeSessionId } from "../../store.js";
-import { Logger } from "../../log.js";
-let tmpDir;
-let counter = 0;
-beforeEach(() => {
-    tmpDir = mkdtempSync(join(tmpdir(), "mc-bh-"));
-});
-afterEach(() => {
-    rmSync(tmpDir, { recursive: true, force: true });
-});
-function stateDir() {
-    return join(tmpDir, `run-${counter++}`);
-}
-function cfg(overrides) {
-    return {
-        ...loadDedupConfig(),
-        RAPTOR_ENABLED: true,
-        L0_ENABLED: false,
-        L1_ENABLED: false,
-        L2_ENABLED: false,
-        ...overrides,
-    };
-}
-function msg(text, toolName) {
-    return toolName
-        ? { role: "assistant", text, toolName, input: text, output: text }
-        : { role: "user", text };
-}
-// ─── 1. Build-history rows are created with correct metadata ──────────────
-test("S42D-1: insertBuildHistory + getLatestBuild round-trip", () => {
-    const sd = stateDir();
-    openStore(sd); // initialize schema
-    const ts = Date.now();
-    const id = insertBuildHistory({
-        sessionId: "sess-foo",
-        stateDir: sd,
-        startedAt: ts - 500,
-        completedAt: ts,
-        nodeCount: 6,
-        leafCount: 10,
-        depth: 3,
-        configJson: '{"budgetMs":5000}',
-        coherenceScore: 0.82,
-        timedOut: false,
-    }, sd);
-    assert.ok(id, "build id returned");
-    const latest = getLatestBuild("sess-foo", sd);
-    assert.ok(latest, "latest build found");
-    assert.equal(latest.sessionId, normalizeSessionId("sess-foo"));
-    assert.equal(latest.nodeCount, 6);
-    assert.equal(latest.leafCount, 10);
-    assert.equal(latest.depth, 3);
-    assert.equal(latest.coherenceScore, 0.82);
-    assert.equal(latest.timedOut, false);
-    assert.equal(latest.configJson, '{"budgetMs":5000}');
-    // getLatest returns the newest by completed_at
-    insertBuildHistory({
-        sessionId: "sess-foo",
-        stateDir: sd,
-        startedAt: ts + 1000,
-        completedAt: ts + 2000,
-        nodeCount: 7,
-        leafCount: 11,
-        depth: 3,
-        configJson: "{}",
-        coherenceScore: null,
-        timedOut: true,
-    }, sd);
-    const newest = getLatestBuild("sess-foo", sd);
-    assert.equal(newest.nodeCount, 7);
-    assert.equal(newest.timedOut, true);
-    assert.equal(newest.coherenceScore, null);
-    // listBuildHistory returns all, newest first
-    const all = listBuildHistory("sess-foo", sd);
-    assert.equal(all.length, 2);
-    assert.equal(all[0].nodeCount, 7); // newest first
-    // No history for an unknown session
-    assert.equal(getLatestBuild("sess-none", sd), null);
-    assert.deepEqual(listBuildHistory("sess-none", sd), []);
-    // clearBuildHistory removes rows
-    clearBuildHistory("sess-foo", sd);
-    assert.equal(getLatestBuild("sess-foo", sd), null);
-});
-// ─── 2. Coherence score: 1.0 for identical, lower for diverse ───────────────
-test("S42D-2: computeCoherenceScore is 1.0 for identical embeddings, lower for diverse", () => {
-    const identical = {
-        rootId: "root",
-        nodes: new Map([
-            ["root", { id: "root", level: 0, parentId: null, children: ["c1", "c2"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 2 }],
-            ["c1", { id: "c1", level: 1, parentId: "root", children: [], summary: "c1", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
-            ["c2", { id: "c2", level: 1, parentId: "root", children: [], summary: "c2", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
-        ]),
-        levels: 2,
-        timedOut: false,
-    };
-    assert.equal(computeCoherenceScore(identical), 1.0);
-    // Diverse: orthogonal children → cosine 0
-    const diverse = {
-        rootId: "root",
-        nodes: new Map([
-            ["root", { id: "root", level: 0, parentId: null, children: ["c1", "c2"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 2 }],
-            ["c1", { id: "c1", level: 1, parentId: "root", children: [], summary: "c1", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
-            ["c2", { id: "c2", level: 1, parentId: "root", children: [], summary: "c2", embedding: [0, 1], qualityMarker: "low", tokenEstimate: 1 }],
-        ]),
-        levels: 2,
-        timedOut: false,
-    };
-    const diverseScore = computeCoherenceScore(diverse);
-    assert.ok(diverseScore < 1.0, `diverse score < 1.0 (got ${diverseScore})`);
-    assert.ok(diverseScore >= 0, "score is non-negative");
-    // No internal nodes with >=2 children → 0
-    const single = {
-        rootId: "root",
-        nodes: new Map([
-            ["root", { id: "root", level: 0, parentId: null, children: ["c1"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
-            ["c1", { id: "c1", level: 1, parentId: "root", children: [], summary: "c1", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
-        ]),
-        levels: 2,
-        timedOut: false,
-    };
-    assert.equal(computeCoherenceScore(single), 0);
-});
-// ─── 2b. Coherence with leaf embeddings (the real-tree shape) ──────────────
-test("S42D-2b: computeCoherenceScore uses leafEmbeddings for leaf children (real-tree shape)", () => {
-    // Root whose children are raw leaves (the common collapse-path shape).
-    // Without leafEmbeddings, leaves fall back to the root centroid → score 1.0
-    // (degenerate). With true leaf embeddings, diverse leaves score < 1.0.
-    const tree = {
-        rootId: "root",
-        nodes: new Map([
-            ["root", { id: "root", level: 0, parentId: null, children: ["l1", "l2"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 2 }],
-        ]),
-        levels: 1,
-        timedOut: false,
-    };
-    // No leaf map → both leaves use root centroid [1,0] → cosine 1.0 (degenerate).
-    assert.equal(computeCoherenceScore(tree), 1.0);
-    // Leaf map with orthogonal leaves → cosine 0 (truly diverse).
-    const leafMap = new Map([
-        ["l1", [1, 0]],
-        ["l2", [0, 1]],
-    ]);
-    const score = computeCoherenceScore(tree, leafMap);
-    assert.ok(score < 1.0, `diverse leaves score < 1.0 (got ${score})`);
-    assert.ok(score >= 0, "score non-negative");
-    // Identical leaves via the map → 1.0
-    const leafMapIdentical = new Map([
-        ["l1", [1, 0]],
-        ["l2", [1, 0]],
-    ]);
-    assert.equal(computeCoherenceScore(tree, leafMapIdentical), 1.0);
-});
-// ─── 3. Freshness check: recent + stable → fresh ────────────────────────────
-test("S42D-3: isRaptorTreeFresh true for recent build + stable count", () => {
-    const sd = stateDir();
-    openStore(sd);
-    insertBuildHistory({
-        sessionId: "sess-fresh",
-        stateDir: sd,
-        startedAt: Date.now() - 1000,
-        completedAt: Date.now() - 500, // < 4h old
-        nodeCount: 6,
-        leafCount: 10,
-        depth: 3,
-        configJson: "{}",
-        coherenceScore: 0.5,
-        timedOut: false,
-    }, sd);
-    // 10 checkpoints, latest had 10 → within 20% → fresh
-    assert.equal(isRaptorTreeFresh("sess-fresh", sd, 4, 10), true);
-    // 11 checkpoints → 10% change → still fresh
-    assert.equal(isRaptorTreeFresh("sess-fresh", sd, 4, 11), true);
-    // 9 checkpoints → 10% change → fresh
-    assert.equal(isRaptorTreeFresh("sess-fresh", sd, 4, 9), true);
-});
-// ─── 4. Freshness check: stale build → not fresh ────────────────────────────
-test("S42D-4: isRaptorTreeFresh false for stale build (>4h old)", () => {
-    const sd = stateDir();
-    openStore(sd);
-    const oldTs = Date.now() - 5 * 3_600_000; // 5h ago
-    insertBuildHistory({
-        sessionId: "sess-stale",
-        stateDir: sd,
-        startedAt: oldTs - 1000,
-        completedAt: oldTs,
-        nodeCount: 6,
-        leafCount: 10,
-        depth: 3,
-        configJson: "{}",
-        coherenceScore: 0.5,
-        timedOut: false,
-    }, sd);
-    assert.equal(isRaptorTreeFresh("sess-stale", sd, 4, 10), false);
-});
-// ─── 5. Freshness check: count drift > 20% → not fresh ──────────────────────
-test("S42D-5: isRaptorTreeFresh false when checkpoint count changed by >20%", () => {
-    const sd = stateDir();
-    openStore(sd);
-    insertBuildHistory({
-        sessionId: "sess-drift",
-        stateDir: sd,
-        startedAt: Date.now() - 1000,
-        completedAt: Date.now() - 500,
-        nodeCount: 6,
-        leafCount: 10,
-        depth: 3,
-        configJson: "{}",
-        coherenceScore: 0.5,
-        timedOut: false,
-    }, sd);
-    // 13 checkpoints vs 10 → 30% increase → not fresh
-    assert.equal(isRaptorTreeFresh("sess-drift", sd, 4, 13), false);
-    // 7 checkpoints vs 10 → 30% decrease → not fresh
-    assert.equal(isRaptorTreeFresh("sess-drift", sd, 4, 7), false);
-});
-// ─── 6. No build history → freshness false (forces build) ───────────────────
-test("S42D-6: isRaptorTreeFresh false when no build history exists", () => {
-    const sd = stateDir();
-    openStore(sd);
-    assert.equal(isRaptorTreeFresh("sess-none", sd, 4, 50), false);
-    // freshnessHours=0 always false (disables the gate → always rebuild)
-    insertBuildHistory({
-        sessionId: "sess-zero",
-        stateDir: sd,
-        startedAt: Date.now() - 100,
-        completedAt: Date.now() - 50,
-        nodeCount: 1,
-        leafCount: 5,
-        depth: 1,
-        configJson: "{}",
-        coherenceScore: 0.5,
-        timedOut: false,
-    }, sd);
-    assert.equal(isRaptorTreeFresh("sess-zero", sd, 0, 5), false);
-});
-// ─── 7. runRaptor records build history end-to-end ──────────────────────────
-test("S42D-7: runRaptor records a build-history row on real build", () => {
-    const sd = stateDir();
-    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
-    const sid = "bh-e2e";
-    // Seed 12 checkpoints and build a tree.
-    for (let i = 1; i <= 12; i++) {
-        compactSession({
-            sessionId: sid,
-            messages: [
-                msg(`build-history test checkpoint ${i} alpha beta`),
-                msg(`ack ${i}`, "Edit"),
-            ],
-            keepFrom: 2,
-            timestamp: i + 1,
-        }, s);
-    }
-    const nsid = normalizeSessionId(sid);
-    const all = vectorList(s, nsid);
-    const leaves = all.map((cp) => ({
-        id: cp.checkpointId,
-        messages: [],
-        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-        embedding: cp.embedding,
-    }));
-    const tree = runRaptor(leaves, {
-        stateDir: sd,
-        sessionId: nsid,
-        logger: new Logger(),
-    });
-    assert.ok(tree, "tree built");
-    const latest = getLatestBuild(nsid, sd);
-    assert.ok(latest, "build history recorded by runRaptor");
-    assert.equal(latest.leafCount, leaves.length);
-    assert.equal(latest.depth, tree.levels);
-    assert.equal(latest.timedOut, tree.timedOut);
-    assert.equal(latest.nodeCount, tree.nodes.size);
-    assert.ok(latest.coherenceScore !== null && latest.coherenceScore >= 0, "coherence score recorded");
-    // The freshness gate should now report fresh for the same checkpoint count.
-    assert.equal(isRaptorTreeFresh(nsid, sd, 4, all.length), true, "tree is fresh immediately after build");
-});

package/dist/src/dedup/raptor/multilevel-serve.test.js

@@ -1,229 +0,0 @@
-/**
- * multilevel-serve.test.ts — S42B multi-level RAPTOR serve acceptance tests.
- *
- * Verifies the raptorSearchHits multilevel wiring (RAPTOR_MULTILEVEL_ENABLED):
- *   (a) flag ON + leafExpansion OFF → search surfaces cluster hits carrying
- *       raptorSummary + raptorLevel (clusters survive dedup; on a 2-topic
- *       fixture a cluster's aggregate theme wins over its individual leaves
- *       when k is smaller than the cluster's leaf set)
- *   (b) flag OFF → search results never carry cluster markers (leaf-only path)
- *   (c) shadow mode gates the multilevel merge too (no RAPTOR hits regardless)
- *   (d) formatRecallBlock labels a raptorLevel hit as "cluster summary"
- *
- * No network — default extractive summarizer + trigram embedder.
- */
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { VectorStore, vectorList, vectorSearch } from "../../vectorStore.js";
-import { runRaptor, isShadowMode } from "./index.js";
-import { compactSession } from "../../engine.js";
-import { Logger } from "../../log.js";
-import { loadDedupConfig } from "../../config/dedup.js";
-import { formatRecallBlock } from "../../recall.js";
-import { normalizeSessionId } from "../../store.js";
-import { multilevelRetrieval } from "./multilevel.js";
-/* ------------------------------------------------------------------ helpers */
-let tmpDir;
-let counter = 0;
-beforeEach(() => {
-    tmpDir = mkdtempSync(join(tmpdir(), "mc-ml-"));
-});
-afterEach(() => {
-    rmSync(tmpDir, { recursive: true, force: true });
-});
-function stateDir() {
-    return join(tmpDir, `run-${counter++}`);
-}
-/** Config with dedup tiers disabled (distinct checkpoints) + live RAPTOR. */
-function cfg(overrides) {
-    return {
-        ...loadDedupConfig(),
-        RAPTOR_ENABLED: true,
-        L0_ENABLED: false,
-        L1_ENABLED: false,
-        L2_ENABLED: false,
-        ...overrides,
-    };
-}
-function msg(text, toolName) {
-    return toolName
-        ? { role: "assistant", text, toolName, input: text, output: text }
-        : { role: "user", text };
-}
-function seedSession(store, sid, count, topic, startTs = 1) {
-    for (let i = 1; i <= count; i++) {
-        compactSession({
-            sessionId: sid,
-            messages: [
-                msg(`${topic} checkpoint ${i} with unique content alpha beta`),
-                msg(`acknowledged ${i}`, "Edit"),
-            ],
-            keepFrom: 2,
-            timestamp: startTs + i,
-        }, store);
-    }
-}
-function buildTree(store, sid) {
-    const nsid = normalizeSessionId(sid);
-    const all = vectorList(store, nsid);
-    const leaves = all.map((cp) => ({
-        id: cp.checkpointId,
-        messages: [],
-        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-        embedding: cp.embedding,
-    }));
-    return runRaptor(leaves, {
-        stateDir: store.stateDir,
-        sessionId: nsid,
-        logger: new Logger(),
-    });
-}
-function withEnv(env, fn) {
-    const saved = {};
-    for (const k of Object.keys(env))
-        saved[k] = process.env[k];
-    for (const [k, v] of Object.entries(env)) {
-        if (v === undefined)
-            delete process.env[k];
-        else
-            process.env[k] = v;
-    }
-    try {
-        fn();
-    }
-    finally {
-        for (const [k, v] of Object.entries(saved)) {
-            if (v === undefined)
-                delete process.env[k];
-            else
-                process.env[k] = v;
-        }
-    }
-}
-/* ------------------------------------------------------------------- tests */
-// ─── (a) multilevelRetrieval surfaces cluster hits on a controlled fixture ──
-//
-// The full vectorSearch path applies MMR + dedup that, on small trigram
-// fixtures, tends to surface leaves over clusters (a cluster's centroid is at
-// best as similar as its best leaf, and the level weight makes it strictly
-// less). The S42B *wiring* is best asserted at the retrieval layer where the
-// cluster-vs-leaf outcome is deterministic: a hand-built 2-cluster tree with a
-// clean query makes the matching cluster win.
-test("S42B(a): multilevelRetrieval surfaces cluster hits carrying summary+level", () => {
-    // Build via the real pipeline so the tree shape is realistic, then query
-    // multilevelRetrieval directly with leafExpansion OFF so clusters survive.
-    const sd = stateDir();
-    const s = new VectorStore({
-        dedupSim: 0.9,
-        stateDir: sd,
-        config: cfg({ RAPTOR_LEAF_EXPANSION: false }),
-    });
-    const sid = "ml-direct";
-    seedSession(s, sid, 12, "multilevel topic");
-    buildTree(s, sid);
-    const nsid = normalizeSessionId(sid);
-    // Rehydrate and call multilevelRetrieval with k=1 and aggressive level
-    // weights so a cluster can win. This proves the SearchHit synthesis path
-    // (cluster node → raptorSummary/raptorLevel) is reachable.
-    const tree = runRaptor(vectorList(s, nsid).map((cp) => ({
-        id: cp.checkpointId,
-        messages: [],
-        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
-        embedding: cp.embedding,
-    })), { stateDir: sd, sessionId: nsid, logger: new Logger() });
-    assert.ok(tree, "tree built");
-    // When leafExpansion is OFF, deduplicateMultilevelHits keeps clusters whose
-    // leaves are NOT in the (small) result set. With a large fixture and k=2,
-    // at least one internal node must survive — assert that the multilevel path
-    // returns >=1 non-leaf hit under these conditions.
-    const ml = multilevelRetrieval("multilevel topic alpha", tree, {
-        embedder: s.embedder,
-        levelWeights: [1.0, 0.99, 0.98, 0.97, 0.96], // near-flat so clusters compete
-        leafExpansion: false,
-        maxLeafExpansion: 1,
-        k: 2,
-        mmrLambda: 0.99, // near-greedy so top-scored items survive
-    });
-    // S42B contract: the multilevel path CAN return non-leaf hits, and when it
-    // does, they carry summary + level. If the fixture's trigram similarity
-    // diffs make every leaf outscore every cluster even at near-flat weights,
-    // the wiring is still proven by S42B(b)/(c)/(d) below — but we assert that
-    // the returned hit set is well-formed either way.
-    for (const h of ml) {
-        assert.equal(typeof h.summary, "string", "hit carries summary");
-        assert.equal(typeof h.level, "number", "hit carries level");
-    }
-});
-// ─── (b) flag OFF → leaf-only path, no cluster markers ──────────────────────
-test("S42B(b): RAPTOR_MULTILEVEL_ENABLED=false → no cluster hits (leaf-only path)", () => {
-    const sd = stateDir();
-    const s = new VectorStore({
-        dedupSim: 0.9,
-        stateDir: sd,
-        config: cfg({ RAPTOR_MULTILEVEL_ENABLED: false }),
-    });
-    const sid = "ml-off";
-    seedSession(s, sid, 12, "leaftopic");
-    buildTree(s, sid);
-    const origShadow = process.env.RAPTOR_SHADOW_MODE;
-    process.env.RAPTOR_SHADOW_MODE = "false";
-    try {
-        const hits = vectorSearch(s, sid, "leaftopic alpha", 8);
-        assert.ok(hits.length > 0, "leaf search returns hits");
-        const clusterHits = hits.filter((h) => h.raptorLevel !== undefined);
-        assert.equal(clusterHits.length, 0, "no cluster hits when flag OFF");
-    }
-    finally {
-        if (origShadow === undefined)
-            delete process.env.RAPTOR_SHADOW_MODE;
-        else
-            process.env.RAPTOR_SHADOW_MODE = origShadow;
-    }
-});
-// ─── (c) shadow mode gates the multilevel merge ─────────────────────────────
-test("S42B(c): shadow mode → no RAPTOR merge regardless of multilevel flag", () => {
-    const sd = stateDir();
-    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
-    const sid = "ml-shadow";
-    seedSession(s, sid, 12, "shadowtopic");
-    buildTree(s, sid);
-    withEnv({ RAPTOR_SHADOW_MODE: "true", RAPTOR_MULTILEVEL_ENABLED: "true" }, () => {
-        assert.ok(isShadowMode(), "shadow mode active");
-        const hits = vectorSearch(s, sid, "shadowtopic alpha", 8);
-        // Shadow mode returns flat hits only — no cluster markers.
-        const clusterHits = hits.filter((h) => h.raptorLevel !== undefined);
-        assert.equal(clusterHits.length, 0, "shadow mode suppresses multilevel merge");
-    });
-});
-// ─── (d) formatRecallBlock labels cluster hits as "cluster summary" ─────────
-test("S42B(d): formatRecallBlock labels a raptorLevel hit as cluster summary", () => {
-    // Directly exercise the formatter with a synthetic cluster hit. The wiring
-    // (raptorSummary/raptorLevel → "cluster summary" label, raptorSummary body,
-    // no Key files line) is a pure formatter contract, independent of whether
-    // a cluster won the MMR race in (a).
-    const block = formatRecallBlock([
-        {
-            checkpoint: {
-                checkpointId: "r1_0",
-                sessionId: "sess_fmt",
-                summary: "fallback body",
-                keyDecisions: [],
-                nextSteps: [],
-                filesModified: ["src/x.ts"], // present but must NOT be rendered for cluster hits
-                tokenEstimate: 0,
-                regionHash: "x",
-                embedding: [],
-                timestamp: 0,
-            },
-            score: 0.8,
-            raptorSummary: "hierarchical cluster summary text",
-            raptorLevel: 1,
-        },
-    ]);
-    assert.ok(/Recalled cluster summary \[1\] \(level 1, relevance 80%\)/.test(block), "cluster hit labeled with level: " + block.split("\n")[0]);
-    assert.ok(block.includes("hierarchical cluster summary text"), "cluster body uses raptorSummary");
-    assert.ok(!/Key files:/.test(block), "cluster hit has no Key files line even when filesModified is non-empty");
-});