pi-mega-compact 0.8.24 → 0.8.26

package/dist/src/dedup/raptor/buildHistory.js

@@ -0,0 +1,164 @@
+/**
+ * buildHistory.ts — S42D RAPTOR build history + freshness.
+ *
+ * - Stores one structured row per tree build in `raptor_build_history`.
+ * - `computeCoherenceScore`: avg intra-cluster cosine similarity (0–1) for a
+ *   built tree — a health/diagnostics metric, not a gating signal.
+ * - `isRaptorTreeFresh`: lets the build call site skip a rebuild when the last
+ *   build is recent (within `freshnessHours`) and the checkpoint count has not
+ *   drifted by more than 20%. Returns false when no history exists (forces a
+ *   build) — the same default as a missing tree.
+ *
+ * No network. Pure SQLite + local cosine.
+ */
+import { randomUUID } from "node:crypto";
+import { openStore, withTx } from "../../store/sqlite/utils.js";
+import { getStateDir } from "../../store.js";
+import { normalizeSessionId } from "../../store.js";
+import { cosineSimilarity } from "../../embedder.js";
+/** Insert one build-history row. Idempotent on build_id (PK). */
+export function insertBuildHistory(input, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const buildId = randomUUID();
+    const sid = normalizeSessionId(input.sessionId);
+    withTx(db, () => {
+        db.prepare(`INSERT INTO raptor_build_history
+         (build_id, session_id, state_dir, started_at, completed_at,
+          node_count, leaf_count, depth, config_json, coherence_score, timed_out)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(build_id) DO NOTHING`).run(buildId, sid, input.stateDir, input.startedAt, input.completedAt, input.nodeCount, input.leafCount, input.depth, input.configJson, input.coherenceScore, input.timedOut ? 1 : 0);
+    });
+    return buildId;
+}
+/** The most recent build row for a session (by completed_at), or null. */
+export function getLatestBuild(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const row = db
+        .prepare(`SELECT build_id, session_id, state_dir, started_at, completed_at,
+              node_count, leaf_count, depth, config_json, coherence_score, timed_out
+       FROM raptor_build_history
+       WHERE session_id = ?
+       ORDER BY completed_at DESC
+       LIMIT 1`)
+        .get(sid);
+    if (!row)
+        return null;
+    return {
+        buildId: row.build_id,
+        sessionId: row.session_id,
+        stateDir: row.state_dir,
+        startedAt: row.started_at,
+        completedAt: row.completed_at,
+        nodeCount: row.node_count,
+        leafCount: row.leaf_count,
+        depth: row.depth,
+        configJson: row.config_json,
+        coherenceScore: row.coherence_score,
+        timedOut: row.timed_out === 1,
+    };
+}
+/** All build rows for a session, newest first. */
+export function listBuildHistory(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const rows = db
+        .prepare(`SELECT build_id, session_id, state_dir, started_at, completed_at,
+              node_count, leaf_count, depth, config_json, coherence_score, timed_out
+       FROM raptor_build_history
+       WHERE session_id = ?
+       ORDER BY completed_at DESC`)
+        .all(sid);
+    return rows.map((row) => ({
+        buildId: row.build_id,
+        sessionId: row.session_id,
+        stateDir: row.state_dir,
+        startedAt: row.started_at,
+        completedAt: row.completed_at,
+        nodeCount: row.node_count,
+        leafCount: row.leaf_count,
+        depth: row.depth,
+        configJson: row.config_json,
+        coherenceScore: row.coherence_score,
+        timedOut: row.timed_out === 1,
+    }));
+}
+/**
+ * Compute the average intra-cluster cosine similarity for a RAPTOR tree.
+ * For each internal node, compute the mean pairwise cosine similarity of its
+ * children's embeddings. Average across all internal nodes. Returns 0–1
+ * (higher = more coherent clusters). A tree with no internal nodes returns 0.
+ *
+ * Child embeddings: a child that is itself an internal node contributes its own
+ * centroid. A child id not in `tree.nodes` is a raw leaf — leaves are not stored
+ * on RaptorNodes, so `leafEmbeddings` (id → embedding, from the build-time Leaf[]
+ * set) supplies the true leaf embedding; if absent, we fall back to the parent
+ * node's centroid. Passing `leafEmbeddings` is strongly recommended — without it
+ * every leaf collapses to its parent's centroid and the score inflates to ~1.0.
+ */
+export function computeCoherenceScore(tree, leafEmbeddings) {
+    let sum = 0;
+    let count = 0;
+    for (const node of tree.nodes.values()) {
+        if (node.embedding.length === 0)
+            continue;
+        const childEmbeddings = [];
+        for (const cid of node.children) {
+            const child = tree.nodes.get(cid);
+            if (child && child.embedding.length > 0) {
+                childEmbeddings.push(child.embedding);
+            }
+            else {
+                // Raw leaf: use its true embedding if known, else parent centroid.
+                const leafEmb = leafEmbeddings?.get(cid);
+                childEmbeddings.push(leafEmb && leafEmb.length > 0 ? leafEmb : node.embedding);
+            }
+        }
+        if (childEmbeddings.length < 2)
+            continue; // need >=2 to measure spread
+        let pairSum = 0;
+        let pairCount = 0;
+        for (let i = 0; i < childEmbeddings.length; i++) {
+            for (let j = i + 1; j < childEmbeddings.length; j++) {
+                pairSum += cosineSimilarity(childEmbeddings[i], childEmbeddings[j]);
+                pairCount++;
+            }
+        }
+        if (pairCount > 0) {
+            sum += pairSum / pairCount;
+            count++;
+        }
+    }
+    return count > 0 ? sum / count : 0;
+}
+/**
+ * Check if the RAPTOR tree is fresh enough to skip a rebuild.
+ * Returns true iff:
+ *   - a build-history row exists, AND
+ *   - it completed within `freshnessHours`, AND
+ *   - the checkpoint count has not changed by more than 20% since the build.
+ * A `freshnessHours` of 0 always returns false (rebuild every time) — the spec's
+ * "0 disables" escape hatch, symmetric to the S38 retry caps.
+ */
+export function isRaptorTreeFresh(sessionId, stateDir, freshnessHours, currentCheckpointCount) {
+    if (freshnessHours <= 0)
+        return false;
+    const latest = getLatestBuild(sessionId, stateDir);
+    if (!latest)
+        return false;
+    const ageHours = (Date.now() - latest.completedAt) / 3_600_000;
+    if (ageHours > freshnessHours)
+        return false;
+    if (latest.leafCount <= 0)
+        return false;
+    const changeRatio = Math.abs(currentCheckpointCount - latest.leafCount) / latest.leafCount;
+    return changeRatio <= 0.2;
+}
+/** Clear build history for a session (used by tests + DR). */
+export function clearBuildHistory(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    withTx(db, () => {
+        db.prepare("DELETE FROM raptor_build_history WHERE session_id = ?").run(sid);
+    });
+}

package/dist/src/dedup/raptor/buildHistory.test.js

@@ -0,0 +1,292 @@
+/**
+ * buildHistory.test.ts — S42D RAPTOR build history + freshness tests.
+ *
+ * No network. Real stores with temp state dirs.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openStore } from "../../store/sqlite/utils.js";
+import { insertBuildHistory, getLatestBuild, listBuildHistory, computeCoherenceScore, isRaptorTreeFresh, clearBuildHistory, } from "./buildHistory.js";
+import { runRaptor } from "./index.js";
+import { VectorStore, vectorList } from "../../vectorStore.js";
+import { compactSession } from "../../engine.js";
+import { loadDedupConfig } from "../../config/dedup.js";
+import { normalizeSessionId } from "../../store.js";
+import { Logger } from "../../log.js";
+let tmpDir;
+let counter = 0;
+beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "mc-bh-"));
+});
+afterEach(() => {
+    rmSync(tmpDir, { recursive: true, force: true });
+});
+function stateDir() {
+    return join(tmpDir, `run-${counter++}`);
+}
+function cfg(overrides) {
+    return {
+        ...loadDedupConfig(),
+        RAPTOR_ENABLED: true,
+        L0_ENABLED: false,
+        L1_ENABLED: false,
+        L2_ENABLED: false,
+        ...overrides,
+    };
+}
+function msg(text, toolName) {
+    return toolName
+        ? { role: "assistant", text, toolName, input: text, output: text }
+        : { role: "user", text };
+}
+// ─── 1. Build-history rows are created with correct metadata ──────────────
+test("S42D-1: insertBuildHistory + getLatestBuild round-trip", () => {
+    const sd = stateDir();
+    openStore(sd); // initialize schema
+    const ts = Date.now();
+    const id = insertBuildHistory({
+        sessionId: "sess-foo",
+        stateDir: sd,
+        startedAt: ts - 500,
+        completedAt: ts,
+        nodeCount: 6,
+        leafCount: 10,
+        depth: 3,
+        configJson: '{"budgetMs":5000}',
+        coherenceScore: 0.82,
+        timedOut: false,
+    }, sd);
+    assert.ok(id, "build id returned");
+    const latest = getLatestBuild("sess-foo", sd);
+    assert.ok(latest, "latest build found");
+    assert.equal(latest.sessionId, normalizeSessionId("sess-foo"));
+    assert.equal(latest.nodeCount, 6);
+    assert.equal(latest.leafCount, 10);
+    assert.equal(latest.depth, 3);
+    assert.equal(latest.coherenceScore, 0.82);
+    assert.equal(latest.timedOut, false);
+    assert.equal(latest.configJson, '{"budgetMs":5000}');
+    // getLatest returns the newest by completed_at
+    insertBuildHistory({
+        sessionId: "sess-foo",
+        stateDir: sd,
+        startedAt: ts + 1000,
+        completedAt: ts + 2000,
+        nodeCount: 7,
+        leafCount: 11,
+        depth: 3,
+        configJson: "{}",
+        coherenceScore: null,
+        timedOut: true,
+    }, sd);
+    const newest = getLatestBuild("sess-foo", sd);
+    assert.equal(newest.nodeCount, 7);
+    assert.equal(newest.timedOut, true);
+    assert.equal(newest.coherenceScore, null);
+    // listBuildHistory returns all, newest first
+    const all = listBuildHistory("sess-foo", sd);
+    assert.equal(all.length, 2);
+    assert.equal(all[0].nodeCount, 7); // newest first
+    // No history for an unknown session
+    assert.equal(getLatestBuild("sess-none", sd), null);
+    assert.deepEqual(listBuildHistory("sess-none", sd), []);
+    // clearBuildHistory removes rows
+    clearBuildHistory("sess-foo", sd);
+    assert.equal(getLatestBuild("sess-foo", sd), null);
+});
+// ─── 2. Coherence score: 1.0 for identical, lower for diverse ───────────────
+test("S42D-2: computeCoherenceScore is 1.0 for identical embeddings, lower for diverse", () => {
+    const identical = {
+        rootId: "root",
+        nodes: new Map([
+            ["root", { id: "root", level: 0, parentId: null, children: ["c1", "c2"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 2 }],
+            ["c1", { id: "c1", level: 1, parentId: "root", children: [], summary: "c1", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
+            ["c2", { id: "c2", level: 1, parentId: "root", children: [], summary: "c2", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
+        ]),
+        levels: 2,
+        timedOut: false,
+    };
+    assert.equal(computeCoherenceScore(identical), 1.0);
+    // Diverse: orthogonal children → cosine 0
+    const diverse = {
+        rootId: "root",
+        nodes: new Map([
+            ["root", { id: "root", level: 0, parentId: null, children: ["c1", "c2"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 2 }],
+            ["c1", { id: "c1", level: 1, parentId: "root", children: [], summary: "c1", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
+            ["c2", { id: "c2", level: 1, parentId: "root", children: [], summary: "c2", embedding: [0, 1], qualityMarker: "low", tokenEstimate: 1 }],
+        ]),
+        levels: 2,
+        timedOut: false,
+    };
+    const diverseScore = computeCoherenceScore(diverse);
+    assert.ok(diverseScore < 1.0, `diverse score < 1.0 (got ${diverseScore})`);
+    assert.ok(diverseScore >= 0, "score is non-negative");
+    // No internal nodes with >=2 children → 0
+    const single = {
+        rootId: "root",
+        nodes: new Map([
+            ["root", { id: "root", level: 0, parentId: null, children: ["c1"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
+            ["c1", { id: "c1", level: 1, parentId: "root", children: [], summary: "c1", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 1 }],
+        ]),
+        levels: 2,
+        timedOut: false,
+    };
+    assert.equal(computeCoherenceScore(single), 0);
+});
+// ─── 2b. Coherence with leaf embeddings (the real-tree shape) ──────────────
+test("S42D-2b: computeCoherenceScore uses leafEmbeddings for leaf children (real-tree shape)", () => {
+    // Root whose children are raw leaves (the common collapse-path shape).
+    // Without leafEmbeddings, leaves fall back to the root centroid → score 1.0
+    // (degenerate). With true leaf embeddings, diverse leaves score < 1.0.
+    const tree = {
+        rootId: "root",
+        nodes: new Map([
+            ["root", { id: "root", level: 0, parentId: null, children: ["l1", "l2"], summary: "r", embedding: [1, 0], qualityMarker: "low", tokenEstimate: 2 }],
+        ]),
+        levels: 1,
+        timedOut: false,
+    };
+    // No leaf map → both leaves use root centroid [1,0] → cosine 1.0 (degenerate).
+    assert.equal(computeCoherenceScore(tree), 1.0);
+    // Leaf map with orthogonal leaves → cosine 0 (truly diverse).
+    const leafMap = new Map([
+        ["l1", [1, 0]],
+        ["l2", [0, 1]],
+    ]);
+    const score = computeCoherenceScore(tree, leafMap);
+    assert.ok(score < 1.0, `diverse leaves score < 1.0 (got ${score})`);
+    assert.ok(score >= 0, "score non-negative");
+    // Identical leaves via the map → 1.0
+    const leafMapIdentical = new Map([
+        ["l1", [1, 0]],
+        ["l2", [1, 0]],
+    ]);
+    assert.equal(computeCoherenceScore(tree, leafMapIdentical), 1.0);
+});
+// ─── 3. Freshness check: recent + stable → fresh ────────────────────────────
+test("S42D-3: isRaptorTreeFresh true for recent build + stable count", () => {
+    const sd = stateDir();
+    openStore(sd);
+    insertBuildHistory({
+        sessionId: "sess-fresh",
+        stateDir: sd,
+        startedAt: Date.now() - 1000,
+        completedAt: Date.now() - 500, // < 4h old
+        nodeCount: 6,
+        leafCount: 10,
+        depth: 3,
+        configJson: "{}",
+        coherenceScore: 0.5,
+        timedOut: false,
+    }, sd);
+    // 10 checkpoints, latest had 10 → within 20% → fresh
+    assert.equal(isRaptorTreeFresh("sess-fresh", sd, 4, 10), true);
+    // 11 checkpoints → 10% change → still fresh
+    assert.equal(isRaptorTreeFresh("sess-fresh", sd, 4, 11), true);
+    // 9 checkpoints → 10% change → fresh
+    assert.equal(isRaptorTreeFresh("sess-fresh", sd, 4, 9), true);
+});
+// ─── 4. Freshness check: stale build → not fresh ────────────────────────────
+test("S42D-4: isRaptorTreeFresh false for stale build (>4h old)", () => {
+    const sd = stateDir();
+    openStore(sd);
+    const oldTs = Date.now() - 5 * 3_600_000; // 5h ago
+    insertBuildHistory({
+        sessionId: "sess-stale",
+        stateDir: sd,
+        startedAt: oldTs - 1000,
+        completedAt: oldTs,
+        nodeCount: 6,
+        leafCount: 10,
+        depth: 3,
+        configJson: "{}",
+        coherenceScore: 0.5,
+        timedOut: false,
+    }, sd);
+    assert.equal(isRaptorTreeFresh("sess-stale", sd, 4, 10), false);
+});
+// ─── 5. Freshness check: count drift > 20% → not fresh ──────────────────────
+test("S42D-5: isRaptorTreeFresh false when checkpoint count changed by >20%", () => {
+    const sd = stateDir();
+    openStore(sd);
+    insertBuildHistory({
+        sessionId: "sess-drift",
+        stateDir: sd,
+        startedAt: Date.now() - 1000,
+        completedAt: Date.now() - 500,
+        nodeCount: 6,
+        leafCount: 10,
+        depth: 3,
+        configJson: "{}",
+        coherenceScore: 0.5,
+        timedOut: false,
+    }, sd);
+    // 13 checkpoints vs 10 → 30% increase → not fresh
+    assert.equal(isRaptorTreeFresh("sess-drift", sd, 4, 13), false);
+    // 7 checkpoints vs 10 → 30% decrease → not fresh
+    assert.equal(isRaptorTreeFresh("sess-drift", sd, 4, 7), false);
+});
+// ─── 6. No build history → freshness false (forces build) ───────────────────
+test("S42D-6: isRaptorTreeFresh false when no build history exists", () => {
+    const sd = stateDir();
+    openStore(sd);
+    assert.equal(isRaptorTreeFresh("sess-none", sd, 4, 50), false);
+    // freshnessHours=0 always false (disables the gate → always rebuild)
+    insertBuildHistory({
+        sessionId: "sess-zero",
+        stateDir: sd,
+        startedAt: Date.now() - 100,
+        completedAt: Date.now() - 50,
+        nodeCount: 1,
+        leafCount: 5,
+        depth: 1,
+        configJson: "{}",
+        coherenceScore: 0.5,
+        timedOut: false,
+    }, sd);
+    assert.equal(isRaptorTreeFresh("sess-zero", sd, 0, 5), false);
+});
+// ─── 7. runRaptor records build history end-to-end ──────────────────────────
+test("S42D-7: runRaptor records a build-history row on real build", () => {
+    const sd = stateDir();
+    const s = new VectorStore({ dedupSim: 0.9, stateDir: sd, config: cfg() });
+    const sid = "bh-e2e";
+    // Seed 12 checkpoints and build a tree.
+    for (let i = 1; i <= 12; i++) {
+        compactSession({
+            sessionId: sid,
+            messages: [
+                msg(`build-history test checkpoint ${i} alpha beta`),
+                msg(`ack ${i}`, "Edit"),
+            ],
+            keepFrom: 2,
+            timestamp: i + 1,
+        }, s);
+    }
+    const nsid = normalizeSessionId(sid);
+    const all = vectorList(s, nsid);
+    const leaves = all.map((cp) => ({
+        id: cp.checkpointId,
+        messages: [],
+        sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
+        embedding: cp.embedding,
+    }));
+    const tree = runRaptor(leaves, {
+        stateDir: sd,
+        sessionId: nsid,
+        logger: new Logger(),
+    });
+    assert.ok(tree, "tree built");
+    const latest = getLatestBuild(nsid, sd);
+    assert.ok(latest, "build history recorded by runRaptor");
+    assert.equal(latest.leafCount, leaves.length);
+    assert.equal(latest.depth, tree.levels);
+    assert.equal(latest.timedOut, tree.timedOut);
+    assert.equal(latest.nodeCount, tree.nodes.size);
+    assert.ok(latest.coherenceScore !== null && latest.coherenceScore >= 0, "coherence score recorded");
+    // The freshness gate should now report fresh for the same checkpoint count.
+    assert.equal(isRaptorTreeFresh(nsid, sd, 4, all.length), true, "tree is fresh immediately after build");
+});

package/dist/src/dedup/raptor/index.js

@@ -13,6 +13,7 @@ import { defaultEmbedder } from "../../embedder.js";
 import { buildRaptorTree } from "./tree.js";
 import { stagedExpansion } from "./retrieval.js";
 import { saveRaptorTree, listRaptorNodes } from "../../store/sqlite.js";
+import { insertBuildHistory, computeCoherenceScore } from "./buildHistory.js";
 /** Shadow mode is on by default; set RAPTOR_SHADOW_MODE=false to serve live. */
 export function isShadowMode() {
     return process.env.RAPTOR_SHADOW_MODE !== "false";
@@ -28,6 +29,7 @@ export function isShadowMode() {
 export function runRaptor(leaves, opts) {
     const embedder = opts.embedder ?? defaultEmbedder();
     const logger = opts.logger;
+    const startedAt = opts.builtAt ?? Date.now();
     try {
         const tree = buildRaptorTree(leaves, {
             embedder,
@@ -45,6 +47,42 @@ export function runRaptor(leaves, opts) {
             timedOut: tree.timedOut,
             shadow: isShadowMode(),
         });
+        // S42D: record structured build history (coherence score + leaf count) so
+        // the freshness check can skip the next rebuild when the tree is recent
+        // and the chunk count is stable. Best-effort: a history-write failure must
+        // never break the build.
+        try {
+            // Map leaf id → embedding so computeCoherenceScore can score real
+            // intra-cluster spread (without this, leaves collapse to parent centroids
+            // and the score inflates to ~1.0 for every tree).
+            const leafEmbs = new Map();
+            for (const l of leaves)
+                if (l.embedding.length > 0)
+                    leafEmbs.set(l.id, l.embedding);
+            const coherence = computeCoherenceScore(tree, leafEmbs);
+            insertBuildHistory({
+                sessionId: opts.sessionId,
+                stateDir: opts.stateDir,
+                startedAt,
+                completedAt: Date.now(),
+                nodeCount: tree.nodes.size,
+                leafCount: leaves.length,
+                depth: tree.levels,
+                configJson: JSON.stringify({
+                    budgetMs: opts.budgetMs,
+                    clustersPerLevel: opts.clustersPerLevel,
+                    consistencyThreshold: opts.consistencyThreshold,
+                }),
+                coherenceScore: coherence,
+                timedOut: tree.timedOut,
+            }, opts.stateDir);
+        }
+        catch (histErr) {
+            logger?.warn("raptor_build_history_failed", {
+                sessionId: opts.sessionId,
+                error: String(histErr instanceof Error ? histErr.message : histErr),
+            });
+        }
         if (isShadowMode()) {
             // Build + log only. Do NOT replace retrieval.
             logger?.info("raptor_shadow", { sessionId: opts.sessionId, served: false });