pi-mega-compact 0.16.3 → 0.17.1

package/dist/dedup/raptor/buildHistory.js

@@ -0,0 +1,164 @@
+/**
+ * buildHistory.ts — S42D RAPTOR build history + freshness.
+ *
+ * - Stores one structured row per tree build in `raptor_build_history`.
+ * - `computeCoherenceScore`: avg intra-cluster cosine similarity (0–1) for a
+ *   built tree — a health/diagnostics metric, not a gating signal.
+ * - `isRaptorTreeFresh`: lets the build call site skip a rebuild when the last
+ *   build is recent (within `freshnessHours`) and the checkpoint count has not
+ *   drifted by more than 20%. Returns false when no history exists (forces a
+ *   build) — the same default as a missing tree.
+ *
+ * No network. Pure SQLite + local cosine.
+ */
+import { randomUUID } from "node:crypto";
+import { openStore, withTx } from "../../store/sqlite/utils.js";
+import { getStateDir } from "../../store.js";
+import { normalizeSessionId } from "../../store.js";
+import { cosineSimilarity } from "../../embedder.js";
+/** Insert one build-history row. Idempotent on build_id (PK). */
+export function insertBuildHistory(input, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const buildId = randomUUID();
+    const sid = normalizeSessionId(input.sessionId);
+    withTx(db, () => {
+        db.prepare(`INSERT INTO raptor_build_history
+         (build_id, session_id, state_dir, started_at, completed_at,
+          node_count, leaf_count, depth, config_json, coherence_score, timed_out)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(build_id) DO NOTHING`).run(buildId, sid, input.stateDir, input.startedAt, input.completedAt, input.nodeCount, input.leafCount, input.depth, input.configJson, input.coherenceScore, input.timedOut ? 1 : 0);
+    });
+    return buildId;
+}
+/** The most recent build row for a session (by completed_at), or null. */
+export function getLatestBuild(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const row = db
+        .prepare(`SELECT build_id, session_id, state_dir, started_at, completed_at,
+              node_count, leaf_count, depth, config_json, coherence_score, timed_out
+       FROM raptor_build_history
+       WHERE session_id = ?
+       ORDER BY completed_at DESC
+       LIMIT 1`)
+        .get(sid);
+    if (!row)
+        return null;
+    return {
+        buildId: row.build_id,
+        sessionId: row.session_id,
+        stateDir: row.state_dir,
+        startedAt: row.started_at,
+        completedAt: row.completed_at,
+        nodeCount: row.node_count,
+        leafCount: row.leaf_count,
+        depth: row.depth,
+        configJson: row.config_json,
+        coherenceScore: row.coherence_score,
+        timedOut: row.timed_out === 1,
+    };
+}
+/** All build rows for a session, newest first. */
+export function listBuildHistory(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const rows = db
+        .prepare(`SELECT build_id, session_id, state_dir, started_at, completed_at,
+              node_count, leaf_count, depth, config_json, coherence_score, timed_out
+       FROM raptor_build_history
+       WHERE session_id = ?
+       ORDER BY completed_at DESC`)
+        .all(sid);
+    return rows.map((row) => ({
+        buildId: row.build_id,
+        sessionId: row.session_id,
+        stateDir: row.state_dir,
+        startedAt: row.started_at,
+        completedAt: row.completed_at,
+        nodeCount: row.node_count,
+        leafCount: row.leaf_count,
+        depth: row.depth,
+        configJson: row.config_json,
+        coherenceScore: row.coherence_score,
+        timedOut: row.timed_out === 1,
+    }));
+}
+/**
+ * Compute the average intra-cluster cosine similarity for a RAPTOR tree.
+ * For each internal node, compute the mean pairwise cosine similarity of its
+ * children's embeddings. Average across all internal nodes. Returns 0–1
+ * (higher = more coherent clusters). A tree with no internal nodes returns 0.
+ *
+ * Child embeddings: a child that is itself an internal node contributes its own
+ * centroid. A child id not in `tree.nodes` is a raw leaf — leaves are not stored
+ * on RaptorNodes, so `leafEmbeddings` (id → embedding, from the build-time Leaf[]
+ * set) supplies the true leaf embedding; if absent, we fall back to the parent
+ * node's centroid. Passing `leafEmbeddings` is strongly recommended — without it
+ * every leaf collapses to its parent's centroid and the score inflates to ~1.0.
+ */
+export function computeCoherenceScore(tree, leafEmbeddings) {
+    let sum = 0;
+    let count = 0;
+    for (const node of tree.nodes.values()) {
+        if (node.embedding.length === 0)
+            continue;
+        const childEmbeddings = [];
+        for (const cid of node.children) {
+            const child = tree.nodes.get(cid);
+            if (child && child.embedding.length > 0) {
+                childEmbeddings.push(child.embedding);
+            }
+            else {
+                // Raw leaf: use its true embedding if known, else parent centroid.
+                const leafEmb = leafEmbeddings?.get(cid);
+                childEmbeddings.push(leafEmb && leafEmb.length > 0 ? leafEmb : node.embedding);
+            }
+        }
+        if (childEmbeddings.length < 2)
+            continue; // need >=2 to measure spread
+        let pairSum = 0;
+        let pairCount = 0;
+        for (let i = 0; i < childEmbeddings.length; i++) {
+            for (let j = i + 1; j < childEmbeddings.length; j++) {
+                pairSum += cosineSimilarity(childEmbeddings[i], childEmbeddings[j]);
+                pairCount++;
+            }
+        }
+        if (pairCount > 0) {
+            sum += pairSum / pairCount;
+            count++;
+        }
+    }
+    return count > 0 ? sum / count : 0;
+}
+/**
+ * Check if the RAPTOR tree is fresh enough to skip a rebuild.
+ * Returns true iff:
+ *   - a build-history row exists, AND
+ *   - it completed within `freshnessHours`, AND
+ *   - the checkpoint count has not changed by more than 20% since the build.
+ * A `freshnessHours` of 0 always returns false (rebuild every time) — the spec's
+ * "0 disables" escape hatch, symmetric to the S38 retry caps.
+ */
+export function isRaptorTreeFresh(sessionId, stateDir, freshnessHours, currentCheckpointCount) {
+    if (freshnessHours <= 0)
+        return false;
+    const latest = getLatestBuild(sessionId, stateDir);
+    if (!latest)
+        return false;
+    const ageHours = (Date.now() - latest.completedAt) / 3_600_000;
+    if (ageHours > freshnessHours)
+        return false;
+    if (latest.leafCount <= 0)
+        return false;
+    const changeRatio = Math.abs(currentCheckpointCount - latest.leafCount) / latest.leafCount;
+    return changeRatio <= 0.2;
+}
+/** Clear build history for a session (used by tests + DR). */
+export function clearBuildHistory(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    withTx(db, () => {
+        db.prepare("DELETE FROM raptor_build_history WHERE session_id = ?").run(sid);
+    });
+}

package/dist/dedup/raptor/guardrails.js

@@ -0,0 +1,83 @@
+/**
+ * guardrails.ts — hallucination defense for RAPTOR summary nodes (Sprint 13, QA #16).
+ *
+ * Four layers gate a candidate summary before it may be marked high-quality:
+ *   1. Claim grounding — every claim in the summary maps to source text
+ *      (no entity/claim appears that isn't supported by a source chunk).
+ *   2. Entity coverage — fraction of summary entities that are present in source.
+ *   3. Consistency — cosine(reEmbed(summary), cluster centroid) ≥ threshold.
+ *      This is the HARD gate: a low score means the summary drifted from the
+ *      source cluster, so we fall back to extractive (never serve a low-quality
+ *      LLM summary).
+ *   4. Quality markers — 'high' | 'low' | 'extractive_fallback' assigned from the
+ *      above.
+ *
+ * Pure functions, no network, no model. The consistency check uses the caller's
+ * embedder (the same local Embedder used everywhere else).
+ */
+import { cosineSimilarity } from "../../embedder.js";
+const ENTITY_RE = /\b([A-Z][a-zA-Z0-9_]{2,}|[a-z_]+_[a-z_]+|\d{2,})\b/g;
+/** Extract candidate "entities"/tokens from a summary for grounding checks. */
+export function extractEntities(text) {
+    const out = new Set();
+    for (const m of text.matchAll(ENTITY_RE))
+        out.add(m[1].toLowerCase());
+    return [...out];
+}
+/** Lowercase word set from a body of source text (for grounding lookups). */
+export function sourceTokenSet(sources) {
+    const set = new Set();
+    for (const s of sources)
+        for (const w of s.toLowerCase().split(/\W+/))
+            if (w)
+                set.add(w);
+    return set;
+}
+/**
+ * Verify a summary against its sources + centroid.
+ *
+ * Faithfulness (QA #16): consistency is the hard gate. If the summary embedding
+ * is insufficiently similar to the cluster centroid, the summary is NOT faithful
+ * to the source — mark it 'extractive_fallback' so callers fall back to the
+ * deterministic extractive summary instead of serving a drifted LLM summary.
+ *
+ * grounding: every summary entity must appear in the source token set. A single
+ * un-grounded entity fails grounding (caught hallucination).
+ */
+export function applyHallucinationGuardrails(input) {
+    const threshold = input.consistencyThreshold ?? 0.6;
+    const sourceTokens = input.sourceTokens;
+    // Layer 1 + 2: entity grounding & coverage.
+    const entities = extractEntities(input.summary);
+    let groundedCount = 0;
+    for (const e of entities) {
+        if (sourceTokens.has(e))
+            groundedCount++;
+    }
+    const entityCoverage = entities.length === 0 ? 1 : groundedCount / entities.length;
+    const grounded = entities.length === 0 || groundedCount === entities.length;
+    // Layer 3: consistency re-embed.
+    const summEmbed = input.embedder.embed(input.summary);
+    const consistency = cosineSimilarity(summEmbed, input.centroid);
+    // Layer 4: quality marker decision.
+    if (!grounded || consistency < threshold) {
+        return {
+            marker: "extractive_fallback",
+            entityCoverage,
+            consistency,
+            grounded,
+            reason: !grounded
+                ? "ungrounded entity in summary"
+                : `consistency ${consistency.toFixed(2)} < ${threshold} (drift from source)`,
+        };
+    }
+    const marker = entityCoverage >= 0.7 ? "high" : "low";
+    return { marker, entityCoverage, consistency, grounded, reason: "ok" };
+}
+/**
+ * Convenience: build a fixture summary that is deliberately un-grounded (used by
+ * tests to prove the guardrail CATCHES a hallucination). Not used in production.
+ */
+export function makeUngroundedSummary(realSource, fakeEntity) {
+    return `${realSource.slice(0, 60)} The quarterly revenue doubled to ${fakeEntity}.`;
+}

package/dist/dedup/raptor/incremental.js

@@ -0,0 +1,304 @@
+/**
+ * incremental.ts — incremental RAPTOR tree update (Sprint 26, #7).
+ *
+ * Instead of rebuilding the full tree (~2s per compaction), insert only the new
+ * checkpoints since the last build and recompute only the affected cluster
+ * assignments up the tree. Falls back to a full rebuild when the tree is
+ * missing, corrupted, or >50% of nodes are new.
+ *
+ * Non-fatal: every failure logs and returns null (caller falls back to full
+ * rebuild). PREVENT-PI-004: zero network; all operations are local SQLite + CPU.
+ */
+import { cosineSimilarity } from "../../embedder.js";
+import { defaultEmbedder as getDefaultEmbedder } from "../../embedder.js";
+import { buildRaptorTree } from "./tree.js";
+import { applyHallucinationGuardrails, sourceTokenSet } from "./guardrails.js";
+import { meanVector } from "./kmeans.js";
+import { summarizeCluster, extractiveClusterSummary } from "./summarizer.js";
+import { saveRaptorTree, listRaptorNodes, } from "../../store/sqlite.js";
+/** When >FRESH_THRESHOLD of existing leaves are new, a full rebuild is cheaper. */
+const FRESH_THRESHOLD = 0.5;
+/**
+ * Summarize + apply hallucination guardrails (mirrors tree.ts:summarizeInto).
+ * If the guardrail marks the summary as extractive_fallback (drift/hallucination),
+ * re-summarize with the deterministic extractive path so we never serve a
+ * drifted LLM summary from an incremental update.
+ */
+function summarizeGuarded(messages, centroid, embedder, consistencyThreshold) {
+    let cs = summarizeCluster(messages);
+    const sources = messages.map((m) => m.text);
+    const guard = applyHallucinationGuardrails({
+        summary: cs.summary,
+        sources,
+        centroid,
+        embedder,
+        sourceTokens: sourceTokenSet(sources),
+        consistencyThreshold,
+    });
+    if (guard.marker === "extractive_fallback") {
+        cs = extractiveClusterSummary(messages);
+    }
+    return {
+        summary: cs.summary,
+        tokenEstimate: cs.tokenEstimate,
+        qualityMarker: guard.marker === "extractive_fallback" ? "low" : guard.marker,
+    };
+}
+/**
+ * Incrementally update a persisted RAPTOR tree with new leaves.
+ *
+ * 1. Reads the existing tree from the store.
+ * 2. Filters `newLeaves` to those NOT already covered by the tree.
+ * 3. If no new leaves → early-return (existing tree).
+ * 4. If no existing tree or >50% new → full rebuild.
+ * 5. For each new leaf, finds the best-matching Level-0 cluster node,
+ *    reassigns (inserts into its children), updates that node's summary
+ *    and centroid embedding.
+ * 6. Propagates changes up through parent levels to the root.
+ * 7. Persists the updated tree (full delete + reinsert in a tx).
+ *
+ * Never throws. Returns the updated tree, or null on failure/fallback.
+ */
+export function incrementRaptorTree(existingLeaves, newLeaves, opts) {
+    const logger = opts.logger;
+    const builtAt = opts.builtAt ?? Date.now();
+    try {
+        // 1. Read existing tree.
+        const storedNodes = listRaptorNodes(opts.sessionId, opts.stateDir);
+        if (storedNodes.length === 0) {
+            logger?.info("raptor_incremental_no_tree", { sessionId: opts.sessionId });
+            return null; // no existing tree — caller falls back to full buildRaptorTree
+        }
+        // 2. Find new leaf ids not already in the tree.
+        const coveredLeafIds = new Set();
+        for (const n of storedNodes) {
+            for (const cid of n.children)
+                coveredLeafIds.add(cid);
+        }
+        const reallyNewLeaves = newLeaves.filter((l) => !coveredLeafIds.has(l.id));
+        const totalCovered = coveredLeafIds.size;
+        if (reallyNewLeaves.length === 0) {
+            logger?.info("raptor_incremental_no_new", { sessionId: opts.sessionId });
+            return rehydrateFromStored(storedNodes);
+        }
+        // 4. Fallback guard: >50% new → full rebuild.
+        if (totalCovered > 0 &&
+            reallyNewLeaves.length / totalCovered > FRESH_THRESHOLD) {
+            logger?.info("raptor_incremental_fallback_ratio", {
+                sessionId: opts.sessionId,
+                coveredNodes: totalCovered,
+                newLeaves: reallyNewLeaves.length,
+                threshold: FRESH_THRESHOLD,
+            });
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // 5. Build the in-memory tree from stored nodes.
+        const tree = rehydrateFromStored(storedNodes);
+        if (!tree || tree.nodes.size === 0) {
+            logger?.info("raptor_incremental_rehydrate_failed", {
+                sessionId: opts.sessionId,
+            });
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // Group stored nodes by level.
+        const levelGroups = new Map();
+        for (const n of storedNodes) {
+            const g = levelGroups.get(n.level);
+            if (g)
+                g.push(n);
+            else
+                levelGroups.set(n.level, [n]);
+        }
+        // Level-0 covers leaf ids directly.
+        const level0Nodes = levelGroups.get(0) ??
+            storedNodes.filter((n) => {
+                // A node is level 0 if its level is the minimum.
+                const minLevel = Math.min(...storedNodes.map((x) => x.level));
+                return n.level === minLevel;
+            });
+        if (level0Nodes.length === 0) {
+            // Single-root tree with no hierarchy: cannot increment, fall back.
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // For each new leaf, assign to the nearest Level-0 cluster node.
+        const leafEmbeddings = new Map();
+        for (const l of reallyNewLeaves)
+            leafEmbeddings.set(l.id, l.embedding);
+        // Track which Level-0 nodes changed (by their stored id).
+        const affectedNodeIds = new Set();
+        for (const leaf of reallyNewLeaves) {
+            const emb = leafEmbeddings.get(leaf.id);
+            if (!emb || emb.length === 0)
+                continue;
+            // Find best matching Level-0 node.
+            let bestNode = null;
+            let bestSim = -Infinity;
+            for (const cn of level0Nodes) {
+                const sim = cosineSimilarity(emb, cn.embedding);
+                if (sim > bestSim) {
+                    bestSim = sim;
+                    bestNode = cn;
+                }
+            }
+            if (!bestNode)
+                continue;
+            // Add the leaf id to this node's children.
+            const updated = tree.nodes.get(bestNode.id);
+            if (updated) {
+                if (!updated.children.includes(leaf.id)) {
+                    updated.children.push(leaf.id);
+                }
+                affectedNodeIds.add(bestNode.id);
+            }
+        }
+        if (affectedNodeIds.size === 0) {
+            // No matches found — fall back to full rebuild.
+            logger?.info("raptor_incremental_no_matches", {
+                sessionId: opts.sessionId,
+            });
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // 6. Recompute embeddings and summaries for affected nodes, bottom-up.
+        // Build a map: leafId → leaf info for summarization.
+        const leafMap = new Map();
+        for (const l of existingLeaves)
+            leafMap.set(l.id, l);
+        for (const l of reallyNewLeaves)
+            leafMap.set(l.id, l);
+        // Recompute affected Level-0 nodes.
+        const embedder = opts.embedder ?? getDefaultEmbedder();
+        for (const nid of affectedNodeIds) {
+            const node = tree.nodes.get(nid);
+            if (!node)
+                continue;
+            const coveredLeaves = node.children
+                .map((cid) => leafMap.get(cid))
+                .filter((l) => !!l);
+            if (coveredLeaves.length > 0) {
+                node.embedding = meanVector(coveredLeaves.map((l) => l.embedding));
+                // Summarize: flatten messages from all covered leaves.
+                const messages = coveredLeaves.flatMap((l) => l.messages);
+                if (messages.length > 0) {
+                    const cs = summarizeGuarded(messages, node.embedding, embedder, opts.consistencyThreshold);
+                    node.summary = cs.summary;
+                    node.tokenEstimate = cs.tokenEstimate;
+                }
+            }
+        }
+        // Propagate up: for each higher level, recompute nodes whose leaf set
+        // includes any newly inserted leaf. Since `children` stores flattened leaf
+        // ids (not node ids), we check intersection with the new leaf id set.
+        const newLeafIds = new Set(reallyNewLeaves.map((l) => l.id));
+        const sortedLevels = [...levelGroups.keys()].sort((a, b) => a - b);
+        for (const level of sortedLevels.slice(1)) {
+            const levelNodes = levelGroups.get(level) ?? [];
+            for (const stored of levelNodes) {
+                const node = tree.nodes.get(stored.id);
+                if (!node)
+                    continue;
+                if (node.children.some((cid) => newLeafIds.has(cid))) {
+                    // Recompute embedding as centroid of covered leaves.
+                    const covered = node.children
+                        .map((cid) => leafMap.get(cid))
+                        .filter((l) => !!l);
+                    if (covered.length > 0) {
+                        node.embedding = meanVector(covered.map((l) => l.embedding));
+                        const messages = covered.flatMap((l) => l.messages);
+                        if (messages.length > 0) {
+                            const cs = summarizeGuarded(messages, node.embedding, embedder, opts.consistencyThreshold);
+                            node.summary = cs.summary;
+                            node.tokenEstimate = cs.tokenEstimate;
+                        }
+                    }
+                }
+            }
+        }
+        // 7. Persist the full updated tree (delete + reinsert in a tx).
+        tree.builtAt = builtAt;
+        saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        logger?.info("raptor_incremental_success", {
+            sessionId: opts.sessionId,
+            nodeCount: tree.nodes.size,
+            newLeaves: reallyNewLeaves.length,
+            affectedNodes: affectedNodeIds.size,
+        });
+        return tree;
+    }
+    catch (e) {
+        logger?.error("raptor_incremental_failed", {
+            sessionId: opts.sessionId,
+            error: String(e instanceof Error ? e.message : e),
+        });
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Full rebuild fallback: merges existingLeaves + newLeaves, builds from scratch.
+ */
+function fullRebuild(existingLeaves, newLeaves, opts, logger) {
+    logger?.info("raptor_incremental_full_rebuild", {
+        sessionId: opts.sessionId,
+    });
+    try {
+        // Merge: deduplicate by leaf id.
+        const seen = new Set();
+        const allLeaves = [];
+        for (const l of existingLeaves) {
+            if (!seen.has(l.id)) {
+                seen.add(l.id);
+                allLeaves.push(l);
+            }
+        }
+        for (const l of newLeaves) {
+            if (!seen.has(l.id)) {
+                seen.add(l.id);
+                allLeaves.push(l);
+            }
+        }
+        const tree = buildRaptorTree(allLeaves, {
+            embedder: opts.embedder ?? getDefaultEmbedder(),
+            budgetMs: opts.budgetMs,
+            clustersPerLevel: opts.clustersPerLevel,
+            consistencyThreshold: opts.consistencyThreshold,
+        });
+        const builtAt = opts.builtAt ?? Date.now();
+        saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        return tree;
+    }
+    catch (e2) {
+        logger?.error("raptor_incremental_full_rebuild_failed", {
+            sessionId: opts.sessionId,
+            error: String(e2 instanceof Error ? e2.message : e2),
+        });
+        return null;
+    }
+}
+/** Rebuild the in-memory RaptorTree from stored nodes. */
+function rehydrateFromStored(nodes) {
+    if (nodes.length === 0)
+        return null;
+    const root = nodes.reduce((best, n) => (!best || n.level > (best?.level ?? -1) ? n : best), null);
+    const tree = {
+        nodes: new Map(nodes.map((n) => [
+            n.id,
+            {
+                id: n.id,
+                level: n.level,
+                parentId: n.parentId,
+                children: n.children,
+                summary: n.summary,
+                embedding: n.embedding,
+                qualityMarker: n.qualityMarker,
+                tokenEstimate: n.tokenEstimate,
+            },
+        ])),
+        rootId: root?.id ?? null,
+        levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
+        timedOut: root != null && root.level >= 99,
+        builtAt: nodes.reduce((max, n) => Math.max(max, n.builtAt), 0),
+    };
+    return tree;
+}