pi-mega-compact 0.17.0 → 0.18.0

package/dist/dedup/raptor/index.js

@@ -0,0 +1,190 @@
+/**
+ * index.ts — RAPTOR orchestrator (Sprint 13, Phase 6 → promoted to live in S42B).
+ *
+ * Entry point that turns a session's leaves into a hierarchical summary tree.
+ * The tree is BUILT, PERSISTED to raptor_nodes, and SERVED in recall queries
+ * when RAPTOR_ENABLED=true (default). Set RAPTOR_SHADOW_MODE=true to build +
+ * persist only without serving (transition/eval use).
+ *
+ * PREVENT-PI-004: no network here. Any Ollama call lives in summarizer.ts
+ * (localhost-only, annotated). This module is pure orchestration.
+ */
+import { defaultEmbedder } from "../../embedder.js";
+import { buildRaptorTree } from "./tree.js";
+import { incrementRaptorTree } from "./incremental.js";
+import { stagedExpansion } from "./retrieval.js";
+import { saveRaptorTree, listRaptorNodes } from "../../store/sqlite.js";
+import { insertBuildHistory, computeCoherenceScore } from "./buildHistory.js";
+import { loadDedupConfig } from "../../config/dedup.js";
+/**
+ * Shadow mode default OFF: the tree is built, persisted, AND served when
+ * RAPTOR_ENABLED=true (the Setup tab toggle). Set RAPTOR_SHADOW_MODE=true to
+ * build + persist only (transition/eval use without serving live recall).
+ */
+export function isShadowMode() {
+    return process.env.RAPTOR_SHADOW_MODE === "true";
+}
+/**
+ * Build the RAPTOR tree for a session's leaves. Per shadow-mode rules, the tree
+ * is persisted + logged regardless; whether it is served is the caller's choice
+ * (Sprint 13: it is built but NOT injected into recallAndInline).
+ *
+ * Returns the built tree (in-memory) for eval/tests, and persists it to the
+ * store. Never throws — on any build error it logs and returns null.
+ */
+export function runRaptor(leaves, opts) {
+    const embedder = opts.embedder ?? defaultEmbedder();
+    const logger = opts.logger;
+    const startedAt = opts.builtAt ?? Date.now();
+    const builtAt = startedAt;
+    try {
+        // Try incremental update first when the flag is ON (Sprint 26, #7).
+        // The incremental path reads the existing tree, diffs by leaf id, and
+        // inserts only new leaves into the nearest clusters. Falls back to a full
+        // rebuild when the tree is missing, corrupted, or >50% of nodes are new.
+        const cfg = loadDedupConfig();
+        let tree = null;
+        if (cfg.RAPTOR_INCREMENTAL) {
+            tree = incrementRaptorTree(leaves, leaves, {
+                embedder,
+                stateDir: opts.stateDir,
+                sessionId: opts.sessionId,
+                budgetMs: opts.budgetMs,
+                clustersPerLevel: opts.clustersPerLevel,
+                consistencyThreshold: opts.consistencyThreshold,
+                logger,
+                builtAt,
+            });
+        }
+        // Fall back to full rebuild when incremental returned null (not attempted,
+        // tree missing, ratio exceeded, or internal error).
+        if (!tree) {
+            tree = buildRaptorTree(leaves, {
+                embedder,
+                budgetMs: opts.budgetMs,
+                clustersPerLevel: opts.clustersPerLevel,
+                consistencyThreshold: opts.consistencyThreshold,
+            });
+            saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        }
+        logger?.info("raptor_build", {
+            sessionId: opts.sessionId,
+            nodes: tree.nodes.size,
+            levels: tree.levels,
+            rootId: tree.rootId,
+            timedOut: tree.timedOut,
+            shadow: isShadowMode(),
+        });
+        // S42D: record structured build history (coherence score + leaf count) so
+        // the freshness check can skip the next rebuild when the tree is recent
+        // and the chunk count is stable. Best-effort: a history-write failure must
+        // never break the build.
+        try {
+            // Map leaf id → embedding so computeCoherenceScore can score real
+            // intra-cluster spread (without this, leaves collapse to parent centroids
+            // and the score inflates to ~1.0 for every tree).
+            const leafEmbs = new Map();
+            for (const l of leaves)
+                if (l.embedding.length > 0)
+                    leafEmbs.set(l.id, l.embedding);
+            const coherence = computeCoherenceScore(tree, leafEmbs);
+            insertBuildHistory({
+                sessionId: opts.sessionId,
+                stateDir: opts.stateDir,
+                startedAt,
+                completedAt: Date.now(),
+                nodeCount: tree.nodes.size,
+                leafCount: leaves.length,
+                depth: tree.levels,
+                configJson: JSON.stringify({
+                    budgetMs: opts.budgetMs,
+                    clustersPerLevel: opts.clustersPerLevel,
+                    consistencyThreshold: opts.consistencyThreshold,
+                }),
+                coherenceScore: coherence,
+                timedOut: tree.timedOut,
+            }, opts.stateDir);
+        }
+        catch (histErr) {
+            logger?.warn("raptor_build_history_failed", {
+                sessionId: opts.sessionId,
+                error: String(histErr instanceof Error ? histErr.message : histErr),
+            });
+        }
+        if (isShadowMode()) {
+            // Build + log only. Do NOT replace retrieval.
+            logger?.info("raptor_shadow", { sessionId: opts.sessionId, served: false });
+        }
+        return tree;
+    }
+    catch (e) {
+        logger?.error("raptor_build_failed", {
+            sessionId: opts.sessionId,
+            error: String(e instanceof Error ? e.message : e),
+        });
+        return null;
+    }
+}
+/**
+ * Staged retrieval over a persisted/session tree. Only meaningful when RAPTOR
+ * is promoted (Sprint 14); provided here so eval can measure it in shadow.
+ *
+ * Returns the leaf ids the staged expansion would serve for `query`.
+ */
+export function recallRaptor(query, sessionId, opts) {
+    const embedder = opts.embedder ?? defaultEmbedder();
+    const tree = rehydrateRaptorTree(sessionId, opts.stateDir);
+    if (!tree)
+        return [];
+    return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM });
+}
+/**
+ * Rehydrate a persisted RAPTOR tree from raptor_nodes (Fix D): rebuild the
+ * in-memory RaptorTree + parent links so vectorStore.search can serve it live.
+ * Returns null when no tree exists (caller falls back to the flat path).
+ */
+export function rehydrateRaptorTree(sessionId, stateDir) {
+    const nodes = listRaptorNodes(sessionId, stateDir);
+    if (nodes.length === 0)
+        return null;
+    // S25: derive freshness + fallback metadata from the persisted nodes.
+    // builtAt = max node built_at (0 when unknown → caller treats as stale).
+    // timedOut = the tree's root is the extractive-fallback marker (level 99).
+    const builtAt = nodes.reduce((max, n) => Math.max(max, n.builtAt), 0);
+    const root = nodes.reduce((best, n) => (!best || n.level > (best?.level ?? -1) ? n : best), null);
+    const timedOut = root != null && root.level >= 99;
+    const tree = {
+        nodes: new Map(nodes.map((n) => [
+            n.id,
+            {
+                id: n.id,
+                level: n.level,
+                parentId: n.parentId,
+                children: n.children,
+                summary: n.summary,
+                embedding: n.embedding,
+                qualityMarker: n.qualityMarker,
+                tokenEstimate: n.tokenEstimate,
+            },
+        ])),
+        rootId: root?.id ?? null,
+        levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
+        timedOut,
+        builtAt,
+    };
+    return tree;
+}
+/**
+ * Return the RAPTOR root summary for a session, if a tree has been built.
+ * Used by the durable-trim driver (Fix B/D) to supply pi a session-level
+ * compressed summary instead of one slice's extractive summary. Returns
+ * undefined when no tree exists yet (caller falls back to the slice summary).
+ */
+export function recallRaptorRootSummary(sessionId, stateDir) {
+    const nodes = listRaptorNodes(sessionId, stateDir);
+    if (nodes.length === 0)
+        return undefined;
+    // Highest-level node = the root (covers all leaves).
+    const root = nodes.reduce((best, n) => (!best || n.level > best.level ? n : best), null);
+    return root?.summary || undefined;
+}

package/dist/dedup/raptor/kmeans.js

@@ -0,0 +1,152 @@
+/**
+ * kmeans.ts — k-means++ clustering over embeddings (Sprint 13, RAPTOR Phase 6).
+ *
+ * Pure TS, zero deps, deterministic (seeded). Used to group leaf chunks into
+ * nodes for the RAPTOR summary tree. QA #11: GMM is the spec's long-term target
+ * for cosine space; k-means++ is the shippable local default, with a
+ * near-zero-variance merge guard so degenerate inputs can't spin forever.
+ *
+ * PREVENT-PI-004: math only, no network, no model.
+ */
+import { cosineSimilarity } from "../../embedder.js";
+/** Seeded PRNG (mulberry32) so clustering is deterministic across runs. */
+function rng(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a |= 0;
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/** L2 distance squared between two equal-length vectors. */
+function dist2(a, b) {
+    let s = 0;
+    for (let i = 0; i < a.length; i++) {
+        const d = a[i] - b[i];
+        s += d * d;
+    }
+    return s;
+}
+/**
+ * k-means++ clustering. Deterministic given `seed`.
+ *
+ * Near-zero-variance guard (QA #11): if the max pairwise distance among input
+ * points is below `varianceFloor`, all points are effectively identical — return
+ * a single cluster (the mean) instead of iterating. Prevents the centroid-init
+ * from dividing by ~0 and keeps degenerate sessions from spinning.
+ */
+export function kmeanspp(points, k, opts = {}) {
+    const n = points.length;
+    const seed = opts.seed ?? 0x9e3779b9;
+    const maxIter = opts.maxIter ?? 25;
+    const varianceFloor = opts.varianceFloor ?? 1e-12;
+    if (n === 0)
+        return { assignments: [], centroids: [], k: 0 };
+    if (n === 1)
+        return { assignments: [0], centroids: [points[0].slice()], k: 1 };
+    // Near-zero-variance merge guard.
+    let maxPair = 0;
+    for (let i = 0; i < n; i++) {
+        for (let j = i + 1; j < n; j++) {
+            const d = dist2(points[i], points[j]);
+            if (d > maxPair)
+                maxPair = d;
+        }
+    }
+    if (maxPair < varianceFloor) {
+        const mean = new Array(points[0].length).fill(0);
+        for (const p of points)
+            for (let i = 0; i < mean.length; i++)
+                mean[i] += p[i] / n;
+        return { assignments: new Array(n).fill(0), centroids: [mean], k: 1 };
+    }
+    const kk = Math.min(k, n);
+    const rand = rng(seed);
+    // k-means++ seeding: first centroid random, rest weighted by squared distance.
+    const centroids = [points[Math.floor(rand() * n)].slice()];
+    const d2 = new Array(n).fill(Infinity);
+    while (centroids.length < kk) {
+        let sum = 0;
+        for (let i = 0; i < n; i++) {
+            const d = dist2(points[i], centroids[centroids.length - 1]);
+            if (d < d2[i])
+                d2[i] = d;
+            sum += d2[i];
+        }
+        if (sum === 0) {
+            // All remaining points coincide with an existing centroid.
+            for (const p of points)
+                if (!centroids.some((c) => dist2(c, p) < 1e-12))
+                    centroids.push(p.slice());
+            break;
+        }
+        let target = rand() * sum;
+        let chosen = 0;
+        for (let i = 0; i < n; i++) {
+            target -= d2[i];
+            if (target <= 0) {
+                chosen = i;
+                break;
+            }
+        }
+        centroids.push(points[chosen].slice());
+    }
+    const assignments = new Array(n).fill(0);
+    for (let iter = 0; iter < maxIter; iter++) {
+        let changed = false;
+        // Assign each point to its nearest centroid.
+        for (let i = 0; i < n; i++) {
+            let best = 0;
+            let bestD = Infinity;
+            for (let c = 0; c < centroids.length; c++) {
+                const d = dist2(points[i], centroids[c]);
+                if (d < bestD) {
+                    bestD = d;
+                    best = c;
+                }
+            }
+            if (assignments[i] !== best) {
+                assignments[i] = best;
+                changed = true;
+            }
+        }
+        // Recompute centroids as the mean of assigned points.
+        const sums = centroids.map(() => new Array(points[0].length).fill(0));
+        const counts = new Array(centroids.length).fill(0);
+        for (let i = 0; i < n; i++) {
+            const c = assignments[i];
+            counts[c]++;
+            for (let d = 0; d < points[i].length; d++)
+                sums[c][d] += points[i][d];
+        }
+        for (let c = 0; c < centroids.length; c++) {
+            if (counts[c] === 0)
+                continue; // keep prior centroid if a cluster emptied
+            centroids[c] = sums[c].map((s) => s / counts[c]);
+        }
+        if (!changed && iter > 0)
+            break;
+    }
+    return { assignments, centroids, k: centroids.length };
+}
+/** Mean embedding of a set of vectors (used to reduce a cluster to its centroid). */
+export function meanVector(vectors) {
+    if (vectors.length === 0)
+        return [];
+    const dim = vectors[0].length;
+    const sum = new Array(dim).fill(0);
+    for (const v of vectors)
+        for (let i = 0; i < dim; i++)
+            sum[i] += v[i];
+    return sum.map((s) => s / vectors.length);
+}
+/** Centroid of the union of two clusters (cosine-space "midpoint" via mean). */
+export function mergeCentroids(a, b) {
+    return meanVector([a, b]);
+}
+/** Cosine distance (1 - cosine) — used by tree.ts for budget/quality checks. */
+export function cosineDistance(a, b) {
+    return 1 - cosineSimilarity(a, b);
+}

package/dist/dedup/raptor/multilevel.js

@@ -0,0 +1,184 @@
+/**
+ * multilevel.ts — Multi-level RAPTOR retrieval engine (S42A).
+ *
+ * Upgrades the RAPTOR recall path from flat (leaf-only) to multi-level
+ * retrieval across the entire hierarchical tree. Searches ALL levels with
+ * configurable level weights, supports leaf expansion for cluster hits,
+ * and deduplicates overlapping results.
+ *
+ * PREVENT-PI-004: pure in-process math (cosine, BFS, extractive). No network.
+ * PREVENT-PI-001: produces SearchHit[] that feed into recallAndInline() —
+ * affects which checkpoints are recalled, not how messages are dropped.
+ */
+import { cosineSimilarity } from "../../embedder.js";
+import { mmrRerank } from "../mmr.js";
+import { leafDescendants } from "./retrieval.js";
+const DEFAULT_LEVEL_WEIGHTS = [1.0, 0.9, 0.8, 0.7, 0.5];
+// ── S42A-2: Level-weighted scoring ─────────────────────────────────────────
+/**
+ * Score all RAPTOR tree nodes by cosine similarity to the query, then apply
+ * level-specific weights. Returns hits sorted by weighted score descending.
+ *
+ * Level weights: leaves (level 0) get weight 1.0, level 1 gets 0.9, etc.
+ * This ensures detailed leaves score highest while still surfacing higher-level
+ * summaries when they're highly relevant.
+ */
+export function scoreTreeLevels(query, tree, opts) {
+    const { embedder } = opts;
+    const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+    const qv = embedder.embed(query);
+    const hits = [];
+    // 1. Score all internal (summary) nodes.
+    for (const node of tree.nodes.values()) {
+        const rawScore = cosineSimilarity(qv, node.embedding);
+        const levelWeight = weights[Math.min(node.level, weights.length - 1)];
+        hits.push({
+            nodeId: node.id,
+            level: node.level,
+            score: rawScore * levelWeight,
+            rawScore,
+            isLeaf: false,
+            leafIds: node.children,
+            summary: node.summary,
+            embedding: node.embedding,
+        });
+    }
+    // 2. Score leaf nodes. Leaf ids are not in tree.nodes — they are children
+    //    referenced by internal nodes. Each leaf's embedding is the level-0
+    //    parent node that wraps it (same approach as stagedExpansion:95–102).
+    const seenLeaves = new Set();
+    for (const node of tree.nodes.values()) {
+        for (const leafId of node.children) {
+            if (seenLeaves.has(leafId) || tree.nodes.has(leafId))
+                continue;
+            seenLeaves.add(leafId);
+            const rawScore = cosineSimilarity(qv, node.embedding);
+            const leafWeight = weights[0];
+            hits.push({
+                nodeId: leafId,
+                level: 0,
+                score: rawScore * leafWeight,
+                rawScore,
+                isLeaf: true,
+                leafIds: [leafId],
+                summary: "", // leaves have no summary — they are raw checkpoint ids
+                embedding: node.embedding,
+            });
+        }
+    }
+    hits.sort((a, b) => b.score - a.score);
+    return hits;
+}
+// ── S42A-3: Leaf expansion ─────────────────────────────────────────────────
+/** Build a reverse index: childId → parent RaptorNode. O(N). */
+export function buildChildParentIndex(tree) {
+    const idx = new Map();
+    for (const node of tree.nodes.values()) {
+        for (const cid of node.children) {
+            if (!idx.has(cid))
+                idx.set(cid, node);
+        }
+    }
+    return idx;
+}
+/**
+ * Given a set of cluster-level hits, expand each one to include its leaf
+ * descendants. Deduplicates: if a leaf is already present as a direct hit,
+ * it is not duplicated. Returns the merged set (original hits + expanded leaves).
+ */
+export function expandLeafDescendants(hits, tree, maxPerCluster, _embedder, queryVector, levelWeights) {
+    const weights = levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+    const existingIds = new Set(hits.map((h) => h.nodeId));
+    const childParentIdx = buildChildParentIndex(tree);
+    const expanded = [];
+    for (const hit of hits) {
+        if (hit.isLeaf) {
+            expanded.push(hit);
+            continue;
+        }
+        // Get all leaf descendants for this cluster node.
+        const node = tree.nodes.get(hit.nodeId);
+        if (!node) {
+            expanded.push(hit);
+            continue;
+        }
+        const rawLeafIds = leafDescendants(node, tree);
+        // Sort by cosine similarity to query, cap at maxPerCluster.
+        // Skip leaves with no parent (orphan) — they have no reliable embedding.
+        const leafHits = rawLeafIds
+            .map((lid) => {
+            const parent = childParentIdx.get(lid);
+            const sim = parent
+                ? cosineSimilarity(queryVector, parent.embedding)
+                : -Infinity; // orphan: excluded
+            return { lid, sim, parent };
+        })
+            .filter((l) => l.sim > -Infinity && !existingIds.has(l.lid))
+            .sort((a, b) => b.sim - a.sim)
+            .slice(0, maxPerCluster)
+            .map((l) => {
+            existingIds.add(l.lid);
+            const rawScore = l.sim;
+            return {
+                nodeId: l.lid,
+                level: 0,
+                score: rawScore * weights[0],
+                rawScore,
+                isLeaf: true,
+                leafIds: [l.lid],
+                summary: "",
+                embedding: l.parent.embedding, // parent guaranteed non-null here
+            };
+        });
+        expanded.push(hit, ...leafHits);
+    }
+    return expanded;
+}
+// ── S42A-4: Result dedup ───────────────────────────────────────────────────
+/**
+ * Deduplicate hits: if both a cluster node and its leaf children appear in
+ * results, remove the cluster hit (leaves provide more specific context).
+ * If no leaves are in the set, keep the cluster hit (it provides the abstract view).
+ */
+export function deduplicateMultilevelHits(hits) {
+    const leafIds = new Set(hits.filter((h) => h.isLeaf).map((h) => h.nodeId));
+    return hits.filter((h) => {
+        if (h.isLeaf)
+            return true;
+        // Cluster hit: keep only if none of its leaf children are present.
+        return !h.leafIds.some((lid) => leafIds.has(lid));
+    });
+}
+// ── S42A-5: Top-level pipeline ─────────────────────────────────────────────
+/**
+ * Full multi-level retrieval pipeline: score → expand → dedup → MMR → top-K.
+ * Drop-in replacement for `stagedExpansion()` in the RAPTOR recall path.
+ */
+export function multilevelRetrieval(query, tree, opts) {
+    if (!tree.rootId)
+        return [];
+    const { embedder } = opts;
+    const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+    const leafExp = opts.leafExpansion !== false; // default true
+    const maxLeafExp = opts.maxLeafExpansion ?? 10;
+    const k = opts.k ?? 5;
+    const lambda = opts.mmrLambda ?? 0.5;
+    const qv = embedder.embed(query);
+    // 1. Score all nodes with level weights.
+    const scored = scoreTreeLevels(query, tree, { embedder, levelWeights: weights });
+    // 2. Top-N candidates for MMR diversity window.
+    const topN = scored.slice(0, k * 3);
+    // 3. Leaf expansion (optional).
+    const expanded = leafExp
+        ? expandLeafDescendants(topN, tree, maxLeafExp, embedder, qv, weights)
+        : topN;
+    // 4. Dedup: remove cluster hits when leaf children are present.
+    const deduped = deduplicateMultilevelHits(expanded);
+    // 5. MMR rerank to k.
+    const mmrItems = deduped.map((h) => ({
+        item: h,
+        vector: h.embedding,
+        relevance: h.score,
+    }));
+    return mmrRerank(mmrItems, k, lambda);
+}

package/dist/dedup/raptor/retrieval.js

@@ -0,0 +1,93 @@
+/**
+ * retrieval.ts — staged RAPTOR retrieval (Sprint 13, Phase 6).
+ *
+ * Given a built tree (shadow or live), answer a query by:
+ *   1. ANN: score the top-level nodes at every level by cosine to the query.
+ *   2. expand: take the top-M nodes (across levels) and descend.
+ *   3. BFS: from those anchors, walk down to leaf nodes.
+ *   4. MMR: diversify the resulting leaf set before returning.
+ *
+ * This module is pure query logic over an in-memory RaptorTree. In Sprint 13 it
+ * is exercised only in shadow/eval; the live store (vectorStore.search) is NOT
+ * replaced until Sprint 14 promotes RAPTOR.
+ */
+import { cosineSimilarity } from "../../embedder.js";
+import { mmrRerank } from "../mmr.js";
+/** Any child id not present in the node map is a raw leaf id. */
+function isLeafId(id, tree) {
+    return !tree.nodes.has(id);
+}
+/** Build a reverse index: childId → parent RaptorNode. O(N). */
+export function buildChildParentIndex(tree) {
+    const idx = new Map();
+    for (const node of tree.nodes.values()) {
+        for (const cid of node.children) {
+            if (!idx.has(cid))
+                idx.set(cid, node);
+        }
+    }
+    return idx;
+}
+/** All leaf (raw) ids reachable beneath a node via BFS. */
+export function leafDescendants(node, tree) {
+    const out = [];
+    const queue = [node];
+    while (queue.length) {
+        const cur = queue.shift();
+        for (const childId of cur.children) {
+            if (isLeafId(childId, tree))
+                out.push(childId);
+            else {
+                const child = tree.nodes.get(childId);
+                if (child)
+                    queue.push(child);
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Staged expansion retrieval over a RAPTOR tree.
+ *
+ * Returns up to `k` leaf ids, diversified by MMR. Deterministic given the
+ * tree + query. Throws nothing — returns [] on empty tree.
+ */
+export function stagedExpansion(query, tree, opts) {
+    if (!tree.rootId)
+        return [];
+    const embedder = opts.embedder;
+    const topM = opts.topM ?? 3;
+    const k = opts.k ?? 5;
+    const lambda = opts.mmrLambda ?? 0.5;
+    const qv = embedder.embed(query);
+    // 1. ANN: score every node at every level by cosine to the query.
+    const scored = [...tree.nodes.values()].map((n) => ({
+        node: n,
+        score: cosineSimilarity(qv, n.embedding),
+    }));
+    // 2. expand: top-M nodes overall (BFS anchors).
+    const anchors = scored
+        .slice()
+        .sort((a, b) => b.score - a.score)
+        .slice(0, topM)
+        .map((s) => s.node);
+    // 3. BFS to leaves from those anchors, using a reverse child→parent index.
+    const childParentIdx = buildChildParentIndex(tree);
+    const leaves = new Map();
+    for (const a of anchors) {
+        for (const lid of leafDescendants(a, tree)) {
+            // Represent each leaf by its nearest internal parent so we can score it.
+            // (The leaf's own centroid is stored on the level-0 node that wraps it.)
+            const parent = childParentIdx.get(lid);
+            if (parent)
+                leaves.set(lid, parent);
+        }
+    }
+    // 4. MMR diversify the expanded leaf set by their (parent) embeddings.
+    const items = [...leaves.entries()].map(([lid, n]) => ({
+        item: lid,
+        vector: n.embedding,
+        relevance: cosineSimilarity(qv, n.embedding),
+    }));
+    return mmrRerank(items, k, lambda);
+}