pi-mega-compact 0.8.22 → 0.8.23

package/LICENSE

@@ -1,6 +1,6 @@
-BSD 2-Clause License
+BSD 3-Clause License
 
-Copyright (c) 2026 TheArchitectit
+Copyright (c) 2026, TheArchitectit
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
@@ -12,6 +12,10 @@ modification, are permitted provided that the following conditions are met:
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.
 
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

package/README.md

@@ -88,7 +88,7 @@ Testing guide: [`TESTER_GUIDE.md`](TESTER_GUIDE.md)
 
 ## License
 
-BSD-2-Clause
+BSD 3-Clause
 
 ## ☕ Support
 

package/dist/src/dedup/raptor/multilevel.js

@@ -0,0 +1,172 @@
+/**
+ * multilevel.ts — Multi-level RAPTOR retrieval engine (S42A).
+ *
+ * Upgrades the RAPTOR recall path from flat (leaf-only) to multi-level
+ * retrieval across the entire hierarchical tree. Searches ALL levels with
+ * configurable level weights, supports leaf expansion for cluster hits,
+ * and deduplicates overlapping results.
+ *
+ * PREVENT-PI-004: pure in-process math (cosine, BFS, extractive). No network.
+ * PREVENT-PI-001: produces SearchHit[] that feed into recallAndInline() —
+ * affects which checkpoints are recalled, not how messages are dropped.
+ */
+import { cosineSimilarity } from "../../embedder.js";
+import { mmrRerank } from "../mmr.js";
+import { leafDescendants } from "./retrieval.js";
+const DEFAULT_LEVEL_WEIGHTS = [1.0, 0.9, 0.8, 0.7, 0.5];
+// ── S42A-2: Level-weighted scoring ─────────────────────────────────────────
+/**
+ * Score all RAPTOR tree nodes by cosine similarity to the query, then apply
+ * level-specific weights. Returns hits sorted by weighted score descending.
+ *
+ * Level weights: leaves (level 0) get weight 1.0, level 1 gets 0.9, etc.
+ * This ensures detailed leaves score highest while still surfacing higher-level
+ * summaries when they're highly relevant.
+ */
+export function scoreTreeLevels(query, tree, opts) {
+    const { embedder } = opts;
+    const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+    const qv = embedder.embed(query);
+    const hits = [];
+    // 1. Score all internal (summary) nodes.
+    for (const node of tree.nodes.values()) {
+        const rawScore = cosineSimilarity(qv, node.embedding);
+        const levelWeight = weights[Math.min(node.level, weights.length - 1)];
+        hits.push({
+            nodeId: node.id,
+            level: node.level,
+            score: rawScore * levelWeight,
+            rawScore,
+            isLeaf: false,
+            leafIds: node.children,
+            summary: node.summary,
+            embedding: node.embedding,
+        });
+    }
+    // 2. Score leaf nodes. Leaf ids are not in tree.nodes — they are children
+    //    referenced by internal nodes. Each leaf's embedding is the level-0
+    //    parent node that wraps it (same approach as stagedExpansion:95–102).
+    const seenLeaves = new Set();
+    for (const node of tree.nodes.values()) {
+        for (const leafId of node.children) {
+            if (seenLeaves.has(leafId) || tree.nodes.has(leafId))
+                continue;
+            seenLeaves.add(leafId);
+            const rawScore = cosineSimilarity(qv, node.embedding);
+            const leafWeight = weights[0];
+            hits.push({
+                nodeId: leafId,
+                level: 0,
+                score: rawScore * leafWeight,
+                rawScore,
+                isLeaf: true,
+                leafIds: [leafId],
+                summary: "", // leaves have no summary — they are raw checkpoint ids
+                embedding: node.embedding,
+            });
+        }
+    }
+    hits.sort((a, b) => b.score - a.score);
+    return hits;
+}
+// ── S42A-3: Leaf expansion ─────────────────────────────────────────────────
+/**
+ * Given a set of cluster-level hits, expand each one to include its leaf
+ * descendants. Deduplicates: if a leaf is already present as a direct hit,
+ * it is not duplicated. Returns the merged set (original hits + expanded leaves).
+ */
+export function expandLeafDescendants(hits, tree, maxPerCluster, _embedder, queryVector, levelWeights) {
+    const weights = levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+    const existingIds = new Set(hits.map((h) => h.nodeId));
+    const expanded = [];
+    for (const hit of hits) {
+        if (hit.isLeaf) {
+            expanded.push(hit);
+            continue;
+        }
+        // Get all leaf descendants for this cluster node.
+        const node = tree.nodes.get(hit.nodeId);
+        if (!node) {
+            expanded.push(hit);
+            continue;
+        }
+        const rawLeafIds = leafDescendants(node, tree);
+        // Sort by cosine similarity to query, cap at maxPerCluster.
+        const leafHits = rawLeafIds
+            .map((lid) => {
+            // Leaf embedding = its nearest internal parent's embedding.
+            const parent = [...tree.nodes.values()].find((n) => n.children.includes(lid));
+            const sim = parent
+                ? cosineSimilarity(queryVector, parent.embedding)
+                : 0;
+            return { lid, sim, parent };
+        })
+            .sort((a, b) => b.sim - a.sim)
+            .slice(0, maxPerCluster)
+            .filter((l) => !existingIds.has(l.lid))
+            .map((l) => {
+            existingIds.add(l.lid);
+            const rawScore = l.sim;
+            return {
+                nodeId: l.lid,
+                level: 0,
+                score: rawScore * weights[0],
+                rawScore,
+                isLeaf: true,
+                leafIds: [l.lid],
+                summary: "",
+                embedding: l.parent?.embedding ?? hit.embedding,
+            };
+        });
+        expanded.push(hit, ...leafHits);
+    }
+    return expanded;
+}
+// ── S42A-4: Result dedup ───────────────────────────────────────────────────
+/**
+ * Deduplicate hits: if both a cluster node and its leaf children appear in
+ * results, remove the cluster hit (leaves provide more specific context).
+ * If no leaves are in the set, keep the cluster hit (it provides the abstract view).
+ */
+export function deduplicateMultilevelHits(hits) {
+    const leafIds = new Set(hits.filter((h) => h.isLeaf).map((h) => h.nodeId));
+    return hits.filter((h) => {
+        if (h.isLeaf)
+            return true;
+        // Cluster hit: keep only if none of its leaf children are present.
+        return !h.leafIds.some((lid) => leafIds.has(lid));
+    });
+}
+// ── S42A-5: Top-level pipeline ─────────────────────────────────────────────
+/**
+ * Full multi-level retrieval pipeline: score → expand → dedup → MMR → top-K.
+ * Drop-in replacement for `stagedExpansion()` in the RAPTOR recall path.
+ */
+export function multilevelRetrieval(query, tree, opts) {
+    if (!tree.rootId)
+        return [];
+    const { embedder } = opts;
+    const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+    const leafExp = opts.leafExpansion !== false; // default true
+    const maxLeafExp = opts.maxLeafExpansion ?? 10;
+    const k = opts.k ?? 5;
+    const lambda = opts.mmrLambda ?? 0.5;
+    const qv = embedder.embed(query);
+    // 1. Score all nodes with level weights.
+    const scored = scoreTreeLevels(query, tree, { embedder, levelWeights: weights });
+    // 2. Top-N candidates for MMR diversity window.
+    const topN = scored.slice(0, k * 3);
+    // 3. Leaf expansion (optional).
+    const expanded = leafExp
+        ? expandLeafDescendants(topN, tree, maxLeafExp, embedder, qv, weights)
+        : topN;
+    // 4. Dedup: remove cluster hits when leaf children are present.
+    const deduped = deduplicateMultilevelHits(expanded);
+    // 5. MMR rerank to k.
+    const mmrItems = deduped.map((h) => ({
+        item: h,
+        vector: h.embedding,
+        relevance: h.score,
+    }));
+    return mmrRerank(mmrItems, k, lambda);
+}

package/dist/src/dedup/raptor/multilevel.test.js

@@ -0,0 +1,203 @@
+/**
+ * multilevel.test.ts — hermetic unit tests for S42A multi-level RAPTOR retrieval.
+ *
+ * Tests the scoreTreeLevels → expandLeafDescendants → deduplicateMultilevelHits
+ * → multilevelRetrieval pipeline. No network, no live store — uses TrigramEmbedder
+ * and synthetic RaptorTrees built from makeLeaves().
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { TrigramEmbedder } from "../../embedder.js";
+import { buildRaptorTree } from "./tree.js";
+import { scoreTreeLevels, expandLeafDescendants, deduplicateMultilevelHits, multilevelRetrieval, } from "./multilevel.js";
+function msg(text) {
+    return { role: "user", text };
+}
+/** Build N distinct leaves with deterministic content. */
+function makeLeaves(n, embedder = new TrigramEmbedder()) {
+    const leaves = [];
+    for (let i = 0; i < n; i++) {
+        const text = `topic ${i % 7}: the module ${i} validated the session token and refreshed the cache for region ${i}`;
+        leaves.push({
+            id: `leaf_${i}`,
+            messages: [msg(text)],
+            sourceText: text,
+            embedding: embedder.embed(text),
+        });
+    }
+    return leaves;
+}
+// ── test: scoreTreeLevels returns nodes at all levels ────────────────────────
+test("scoreTreeLevels returns results at multiple tree levels", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(50);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+    const hits = scoreTreeLevels("the auth module validates the session token", tree, {
+        embedder,
+    });
+    assert.ok(hits.length > 0, "should return hits");
+    // Should include both leaf (level 0) and cluster (level ≥ 1) hits.
+    const levels = new Set(hits.map((h) => h.level));
+    assert.ok(levels.has(0), "should have leaf-level hits");
+    if (tree.levels > 1) {
+        const hasCluster = [...levels].some((l) => l >= 1);
+        assert.ok(hasCluster, "should have cluster-level hits for multi-level tree");
+    }
+    // All hits should have valid scores.
+    for (const h of hits) {
+        assert.ok(h.score >= 0 && h.score <= 1, `score ${h.score} out of range`);
+        assert.ok(h.rawScore >= 0 && h.rawScore <= 1, `rawScore ${h.rawScore} out of range`);
+        assert.ok(h.score <= h.rawScore, "weighted score <= raw score (level weights ≤ 1)");
+    }
+    // Hits should be sorted by score descending.
+    for (let i = 1; i < hits.length; i++) {
+        assert.ok(hits[i - 1].score >= hits[i].score, `hits not sorted: hit[${i - 1}].score=${hits[i - 1].score} < hit[${i}].score=${hits[i].score}`);
+    }
+});
+// ── test: level weights affect scoring ───────────────────────────────────────
+test("level weights shift scores: higher weight for level → higher weighted score", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(50);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+    const query = "the auth module validates the session token";
+    // Uniform weights: all levels scored equally.
+    const uniform = scoreTreeLevels(query, tree, {
+        embedder,
+        levelWeights: [1.0, 1.0, 1.0, 1.0, 1.0],
+    });
+    // Penalized weights: higher levels penalized.
+    const penalized = scoreTreeLevels(query, tree, {
+        embedder,
+        levelWeights: [1.0, 0.1, 0.1, 0.1, 0.1],
+    });
+    // With penalized weights, cluster-level hits should score lower.
+    const clusterUniform = uniform.filter((h) => h.level >= 1);
+    const clusterPenalized = penalized.filter((h) => h.level >= 1);
+    if (clusterUniform.length > 0 && clusterPenalized.length > 0) {
+        const avgUniform = clusterUniform.reduce((s, h) => s + h.score, 0) / clusterUniform.length;
+        const avgPenalized = clusterPenalized.reduce((s, h) => s + h.score, 0) / clusterPenalized.length;
+        assert.ok(avgPenalized < avgUniform, `penalized cluster avg (${avgPenalized.toFixed(3)}) should be < uniform (${avgUniform.toFixed(3)})`);
+    }
+});
+// ── test: expandLeafDescendants adds leaf hits ──────────────────────────────
+test("expandLeafDescendants adds leaf descendants for cluster hits", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(50);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+    const query = "the auth module validates the session token";
+    const scored = scoreTreeLevels(query, tree, { embedder });
+    const qv = embedder.embed(query);
+    // Take only cluster hits (level ≥ 1).
+    const clusterHits = scored.filter((h) => !h.isLeaf).slice(0, 3);
+    assert.ok(clusterHits.length > 0, "should have cluster hits");
+    const expanded = expandLeafDescendants(clusterHits, tree, 5, // maxPerCluster
+    embedder, qv);
+    // Expanded set should include leaf hits.
+    const leafHits = expanded.filter((h) => h.isLeaf);
+    assert.ok(leafHits.length > 0, "should have expanded leaf hits");
+    assert.ok(expanded.length > clusterHits.length, `expanded (${expanded.length}) should be > cluster hits (${clusterHits.length})`);
+    // No duplicate ids.
+    const ids = new Set(expanded.map((h) => h.nodeId));
+    assert.equal(ids.size, expanded.length, "no duplicate node ids");
+});
+// ── test: deduplicateMultilevelHits removes cluster when leaves present ─────
+test("deduplicateMultilevelHits removes cluster hits when leaf children are present", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(50);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+    const query = "the auth module validates the session token";
+    const scored = scoreTreeLevels(query, tree, { embedder });
+    // Take a cluster hit and its leaf children.
+    const cluster = scored.find((h) => !h.isLeaf);
+    assert.ok(cluster, "should have a cluster hit");
+    const leafChildren = cluster.leafIds.slice(0, 2).map((lid) => ({
+        nodeId: lid,
+        level: 0,
+        score: 0.5,
+        rawScore: 0.5,
+        isLeaf: true,
+        leafIds: [lid],
+        summary: "",
+        embedding: cluster.embedding,
+    }));
+    const mixed = [cluster, ...leafChildren];
+    const deduped = deduplicateMultilevelHits(mixed);
+    // Cluster should be removed because its leaf children are present.
+    assert.ok(!deduped.find((h) => h.nodeId === cluster.nodeId), "cluster hit should be removed when leaf children are present");
+    assert.equal(deduped.length, leafChildren.length, "only leaf hits remain");
+});
+test("deduplicateMultilevelHits keeps cluster hits when no leaf children present", () => {
+    const clusterHit = {
+        nodeId: "cluster_1",
+        level: 1,
+        score: 0.8,
+        rawScore: 0.8,
+        isLeaf: false,
+        leafIds: ["leaf_1", "leaf_2"],
+        summary: "summarized content",
+        embedding: [1, 0, 0],
+    };
+    const deduped = deduplicateMultilevelHits([clusterHit]);
+    assert.equal(deduped.length, 1, "cluster should be kept when no leaf children present");
+    assert.equal(deduped[0].nodeId, "cluster_1");
+});
+// ── test: multilevelRetrieval returns cluster + leaf mix ────────────────────
+test("multilevelRetrieval returns a mix of cluster and leaf hits", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(50);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+    const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+        embedder,
+        k: 5,
+        leafExpansion: true,
+        maxLeafExpansion: 3,
+    });
+    assert.ok(hits.length > 0, "should return hits");
+    assert.ok(hits.length <= 5, "should respect k=5");
+    // Should include leaf hits.
+    const leafHits = hits.filter((h) => h.isLeaf);
+    assert.ok(leafHits.length > 0, "should include leaf hits");
+    // All hits should have valid properties.
+    for (const h of hits) {
+        assert.ok(h.nodeId, "hit should have nodeId");
+        assert.ok(typeof h.level === "number", "hit should have numeric level");
+        assert.ok(h.score >= 0, "hit score should be non-negative");
+        assert.ok(Array.isArray(h.leafIds), "hit should have leafIds array");
+    }
+});
+test("multilevelRetrieval respects k parameter", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(100);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 8 });
+    for (const k of [1, 3, 5, 10]) {
+        const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+            embedder,
+            k,
+        });
+        assert.ok(hits.length <= k, `k=${k}: got ${hits.length} hits, should be ≤ ${k}`);
+    }
+});
+test("multilevelRetrieval returns empty for empty tree", () => {
+    const embedder = new TrigramEmbedder();
+    const emptyTree = { nodes: new Map(), rootId: null, levels: 0, timedOut: false };
+    const hits = multilevelRetrieval("any query", emptyTree, { embedder });
+    assert.deepEqual(hits, []);
+});
+test("multilevelRetrieval with leafExpansion=false skips leaf expansion", () => {
+    const embedder = new TrigramEmbedder();
+    const leaves = makeLeaves(50);
+    const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+    const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+        embedder,
+        k: 5,
+        leafExpansion: false,
+    });
+    assert.ok(hits.length > 0, "should still return hits");
+    // With leaf expansion off and tree having multiple levels, we may get
+    // cluster hits that are NOT expanded. The mix depends on tree structure.
+    // Just verify we got valid results.
+    for (const h of hits) {
+        assert.ok(h.nodeId, "hit should have nodeId");
+        assert.ok(h.score >= 0, "hit score should be non-negative");
+    }
+});

package/dist/src/dedup/raptor/retrieval.js

@@ -18,7 +18,7 @@ function isLeafId(id, tree) {
     return !tree.nodes.has(id);
 }
 /** All leaf (raw) ids reachable beneath a node via BFS. */
-function leafDescendants(node, tree) {
+export function leafDescendants(node, tree) {
     const out = [];
     const queue = [node];
     while (queue.length) {

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.8.22",
+  "version": "0.8.23",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
-  "license": "BSD-2-Clause",
+  "license": "BSD-3-Clause",
   "author": "TheArchitectit",
   "homepage": "https://github.com/TheArchitectit/pi-mega-compact",
   "repository": {

package/src/dedup/raptor/multilevel.test.ts

@@ -0,0 +1,278 @@
+/**
+ * multilevel.test.ts — hermetic unit tests for S42A multi-level RAPTOR retrieval.
+ *
+ * Tests the scoreTreeLevels → expandLeafDescendants → deduplicateMultilevelHits
+ * → multilevelRetrieval pipeline. No network, no live store — uses TrigramEmbedder
+ * and synthetic RaptorTrees built from makeLeaves().
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { TrigramEmbedder } from "../../embedder.js";
+import { buildRaptorTree, type Leaf } from "./tree.js";
+import {
+  scoreTreeLevels,
+  expandLeafDescendants,
+  deduplicateMultilevelHits,
+  multilevelRetrieval,
+  type MultilevelHit,
+} from "./multilevel.js";
+import type { EngineMessage } from "../../types.js";
+
+function msg(text: string): EngineMessage {
+  return { role: "user", text };
+}
+
+/** Build N distinct leaves with deterministic content. */
+function makeLeaves(n: number, embedder = new TrigramEmbedder()): Leaf[] {
+  const leaves: Leaf[] = [];
+  for (let i = 0; i < n; i++) {
+    const text = `topic ${i % 7}: the module ${i} validated the session token and refreshed the cache for region ${i}`;
+    leaves.push({
+      id: `leaf_${i}`,
+      messages: [msg(text)],
+      sourceText: text,
+      embedding: embedder.embed(text),
+    });
+  }
+  return leaves;
+}
+
+// ── test: scoreTreeLevels returns nodes at all levels ────────────────────────
+
+test("scoreTreeLevels returns results at multiple tree levels", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const hits = scoreTreeLevels("the auth module validates the session token", tree, {
+    embedder,
+  });
+
+  assert.ok(hits.length > 0, "should return hits");
+
+  // Should include both leaf (level 0) and cluster (level ≥ 1) hits.
+  const levels = new Set(hits.map((h) => h.level));
+  assert.ok(levels.has(0), "should have leaf-level hits");
+  if (tree.levels > 1) {
+    const hasCluster = [...levels].some((l) => l >= 1);
+    assert.ok(hasCluster, "should have cluster-level hits for multi-level tree");
+  }
+
+  // All hits should have valid scores.
+  for (const h of hits) {
+    assert.ok(h.score >= 0 && h.score <= 1, `score ${h.score} out of range`);
+    assert.ok(h.rawScore >= 0 && h.rawScore <= 1, `rawScore ${h.rawScore} out of range`);
+    assert.ok(h.score <= h.rawScore, "weighted score <= raw score (level weights ≤ 1)");
+  }
+
+  // Hits should be sorted by score descending.
+  for (let i = 1; i < hits.length; i++) {
+    assert.ok(
+      hits[i - 1].score >= hits[i].score,
+      `hits not sorted: hit[${i - 1}].score=${hits[i - 1].score} < hit[${i}].score=${hits[i].score}`,
+    );
+  }
+});
+
+// ── test: level weights affect scoring ───────────────────────────────────────
+
+test("level weights shift scores: higher weight for level → higher weighted score", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const query = "the auth module validates the session token";
+
+  // Uniform weights: all levels scored equally.
+  const uniform = scoreTreeLevels(query, tree, {
+    embedder,
+    levelWeights: [1.0, 1.0, 1.0, 1.0, 1.0],
+  });
+
+  // Penalized weights: higher levels penalized.
+  const penalized = scoreTreeLevels(query, tree, {
+    embedder,
+    levelWeights: [1.0, 0.1, 0.1, 0.1, 0.1],
+  });
+
+  // With penalized weights, cluster-level hits should score lower.
+  const clusterUniform = uniform.filter((h) => h.level >= 1);
+  const clusterPenalized = penalized.filter((h) => h.level >= 1);
+
+  if (clusterUniform.length > 0 && clusterPenalized.length > 0) {
+    const avgUniform =
+      clusterUniform.reduce((s, h) => s + h.score, 0) / clusterUniform.length;
+    const avgPenalized =
+      clusterPenalized.reduce((s, h) => s + h.score, 0) / clusterPenalized.length;
+    assert.ok(
+      avgPenalized < avgUniform,
+      `penalized cluster avg (${avgPenalized.toFixed(3)}) should be < uniform (${avgUniform.toFixed(3)})`,
+    );
+  }
+});
+
+// ── test: expandLeafDescendants adds leaf hits ──────────────────────────────
+
+test("expandLeafDescendants adds leaf descendants for cluster hits", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const query = "the auth module validates the session token";
+  const scored = scoreTreeLevels(query, tree, { embedder });
+  const qv = embedder.embed(query);
+
+  // Take only cluster hits (level ≥ 1).
+  const clusterHits = scored.filter((h) => !h.isLeaf).slice(0, 3);
+  assert.ok(clusterHits.length > 0, "should have cluster hits");
+
+  const expanded = expandLeafDescendants(
+    clusterHits,
+    tree,
+    5, // maxPerCluster
+    embedder,
+    qv,
+  );
+
+  // Expanded set should include leaf hits.
+  const leafHits = expanded.filter((h) => h.isLeaf);
+  assert.ok(leafHits.length > 0, "should have expanded leaf hits");
+  assert.ok(
+    expanded.length > clusterHits.length,
+    `expanded (${expanded.length}) should be > cluster hits (${clusterHits.length})`,
+  );
+
+  // No duplicate ids.
+  const ids = new Set(expanded.map((h) => h.nodeId));
+  assert.equal(ids.size, expanded.length, "no duplicate node ids");
+});
+
+// ── test: deduplicateMultilevelHits removes cluster when leaves present ─────
+
+test("deduplicateMultilevelHits removes cluster hits when leaf children are present", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const query = "the auth module validates the session token";
+  const scored = scoreTreeLevels(query, tree, { embedder });
+
+  // Take a cluster hit and its leaf children.
+  const cluster = scored.find((h) => !h.isLeaf);
+  assert.ok(cluster, "should have a cluster hit");
+
+  const leafChildren: MultilevelHit[] = cluster!.leafIds.slice(0, 2).map((lid) => ({
+    nodeId: lid,
+    level: 0,
+    score: 0.5,
+    rawScore: 0.5,
+    isLeaf: true,
+    leafIds: [lid],
+    summary: "",
+    embedding: cluster!.embedding,
+  }));
+
+  const mixed = [cluster!, ...leafChildren];
+  const deduped = deduplicateMultilevelHits(mixed);
+
+  // Cluster should be removed because its leaf children are present.
+  assert.ok(
+    !deduped.find((h) => h.nodeId === cluster!.nodeId),
+    "cluster hit should be removed when leaf children are present",
+  );
+  assert.equal(deduped.length, leafChildren.length, "only leaf hits remain");
+});
+
+test("deduplicateMultilevelHits keeps cluster hits when no leaf children present", () => {
+  const clusterHit: MultilevelHit = {
+    nodeId: "cluster_1",
+    level: 1,
+    score: 0.8,
+    rawScore: 0.8,
+    isLeaf: false,
+    leafIds: ["leaf_1", "leaf_2"],
+    summary: "summarized content",
+    embedding: [1, 0, 0],
+  };
+
+  const deduped = deduplicateMultilevelHits([clusterHit]);
+  assert.equal(deduped.length, 1, "cluster should be kept when no leaf children present");
+  assert.equal(deduped[0].nodeId, "cluster_1");
+});
+
+// ── test: multilevelRetrieval returns cluster + leaf mix ────────────────────
+
+test("multilevelRetrieval returns a mix of cluster and leaf hits", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+    embedder,
+    k: 5,
+    leafExpansion: true,
+    maxLeafExpansion: 3,
+  });
+
+  assert.ok(hits.length > 0, "should return hits");
+  assert.ok(hits.length <= 5, "should respect k=5");
+
+  // Should include leaf hits.
+  const leafHits = hits.filter((h) => h.isLeaf);
+  assert.ok(leafHits.length > 0, "should include leaf hits");
+
+  // All hits should have valid properties.
+  for (const h of hits) {
+    assert.ok(h.nodeId, "hit should have nodeId");
+    assert.ok(typeof h.level === "number", "hit should have numeric level");
+    assert.ok(h.score >= 0, "hit score should be non-negative");
+    assert.ok(Array.isArray(h.leafIds), "hit should have leafIds array");
+  }
+});
+
+test("multilevelRetrieval respects k parameter", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(100);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 8 });
+
+  for (const k of [1, 3, 5, 10]) {
+    const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+      embedder,
+      k,
+    });
+    assert.ok(
+      hits.length <= k,
+      `k=${k}: got ${hits.length} hits, should be ≤ ${k}`,
+    );
+  }
+});
+
+test("multilevelRetrieval returns empty for empty tree", () => {
+  const embedder = new TrigramEmbedder();
+  const emptyTree = { nodes: new Map(), rootId: null, levels: 0, timedOut: false };
+  const hits = multilevelRetrieval("any query", emptyTree, { embedder });
+  assert.deepEqual(hits, []);
+});
+
+test("multilevelRetrieval with leafExpansion=false skips leaf expansion", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+    embedder,
+    k: 5,
+    leafExpansion: false,
+  });
+
+  assert.ok(hits.length > 0, "should still return hits");
+
+  // With leaf expansion off and tree having multiple levels, we may get
+  // cluster hits that are NOT expanded. The mix depends on tree structure.
+  // Just verify we got valid results.
+  for (const h of hits) {
+    assert.ok(h.nodeId, "hit should have nodeId");
+    assert.ok(h.score >= 0, "hit score should be non-negative");
+  }
+});

package/src/dedup/raptor/multilevel.ts

@@ -0,0 +1,246 @@
+/**
+ * multilevel.ts — Multi-level RAPTOR retrieval engine (S42A).
+ *
+ * Upgrades the RAPTOR recall path from flat (leaf-only) to multi-level
+ * retrieval across the entire hierarchical tree. Searches ALL levels with
+ * configurable level weights, supports leaf expansion for cluster hits,
+ * and deduplicates overlapping results.
+ *
+ * PREVENT-PI-004: pure in-process math (cosine, BFS, extractive). No network.
+ * PREVENT-PI-001: produces SearchHit[] that feed into recallAndInline() —
+ * affects which checkpoints are recalled, not how messages are dropped.
+ */
+
+import type { Embedder, Vector } from "../../embedder.js";
+import { cosineSimilarity } from "../../embedder.js";
+import { mmrRerank } from "../mmr.js";
+import type { RaptorTree } from "./tree.js";
+import { leafDescendants } from "./retrieval.js";
+
+// ── S42A-1: Types ──────────────────────────────────────────────────────────
+
+export interface MultilevelRetrieveOptions {
+  embedder: Embedder;
+  /** Weight per tree level (index 0 = leaves, index 1 = level 1, etc.).
+   *  Default: [1.0, 0.9, 0.8, 0.7, 0.5]. Capped at tree depth.
+   *  UNCALIBRATED — requires real-data calibration before stable. */
+  levelWeights?: number[];
+  /** When true, expand cluster hits to include leaf descendants. Default: true. */
+  leafExpansion?: boolean;
+  /** Max leaf descendants to fetch per cluster hit. Default: 10. */
+  maxLeafExpansion?: number;
+  /** Final number of results to return. Default: 5. */
+  k?: number;
+  /** MMR diversity weight. Default: 0.5. */
+  mmrLambda?: number;
+}
+
+export interface MultilevelHit {
+  nodeId: string;
+  level: number;
+  /** Weighted score after level weighting. */
+  score: number;
+  /** Raw cosine similarity before level weighting. */
+  rawScore: number;
+  isLeaf: boolean;
+  /** Leaf ids covered by this node (for leaf expansion). */
+  leafIds: string[];
+  summary: string;
+  embedding: Vector;
+}
+
+const DEFAULT_LEVEL_WEIGHTS = [1.0, 0.9, 0.8, 0.7, 0.5];
+
+// ── S42A-2: Level-weighted scoring ─────────────────────────────────────────
+
+/**
+ * Score all RAPTOR tree nodes by cosine similarity to the query, then apply
+ * level-specific weights. Returns hits sorted by weighted score descending.
+ *
+ * Level weights: leaves (level 0) get weight 1.0, level 1 gets 0.9, etc.
+ * This ensures detailed leaves score highest while still surfacing higher-level
+ * summaries when they're highly relevant.
+ */
+export function scoreTreeLevels(
+  query: string,
+  tree: RaptorTree,
+  opts: Pick<MultilevelRetrieveOptions, "embedder" | "levelWeights">,
+): MultilevelHit[] {
+  const { embedder } = opts;
+  const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+  const qv = embedder.embed(query);
+  const hits: MultilevelHit[] = [];
+
+  // 1. Score all internal (summary) nodes.
+  for (const node of tree.nodes.values()) {
+    const rawScore = cosineSimilarity(qv, node.embedding);
+    const levelWeight = weights[Math.min(node.level, weights.length - 1)];
+    hits.push({
+      nodeId: node.id,
+      level: node.level,
+      score: rawScore * levelWeight,
+      rawScore,
+      isLeaf: false,
+      leafIds: node.children,
+      summary: node.summary,
+      embedding: node.embedding,
+    });
+  }
+
+  // 2. Score leaf nodes. Leaf ids are not in tree.nodes — they are children
+  //    referenced by internal nodes. Each leaf's embedding is the level-0
+  //    parent node that wraps it (same approach as stagedExpansion:95–102).
+  const seenLeaves = new Set<string>();
+  for (const node of tree.nodes.values()) {
+    for (const leafId of node.children) {
+      if (seenLeaves.has(leafId) || tree.nodes.has(leafId)) continue;
+      seenLeaves.add(leafId);
+      const rawScore = cosineSimilarity(qv, node.embedding);
+      const leafWeight = weights[0];
+      hits.push({
+        nodeId: leafId,
+        level: 0,
+        score: rawScore * leafWeight,
+        rawScore,
+        isLeaf: true,
+        leafIds: [leafId],
+        summary: "", // leaves have no summary — they are raw checkpoint ids
+        embedding: node.embedding,
+      });
+    }
+  }
+
+  hits.sort((a, b) => b.score - a.score);
+  return hits;
+}
+
+// ── S42A-3: Leaf expansion ─────────────────────────────────────────────────
+
+/**
+ * Given a set of cluster-level hits, expand each one to include its leaf
+ * descendants. Deduplicates: if a leaf is already present as a direct hit,
+ * it is not duplicated. Returns the merged set (original hits + expanded leaves).
+ */
+export function expandLeafDescendants(
+  hits: MultilevelHit[],
+  tree: RaptorTree,
+  maxPerCluster: number,
+  _embedder: Embedder,
+  queryVector: Vector,
+  levelWeights?: number[],
+): MultilevelHit[] {
+  const weights = levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+  const existingIds = new Set(hits.map((h) => h.nodeId));
+  const expanded: MultilevelHit[] = [];
+
+  for (const hit of hits) {
+    if (hit.isLeaf) {
+      expanded.push(hit);
+      continue;
+    }
+
+    // Get all leaf descendants for this cluster node.
+    const node = tree.nodes.get(hit.nodeId);
+    if (!node) {
+      expanded.push(hit);
+      continue;
+    }
+
+    const rawLeafIds = leafDescendants(node, tree);
+
+    // Sort by cosine similarity to query, cap at maxPerCluster.
+    const leafHits: MultilevelHit[] = rawLeafIds
+      .map((lid) => {
+        // Leaf embedding = its nearest internal parent's embedding.
+        const parent = [...tree.nodes.values()].find((n) =>
+          n.children.includes(lid),
+        );
+        const sim = parent
+          ? cosineSimilarity(queryVector, parent.embedding)
+          : 0;
+        return { lid, sim, parent };
+      })
+      .sort((a, b) => b.sim - a.sim)
+      .slice(0, maxPerCluster)
+      .filter((l) => !existingIds.has(l.lid))
+      .map((l) => {
+        existingIds.add(l.lid);
+        const rawScore = l.sim;
+        return {
+          nodeId: l.lid,
+          level: 0,
+          score: rawScore * weights[0],
+          rawScore,
+          isLeaf: true,
+          leafIds: [l.lid],
+          summary: "",
+          embedding: l.parent?.embedding ?? hit.embedding,
+        } as MultilevelHit;
+      });
+
+    expanded.push(hit, ...leafHits);
+  }
+
+  return expanded;
+}
+
+// ── S42A-4: Result dedup ───────────────────────────────────────────────────
+
+/**
+ * Deduplicate hits: if both a cluster node and its leaf children appear in
+ * results, remove the cluster hit (leaves provide more specific context).
+ * If no leaves are in the set, keep the cluster hit (it provides the abstract view).
+ */
+export function deduplicateMultilevelHits(hits: MultilevelHit[]): MultilevelHit[] {
+  const leafIds = new Set(hits.filter((h) => h.isLeaf).map((h) => h.nodeId));
+  return hits.filter((h) => {
+    if (h.isLeaf) return true;
+    // Cluster hit: keep only if none of its leaf children are present.
+    return !h.leafIds.some((lid) => leafIds.has(lid));
+  });
+}
+
+// ── S42A-5: Top-level pipeline ─────────────────────────────────────────────
+
+/**
+ * Full multi-level retrieval pipeline: score → expand → dedup → MMR → top-K.
+ * Drop-in replacement for `stagedExpansion()` in the RAPTOR recall path.
+ */
+export function multilevelRetrieval(
+  query: string,
+  tree: RaptorTree,
+  opts: MultilevelRetrieveOptions,
+): MultilevelHit[] {
+  if (!tree.rootId) return [];
+
+  const { embedder } = opts;
+  const weights = opts.levelWeights ?? DEFAULT_LEVEL_WEIGHTS;
+  const leafExp = opts.leafExpansion !== false; // default true
+  const maxLeafExp = opts.maxLeafExpansion ?? 10;
+  const k = opts.k ?? 5;
+  const lambda = opts.mmrLambda ?? 0.5;
+
+  const qv = embedder.embed(query);
+
+  // 1. Score all nodes with level weights.
+  const scored = scoreTreeLevels(query, tree, { embedder, levelWeights: weights });
+
+  // 2. Top-N candidates for MMR diversity window.
+  const topN = scored.slice(0, k * 3);
+
+  // 3. Leaf expansion (optional).
+  const expanded = leafExp
+    ? expandLeafDescendants(topN, tree, maxLeafExp, embedder, qv, weights)
+    : topN;
+
+  // 4. Dedup: remove cluster hits when leaf children are present.
+  const deduped = deduplicateMultilevelHits(expanded);
+
+  // 5. MMR rerank to k.
+  const mmrItems = deduped.map((h) => ({
+    item: h,
+    vector: h.embedding as Vector,
+    relevance: h.score,
+  }));
+  return mmrRerank(mmrItems, k, lambda);
+}

package/src/dedup/raptor/retrieval.ts

@@ -33,7 +33,7 @@ function isLeafId(id: string, tree: RaptorTree): boolean {
 }
 
 /** All leaf (raw) ids reachable beneath a node via BFS. */
-function leafDescendants(node: RaptorNode, tree: RaptorTree): string[] {
+export function leafDescendants(node: RaptorNode, tree: RaptorTree): string[] {
   const out: string[] = [];
   const queue = [node];
   while (queue.length) {