pi-mega-compact 0.11.9 → 0.11.10

package/dist/extensions/mega-events/context-handler.js

@@ -240,7 +240,7 @@ export function registerContextHandler(pi, runtime, config) {
             if (runtime.trimCache &&
                 runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
                 runtime.trimCache.cut <= messages.length) {
-                const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002
+                const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
                 runtime.diagLiveTrimFires++;
                 runtime.diagLiveTrimReplays++;
                 runtime.snapshot(ctx);

package/dist/src/config/dedup.js

@@ -64,6 +64,7 @@ export function loadDedupConfig() {
         RAPTOR_MULTILEVEL_ENABLED: envBool("MEGACOMPACT_RAPTOR_MULTILEVEL", true),
         RAPTOR_LEVEL_WEIGHTS: envNumArray("MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS", [1.0, 0.9, 0.8, 0.7, 0.5]),
         RAPTOR_LEAF_EXPANSION: envBool("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", true),
+        RAPTOR_INCREMENTAL: envBool("MEGACOMPACT_RAPTOR_INCREMENTAL", true),
         RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
         RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
         RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),

package/dist/src/dedup/raptor/incremental.js

@@ -0,0 +1,276 @@
+/**
+ * incremental.ts — incremental RAPTOR tree update (Sprint 26, #7).
+ *
+ * Instead of rebuilding the full tree (~2s per compaction), insert only the new
+ * checkpoints since the last build and recompute only the affected cluster
+ * assignments up the tree. Falls back to a full rebuild when the tree is
+ * missing, corrupted, or >50% of nodes are new.
+ *
+ * Non-fatal: every failure logs and returns null (caller falls back to full
+ * rebuild). PREVENT-PI-004: zero network; all operations are local SQLite + CPU.
+ */
+import { cosineSimilarity } from "../../embedder.js";
+import { defaultEmbedder as getDefaultEmbedder } from "../../embedder.js";
+import { buildRaptorTree } from "./tree.js";
+import { meanVector } from "./kmeans.js";
+import { summarizeCluster } from "./summarizer.js";
+import { saveRaptorTree, listRaptorNodes, } from "../../store/sqlite.js";
+/** When >FRESH_THRESHOLD of existing leaves are new, a full rebuild is cheaper. */
+const FRESH_THRESHOLD = 0.5;
+/**
+ * Incrementally update a persisted RAPTOR tree with new leaves.
+ *
+ * 1. Reads the existing tree from the store.
+ * 2. Filters `newLeaves` to those NOT already covered by the tree.
+ * 3. If no new leaves → early-return (existing tree).
+ * 4. If no existing tree or >50% new → full rebuild.
+ * 5. For each new leaf, finds the best-matching Level-0 cluster node,
+ *    reassigns (inserts into its children), updates that node's summary
+ *    and centroid embedding.
+ * 6. Propagates changes up through parent levels to the root.
+ * 7. Persists the updated tree (full delete + reinsert in a tx).
+ *
+ * Never throws. Returns the updated tree, or null on failure/fallback.
+ */
+export function incrementRaptorTree(existingLeaves, newLeaves, opts) {
+    const logger = opts.logger;
+    const builtAt = opts.builtAt ?? Date.now();
+    try {
+        // 1. Read existing tree.
+        const storedNodes = listRaptorNodes(opts.sessionId, opts.stateDir);
+        if (storedNodes.length === 0) {
+            logger?.info("raptor_incremental_no_tree", { sessionId: opts.sessionId });
+            return null; // no existing tree — caller falls back to full buildRaptorTree
+        }
+        // 2. Find new leaf ids not already in the tree.
+        const coveredLeafIds = new Set();
+        for (const n of storedNodes) {
+            for (const cid of n.children)
+                coveredLeafIds.add(cid);
+        }
+        const reallyNewLeaves = newLeaves.filter((l) => !coveredLeafIds.has(l.id));
+        const totalCovered = coveredLeafIds.size;
+        if (reallyNewLeaves.length === 0) {
+            logger?.info("raptor_incremental_no_new", { sessionId: opts.sessionId });
+            return rehydrateFromStored(storedNodes);
+        }
+        // 4. Fallback guard: >50% new → full rebuild.
+        if (totalCovered > 0 &&
+            reallyNewLeaves.length / totalCovered > FRESH_THRESHOLD) {
+            logger?.info("raptor_incremental_fallback_ratio", {
+                sessionId: opts.sessionId,
+                coveredNodes: totalCovered,
+                newLeaves: reallyNewLeaves.length,
+                threshold: FRESH_THRESHOLD,
+            });
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // 5. Build the in-memory tree from stored nodes.
+        const tree = rehydrateFromStored(storedNodes);
+        if (!tree || tree.nodes.size === 0) {
+            logger?.info("raptor_incremental_rehydrate_failed", {
+                sessionId: opts.sessionId,
+            });
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // Group stored nodes by level.
+        const levelGroups = new Map();
+        for (const n of storedNodes) {
+            const g = levelGroups.get(n.level);
+            if (g)
+                g.push(n);
+            else
+                levelGroups.set(n.level, [n]);
+        }
+        // Level-0 covers leaf ids directly.
+        const level0Nodes = levelGroups.get(0) ??
+            storedNodes.filter((n) => {
+                // A node is level 0 if its level is the minimum.
+                const minLevel = Math.min(...storedNodes.map((x) => x.level));
+                return n.level === minLevel;
+            });
+        if (level0Nodes.length === 0) {
+            // Single-root tree with no hierarchy: cannot increment, fall back.
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // For each new leaf, assign to the nearest Level-0 cluster node.
+        const leafEmbeddings = new Map();
+        for (const l of reallyNewLeaves)
+            leafEmbeddings.set(l.id, l.embedding);
+        // Track which Level-0 nodes changed (by their stored id).
+        const affectedNodeIds = new Set();
+        for (const leaf of reallyNewLeaves) {
+            const emb = leafEmbeddings.get(leaf.id);
+            if (!emb || emb.length === 0)
+                continue;
+            // Find best matching Level-0 node.
+            let bestNode = null;
+            let bestSim = -Infinity;
+            for (const cn of level0Nodes) {
+                const sim = cosineSimilarity(emb, cn.embedding);
+                if (sim > bestSim) {
+                    bestSim = sim;
+                    bestNode = cn;
+                }
+            }
+            if (!bestNode)
+                continue;
+            // Add the leaf id to this node's children.
+            const updated = tree.nodes.get(bestNode.id);
+            if (updated) {
+                if (!updated.children.includes(leaf.id)) {
+                    updated.children.push(leaf.id);
+                }
+                affectedNodeIds.add(bestNode.id);
+            }
+        }
+        if (affectedNodeIds.size === 0) {
+            // No matches found — fall back to full rebuild.
+            logger?.info("raptor_incremental_no_matches", {
+                sessionId: opts.sessionId,
+            });
+            return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+        }
+        // 6. Recompute embeddings and summaries for affected nodes, bottom-up.
+        // Build a map: leafId → leaf info for summarization.
+        const leafMap = new Map();
+        for (const l of existingLeaves)
+            leafMap.set(l.id, l);
+        for (const l of reallyNewLeaves)
+            leafMap.set(l.id, l);
+        // Recompute affected Level-0 nodes.
+        for (const nid of affectedNodeIds) {
+            const node = tree.nodes.get(nid);
+            if (!node)
+                continue;
+            const coveredLeaves = node.children
+                .map((cid) => leafMap.get(cid))
+                .filter((l) => !!l);
+            if (coveredLeaves.length > 0) {
+                node.embedding = meanVector(coveredLeaves.map((l) => l.embedding));
+                // Summarize: flatten messages from all covered leaves.
+                const messages = coveredLeaves.flatMap((l) => l.messages);
+                if (messages.length > 0) {
+                    const cs = summarizeCluster(messages);
+                    node.summary = cs.summary;
+                    node.tokenEstimate = cs.tokenEstimate;
+                }
+            }
+        }
+        // Propagate up: for each higher level, recompute nodes whose leaf set
+        // includes any newly inserted leaf. Since `children` stores flattened leaf
+        // ids (not node ids), we check intersection with the new leaf id set.
+        const newLeafIds = new Set(reallyNewLeaves.map((l) => l.id));
+        const sortedLevels = [...levelGroups.keys()].sort((a, b) => a - b);
+        for (const level of sortedLevels.slice(1)) {
+            const levelNodes = levelGroups.get(level) ?? [];
+            for (const stored of levelNodes) {
+                const node = tree.nodes.get(stored.id);
+                if (!node)
+                    continue;
+                if (node.children.some((cid) => newLeafIds.has(cid))) {
+                    // Recompute embedding as centroid of covered leaves.
+                    const covered = node.children
+                        .map((cid) => leafMap.get(cid))
+                        .filter((l) => !!l);
+                    if (covered.length > 0) {
+                        node.embedding = meanVector(covered.map((l) => l.embedding));
+                        const messages = covered.flatMap((l) => l.messages);
+                        if (messages.length > 0) {
+                            const cs = summarizeCluster(messages);
+                            node.summary = cs.summary;
+                            node.tokenEstimate = cs.tokenEstimate;
+                        }
+                    }
+                }
+            }
+        }
+        // 7. Persist the full updated tree (delete + reinsert in a tx).
+        tree.builtAt = builtAt;
+        saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        logger?.info("raptor_incremental_success", {
+            sessionId: opts.sessionId,
+            nodeCount: tree.nodes.size,
+            newLeaves: reallyNewLeaves.length,
+            affectedNodes: affectedNodeIds.size,
+        });
+        return tree;
+    }
+    catch (e) {
+        logger?.error("raptor_incremental_failed", {
+            sessionId: opts.sessionId,
+            error: String(e instanceof Error ? e.message : e),
+        });
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Full rebuild fallback: merges existingLeaves + newLeaves, builds from scratch.
+ */
+function fullRebuild(existingLeaves, newLeaves, opts, logger) {
+    logger?.info("raptor_incremental_full_rebuild", {
+        sessionId: opts.sessionId,
+    });
+    try {
+        // Merge: deduplicate by leaf id.
+        const seen = new Set();
+        const allLeaves = [];
+        for (const l of existingLeaves) {
+            if (!seen.has(l.id)) {
+                seen.add(l.id);
+                allLeaves.push(l);
+            }
+        }
+        for (const l of newLeaves) {
+            if (!seen.has(l.id)) {
+                seen.add(l.id);
+                allLeaves.push(l);
+            }
+        }
+        const tree = buildRaptorTree(allLeaves, {
+            embedder: opts.embedder ?? getDefaultEmbedder(),
+            budgetMs: opts.budgetMs,
+            clustersPerLevel: opts.clustersPerLevel,
+            consistencyThreshold: opts.consistencyThreshold,
+        });
+        const builtAt = opts.builtAt ?? Date.now();
+        saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        return tree;
+    }
+    catch (e2) {
+        logger?.error("raptor_incremental_full_rebuild_failed", {
+            sessionId: opts.sessionId,
+            error: String(e2 instanceof Error ? e2.message : e2),
+        });
+        return null;
+    }
+}
+/** Rebuild the in-memory RaptorTree from stored nodes. */
+function rehydrateFromStored(nodes) {
+    if (nodes.length === 0)
+        return null;
+    const root = nodes.reduce((best, n) => (!best || n.level > (best?.level ?? -1) ? n : best), null);
+    const tree = {
+        nodes: new Map(nodes.map((n) => [
+            n.id,
+            {
+                id: n.id,
+                level: n.level,
+                parentId: n.parentId,
+                children: n.children,
+                summary: n.summary,
+                embedding: n.embedding,
+                qualityMarker: n.qualityMarker,
+                tokenEstimate: n.tokenEstimate,
+            },
+        ])),
+        rootId: root?.id ?? null,
+        levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
+        timedOut: root != null && root.level >= 99,
+        builtAt: nodes.reduce((max, n) => Math.max(max, n.builtAt), 0),
+    };
+    return tree;
+}

package/dist/src/dedup/raptor/index.js

@@ -11,9 +11,11 @@
  */
 import { defaultEmbedder } from "../../embedder.js";
 import { buildRaptorTree } from "./tree.js";
+import { incrementRaptorTree } from "./incremental.js";
 import { stagedExpansion } from "./retrieval.js";
 import { saveRaptorTree, listRaptorNodes } from "../../store/sqlite.js";
 import { insertBuildHistory, computeCoherenceScore } from "./buildHistory.js";
+import { loadDedupConfig } from "../../config/dedup.js";
 /** Shadow mode is on by default; set RAPTOR_SHADOW_MODE=false to serve live. */
 export function isShadowMode() {
     return process.env.RAPTOR_SHADOW_MODE !== "false";
@@ -30,15 +32,37 @@ export function runRaptor(leaves, opts) {
     const embedder = opts.embedder ?? defaultEmbedder();
     const logger = opts.logger;
     const startedAt = opts.builtAt ?? Date.now();
+    const builtAt = startedAt;
     try {
-        const tree = buildRaptorTree(leaves, {
-            embedder,
-            budgetMs: opts.budgetMs,
-            clustersPerLevel: opts.clustersPerLevel,
-            consistencyThreshold: opts.consistencyThreshold,
-        });
-        const builtAt = opts.builtAt ?? Date.now();
-        saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        // Try incremental update first when the flag is ON (Sprint 26, #7).
+        // The incremental path reads the existing tree, diffs by leaf id, and
+        // inserts only new leaves into the nearest clusters. Falls back to a full
+        // rebuild when the tree is missing, corrupted, or >50% of nodes are new.
+        const cfg = loadDedupConfig();
+        let tree = null;
+        if (cfg.RAPTOR_INCREMENTAL) {
+            tree = incrementRaptorTree(leaves, leaves, {
+                embedder,
+                stateDir: opts.stateDir,
+                sessionId: opts.sessionId,
+                budgetMs: opts.budgetMs,
+                clustersPerLevel: opts.clustersPerLevel,
+                consistencyThreshold: opts.consistencyThreshold,
+                logger,
+                builtAt,
+            });
+        }
+        // Fall back to full rebuild when incremental returned null (not attempted,
+        // tree missing, ratio exceeded, or internal error).
+        if (!tree) {
+            tree = buildRaptorTree(leaves, {
+                embedder,
+                budgetMs: opts.budgetMs,
+                clustersPerLevel: opts.clustersPerLevel,
+                consistencyThreshold: opts.consistencyThreshold,
+            });
+            saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+        }
         logger?.info("raptor_build", {
             sessionId: opts.sessionId,
             nodes: tree.nodes.size,

package/dist/src/dedup/raptor/summarizer.js

@@ -17,7 +17,7 @@ function ollamaEndpoint() {
     const model = process.env.MEGACOMPACT_RAPTOR_MODEL;
     if (!model)
         return null;
-    const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434";
+    const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434"; // guardrails-allow PREVENT-PI-004: loopback-only Ollama endpoint for local RAPTOR summarization
     // Guard: only loopback is permitted (remote Ollama would violate PREVENT-PI-004).
     if (!/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/.test(base)) {
         throw new Error(`MEGACOMPACT_RAPTOR_URL must be localhost/127.0.0.1 (got ${base}). ` +

package/dist/src/httpEmbedder.js

@@ -26,7 +26,7 @@
  */
 import { l2Normalize } from "./embedder.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
-import { isIP } from "node:net";
+import { isIP } from "node:net"; // guardrails-allow PREVENT-PI-004: localhost-only loopback address validation in BYO embedding server
 // Inline worker script: resolves a hostname via dns.lookup in a child process
 // (dns.lookup is callback-async; the child has its own event loop). Used to
 // verify that a hostname in the embedding URL resolves to loopback ONLY.

package/extensions/mega-events/context-handler.ts

@@ -293,7 +293,7 @@ export function registerContextHandler(
 				runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
 				runtime.trimCache.cut <= messages.length
 			) {
-				const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002
+				const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
 				runtime.diagLiveTrimFires++;
 				runtime.diagLiveTrimReplays++;
 				runtime.snapshot(ctx);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.11.9",
+  "version": "0.11.10",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/dedup.ts

@@ -66,6 +66,7 @@ export interface DedupConfigShape {
   RAPTOR_MULTILEVEL_ENABLED: boolean;
   RAPTOR_LEVEL_WEIGHTS: number[]; // per-level weights, index 0 = leaves (uncalibrated)
   RAPTOR_LEAF_EXPANSION: boolean;
+  RAPTOR_INCREMENTAL: boolean;
   RAPTOR_MAX_LEAF_EXPANSION: number; // uncalibrated
   RAPTOR_FRESHNESS_HOURS: number; // S42D: skip rebuild when tree is fresh (uncalibrated)
   /** S25 Phase-2: inject top-level RAPTOR summary nodes (root + level-1
@@ -109,6 +110,7 @@ export function loadDedupConfig(): DedupConfigShape {
     RAPTOR_MULTILEVEL_ENABLED: envBool("MEGACOMPACT_RAPTOR_MULTILEVEL", true),
     RAPTOR_LEVEL_WEIGHTS: envNumArray("MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS", [1.0, 0.9, 0.8, 0.7, 0.5]),
     RAPTOR_LEAF_EXPANSION: envBool("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", true),
+    RAPTOR_INCREMENTAL: envBool("MEGACOMPACT_RAPTOR_INCREMENTAL", true),
     RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
     RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
     RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),

package/src/dedup/raptor/incremental.ts

@@ -0,0 +1,339 @@
+/**
+ * incremental.ts — incremental RAPTOR tree update (Sprint 26, #7).
+ *
+ * Instead of rebuilding the full tree (~2s per compaction), insert only the new
+ * checkpoints since the last build and recompute only the affected cluster
+ * assignments up the tree. Falls back to a full rebuild when the tree is
+ * missing, corrupted, or >50% of nodes are new.
+ *
+ * Non-fatal: every failure logs and returns null (caller falls back to full
+ * rebuild). PREVENT-PI-004: zero network; all operations are local SQLite + CPU.
+ */
+
+import type { Embedder, Vector } from "../../embedder.js";
+import { cosineSimilarity } from "../../embedder.js";
+import { defaultEmbedder as getDefaultEmbedder } from "../../embedder.js";
+import { buildRaptorTree, type Leaf, type RaptorTree } from "./tree.js";
+import type { QualityMarker } from "./guardrails.js";
+import { meanVector } from "./kmeans.js";
+import { summarizeCluster } from "./summarizer.js";
+import type { Logger } from "../../log.js";
+import {
+  saveRaptorTree,
+  listRaptorNodes,
+  type StoredRaptorNode,
+} from "../../store/sqlite.js";
+
+/** When >FRESH_THRESHOLD of existing leaves are new, a full rebuild is cheaper. */
+const FRESH_THRESHOLD = 0.5;
+
+/**
+ * Incrementally update a persisted RAPTOR tree with new leaves.
+ *
+ * 1. Reads the existing tree from the store.
+ * 2. Filters `newLeaves` to those NOT already covered by the tree.
+ * 3. If no new leaves → early-return (existing tree).
+ * 4. If no existing tree or >50% new → full rebuild.
+ * 5. For each new leaf, finds the best-matching Level-0 cluster node,
+ *    reassigns (inserts into its children), updates that node's summary
+ *    and centroid embedding.
+ * 6. Propagates changes up through parent levels to the root.
+ * 7. Persists the updated tree (full delete + reinsert in a tx).
+ *
+ * Never throws. Returns the updated tree, or null on failure/fallback.
+ */
+export function incrementRaptorTree(
+  existingLeaves: Leaf[],
+  newLeaves: Leaf[],
+  opts: {
+    embedder?: Embedder;
+    stateDir: string;
+    sessionId: string;
+    budgetMs?: number;
+    clustersPerLevel?: number;
+    consistencyThreshold?: number;
+    logger?: Logger;
+    builtAt?: number;
+  },
+): RaptorTree | null {
+  const logger = opts.logger;
+  const builtAt = opts.builtAt ?? Date.now();
+
+  try {
+    // 1. Read existing tree.
+    const storedNodes = listRaptorNodes(opts.sessionId, opts.stateDir);
+    if (storedNodes.length === 0) {
+      logger?.info("raptor_incremental_no_tree", { sessionId: opts.sessionId });
+      return null; // no existing tree — caller falls back to full buildRaptorTree
+    }
+
+    // 2. Find new leaf ids not already in the tree.
+    const coveredLeafIds = new Set<string>();
+    for (const n of storedNodes) {
+      for (const cid of n.children) coveredLeafIds.add(cid);
+    }
+    const reallyNewLeaves = newLeaves.filter(
+      (l) => !coveredLeafIds.has(l.id),
+    );
+    const totalCovered = coveredLeafIds.size;
+
+    if (reallyNewLeaves.length === 0) {
+      logger?.info("raptor_incremental_no_new", { sessionId: opts.sessionId });
+      return rehydrateFromStored(storedNodes);
+    }
+
+    // 4. Fallback guard: >50% new → full rebuild.
+    if (
+      totalCovered > 0 &&
+      reallyNewLeaves.length / totalCovered > FRESH_THRESHOLD
+    ) {
+      logger?.info("raptor_incremental_fallback_ratio", {
+        sessionId: opts.sessionId,
+        coveredNodes: totalCovered,
+        newLeaves: reallyNewLeaves.length,
+        threshold: FRESH_THRESHOLD,
+      });
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // 5. Build the in-memory tree from stored nodes.
+    const tree = rehydrateFromStored(storedNodes);
+    if (!tree || tree.nodes.size === 0) {
+      logger?.info("raptor_incremental_rehydrate_failed", {
+        sessionId: opts.sessionId,
+      });
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // Group stored nodes by level.
+    const levelGroups = new Map<number, StoredRaptorNode[]>();
+    for (const n of storedNodes) {
+      const g = levelGroups.get(n.level);
+      if (g) g.push(n);
+      else levelGroups.set(n.level, [n]);
+    }
+    // Level-0 covers leaf ids directly.
+    const level0Nodes =
+      levelGroups.get(0) ??
+      storedNodes.filter((n) => {
+        // A node is level 0 if its level is the minimum.
+        const minLevel = Math.min(...storedNodes.map((x) => x.level));
+        return n.level === minLevel;
+      });
+    if (level0Nodes.length === 0) {
+      // Single-root tree with no hierarchy: cannot increment, fall back.
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // For each new leaf, assign to the nearest Level-0 cluster node.
+    const leafEmbeddings = new Map<string, Vector>();
+    for (const l of reallyNewLeaves) leafEmbeddings.set(l.id, l.embedding);
+
+    // Track which Level-0 nodes changed (by their stored id).
+    const affectedNodeIds = new Set<string>();
+
+    for (const leaf of reallyNewLeaves) {
+      const emb = leafEmbeddings.get(leaf.id);
+      if (!emb || emb.length === 0) continue;
+
+      // Find best matching Level-0 node.
+      let bestNode: StoredRaptorNode | null = null;
+      let bestSim = -Infinity;
+      for (const cn of level0Nodes) {
+        const sim = cosineSimilarity(emb, cn.embedding);
+        if (sim > bestSim) {
+          bestSim = sim;
+          bestNode = cn;
+        }
+      }
+      if (!bestNode) continue;
+
+      // Add the leaf id to this node's children.
+      const updated = tree.nodes.get(bestNode.id);
+      if (updated) {
+        if (!updated.children.includes(leaf.id)) {
+          updated.children.push(leaf.id);
+        }
+        affectedNodeIds.add(bestNode.id);
+      }
+    }
+
+    if (affectedNodeIds.size === 0) {
+      // No matches found — fall back to full rebuild.
+      logger?.info("raptor_incremental_no_matches", {
+        sessionId: opts.sessionId,
+      });
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // 6. Recompute embeddings and summaries for affected nodes, bottom-up.
+    // Build a map: leafId → leaf info for summarization.
+    const leafMap = new Map<string, Leaf>();
+    for (const l of existingLeaves) leafMap.set(l.id, l);
+    for (const l of reallyNewLeaves) leafMap.set(l.id, l);
+
+    // Recompute affected Level-0 nodes.
+    for (const nid of affectedNodeIds) {
+      const node = tree.nodes.get(nid);
+      if (!node) continue;
+      const coveredLeaves = node.children
+        .map((cid) => leafMap.get(cid))
+        .filter((l): l is Leaf => !!l);
+      if (coveredLeaves.length > 0) {
+        node.embedding = meanVector(
+          coveredLeaves.map((l) => l.embedding),
+        );
+        // Summarize: flatten messages from all covered leaves.
+        const messages = coveredLeaves.flatMap((l) => l.messages);
+        if (messages.length > 0) {
+          const cs = summarizeCluster(messages);
+          node.summary = cs.summary;
+          node.tokenEstimate = cs.tokenEstimate;
+        }
+      }
+    }
+
+    // Propagate up: for each higher level, recompute nodes whose leaf set
+    // includes any newly inserted leaf. Since `children` stores flattened leaf
+    // ids (not node ids), we check intersection with the new leaf id set.
+    const newLeafIds = new Set(reallyNewLeaves.map((l) => l.id));
+    const sortedLevels = [...levelGroups.keys()].sort((a, b) => a - b);
+    for (const level of sortedLevels.slice(1)) {
+      const levelNodes = levelGroups.get(level) ?? [];
+      for (const stored of levelNodes) {
+        const node = tree.nodes.get(stored.id);
+        if (!node) continue;
+
+        if (node.children.some((cid) => newLeafIds.has(cid))) {
+          // Recompute embedding as centroid of covered leaves.
+          const covered = node.children
+            .map((cid) => leafMap.get(cid))
+            .filter((l): l is Leaf => !!l);
+          if (covered.length > 0) {
+            node.embedding = meanVector(
+              covered.map((l) => l.embedding),
+            );
+            const messages = covered.flatMap((l) => l.messages);
+            if (messages.length > 0) {
+              const cs = summarizeCluster(messages);
+              node.summary = cs.summary;
+              node.tokenEstimate = cs.tokenEstimate;
+            }
+          }
+        }
+      }
+    }
+
+    // 7. Persist the full updated tree (delete + reinsert in a tx).
+    tree.builtAt = builtAt;
+    saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+
+    logger?.info("raptor_incremental_success", {
+      sessionId: opts.sessionId,
+      nodeCount: tree.nodes.size,
+      newLeaves: reallyNewLeaves.length,
+      affectedNodes: affectedNodeIds.size,
+    });
+
+    return tree;
+  } catch (e) {
+    logger?.error("raptor_incremental_failed", {
+      sessionId: opts.sessionId,
+      error: String(e instanceof Error ? e.message : e),
+    });
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Full rebuild fallback: merges existingLeaves + newLeaves, builds from scratch.
+ */
+function fullRebuild(
+  existingLeaves: Leaf[],
+  newLeaves: Leaf[],
+  opts: {
+    embedder?: Embedder;
+    budgetMs?: number;
+    clustersPerLevel?: number;
+    consistencyThreshold?: number;
+    stateDir: string;
+    sessionId: string;
+    logger?: Logger;
+    builtAt?: number;
+  },
+  logger?: Logger,
+): RaptorTree | null {
+  logger?.info("raptor_incremental_full_rebuild", {
+    sessionId: opts.sessionId,
+  });
+  try {
+    // Merge: deduplicate by leaf id.
+    const seen = new Set<string>();
+    const allLeaves: Leaf[] = [];
+    for (const l of existingLeaves) {
+      if (!seen.has(l.id)) {
+        seen.add(l.id);
+        allLeaves.push(l);
+      }
+    }
+    for (const l of newLeaves) {
+      if (!seen.has(l.id)) {
+        seen.add(l.id);
+        allLeaves.push(l);
+      }
+    }
+    const tree = buildRaptorTree(allLeaves, {
+      embedder: opts.embedder ?? getDefaultEmbedder(),
+      budgetMs: opts.budgetMs,
+      clustersPerLevel: opts.clustersPerLevel,
+      consistencyThreshold: opts.consistencyThreshold,
+    });
+    const builtAt = opts.builtAt ?? Date.now();
+    saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+    return tree;
+  } catch (e2) {
+    logger?.error("raptor_incremental_full_rebuild_failed", {
+      sessionId: opts.sessionId,
+      error: String(e2 instanceof Error ? e2.message : e2),
+    });
+    return null;
+  }
+}
+
+/** Rebuild the in-memory RaptorTree from stored nodes. */
+function rehydrateFromStored(
+  nodes: StoredRaptorNode[],
+): RaptorTree | null {
+  if (nodes.length === 0) return null;
+  const root = nodes.reduce<StoredRaptorNode | null>(
+    (best, n) => (!best || n.level > (best?.level ?? -1) ? n : best),
+    null,
+  );
+  const tree: RaptorTree = {
+    nodes: new Map(
+      nodes.map((n) => [
+        n.id,
+        {
+          id: n.id,
+          level: n.level,
+          parentId: n.parentId,
+          children: n.children,
+          summary: n.summary,
+          embedding: n.embedding,
+          qualityMarker: n.qualityMarker as QualityMarker,
+          tokenEstimate: n.tokenEstimate,
+        },
+      ]),
+    ),
+    rootId: root?.id ?? null,
+    levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
+    timedOut: root != null && root.level >= 99,
+    builtAt: nodes.reduce((max, n) => Math.max(max, n.builtAt), 0),
+  };
+  return tree;
+}
+

package/src/dedup/raptor/index.ts

@@ -13,10 +13,12 @@
 import type { Embedder } from "../../embedder.js";
 import { defaultEmbedder } from "../../embedder.js";
 import { buildRaptorTree, type Leaf, type RaptorTree } from "./tree.js";
+import { incrementRaptorTree } from "./incremental.js";
 import { stagedExpansion } from "./retrieval.js";
 import { Logger } from "../../log.js";
 import { saveRaptorTree, listRaptorNodes } from "../../store/sqlite.js";
 import { insertBuildHistory, computeCoherenceScore } from "./buildHistory.js";
+import { loadDedupConfig } from "../../config/dedup.js";
 
 /** Shadow mode is on by default; set RAPTOR_SHADOW_MODE=false to serve live. */
 export function isShadowMode(): boolean {
@@ -51,15 +53,38 @@ export function runRaptor(
   const embedder = opts.embedder ?? defaultEmbedder();
   const logger = opts.logger;
   const startedAt = opts.builtAt ?? Date.now();
+  const builtAt = startedAt;
   try {
-    const tree = buildRaptorTree(leaves, {
-      embedder,
-      budgetMs: opts.budgetMs,
-      clustersPerLevel: opts.clustersPerLevel,
-      consistencyThreshold: opts.consistencyThreshold,
-    });
-    const builtAt = opts.builtAt ?? Date.now();
-    saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+    // Try incremental update first when the flag is ON (Sprint 26, #7).
+    // The incremental path reads the existing tree, diffs by leaf id, and
+    // inserts only new leaves into the nearest clusters. Falls back to a full
+    // rebuild when the tree is missing, corrupted, or >50% of nodes are new.
+    const cfg = loadDedupConfig();
+    let tree: RaptorTree | null = null;
+    if (cfg.RAPTOR_INCREMENTAL) {
+      tree = incrementRaptorTree(leaves, leaves, {
+        embedder,
+        stateDir: opts.stateDir,
+        sessionId: opts.sessionId,
+        budgetMs: opts.budgetMs,
+        clustersPerLevel: opts.clustersPerLevel,
+        consistencyThreshold: opts.consistencyThreshold,
+        logger,
+        builtAt,
+      });
+    }
+
+    // Fall back to full rebuild when incremental returned null (not attempted,
+    // tree missing, ratio exceeded, or internal error).
+    if (!tree) {
+      tree = buildRaptorTree(leaves, {
+        embedder,
+        budgetMs: opts.budgetMs,
+        clustersPerLevel: opts.clustersPerLevel,
+        consistencyThreshold: opts.consistencyThreshold,
+      });
+      saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+    }
     logger?.info("raptor_build", {
       sessionId: opts.sessionId,
       nodes: tree.nodes.size,

package/src/dedup/raptor/summarizer.ts

@@ -24,7 +24,7 @@ export interface ClusterSummary {
 function ollamaEndpoint(): { url: string; model: string } | null {
   const model = process.env.MEGACOMPACT_RAPTOR_MODEL;
   if (!model) return null;
-  const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434";
+  const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434"; // guardrails-allow PREVENT-PI-004: loopback-only Ollama endpoint for local RAPTOR summarization
   // Guard: only loopback is permitted (remote Ollama would violate PREVENT-PI-004).
   if (!/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/.test(base)) {
     throw new Error(

package/src/httpEmbedder.ts

@@ -28,7 +28,7 @@
 import type { Embedder, Vector } from "./embedder.js";
 import { l2Normalize } from "./embedder.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
-import { isIP } from "node:net";
+import { isIP } from "node:net"; // guardrails-allow PREVENT-PI-004: localhost-only loopback address validation in BYO embedding server
 
 export interface HttpEmbedderOptions {
   url: string;