pi-mega-compact 0.11.8 → 0.11.10

package/extensions/mega-events/context-handler.ts

@@ -39,7 +39,21 @@ import {
 } from "../mega-config.js";
 import { appendMirrorMessages } from "./mirror-append.js";
 import { stagedForTail, withRecallTail } from "./recall-tail.js";
-import { buildSeparatedPrompt } from "./separated-prompt.js";
+import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "./separated-prompt.js";
+
+/** Best-effort text extraction from an AgentMessage for analytics/logging.
+ *  AgentMessage is a discriminated union; .content exists only on some
+ *  variants (user/assistant/toolResult/custom). Narrow by role, never assume. */
+function messageContentText(m: AgentMessage): string {
+	const c = (m as { content?: unknown }).content;
+	if (typeof c === "string") return c;
+	if (Array.isArray(c)) {
+		return c
+			.map((b) => (b && typeof b === "object" && "text" in b ? String((b as { text?: string }).text ?? "") : ""))
+			.join(" ");
+	}
+	return "";
+}
 
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(
@@ -73,13 +87,33 @@ export function registerContextHandler(
 		// reaches raw_transcript (PREVENT-PI: append-only, view-only).
 		const tailResult = (msgs?: readonly AgentMessage[]) => {
 			const base = msgs ?? messages;
-			if (!stagedForTail(runtime, config) && !config.messageSeparation) return undefined;
+			if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping) return undefined;
 			let result: AgentMessage[] = stagedForTail(runtime, config)
 				? withRecallTail(base, runtime, config)
 				: (base as AgentMessage[]);
-			if (config.messageSeparation) {
+			if (config.cacheStriping) {
+				result = buildCacheOptimizedPrompt(result);
+			} else if (config.messageSeparation) {
 				result = buildSeparatedPrompt(result);
 			}
+			// P2.5: log prefix stability (fire-and-forget, non-fatal).
+			// tailResult is sync, so use .then().catch() on the dynamic import.
+			if (result.length > 1) {
+				import("../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
+					const stableScore = computeStabilityScore(
+						{ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
+						result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
+					);
+					runtime.logger.info("prefix_stability", {
+						stableScore: Number.isFinite(stableScore) ? stableScore : 0,
+						prefixMessages: result.length,
+						separation: config.messageSeparation ? "v2" : "off",
+						striping: config.cacheStriping ? "v3" : "off",
+					});
+				}).catch(() => {
+					// Non-fatal: stability logging is best-effort.
+				});
+			}
 			return { messages: result };
 		};
 		// Always track context for the dashboard/widget, even when auto is off.
@@ -129,6 +163,33 @@ export function registerContextHandler(
 						epochIdFor(runtime.rt.sessionId),
 						runtime.currentTurn,
 					);
+					// P2.2: populate conversation_thread + tool_results tables for
+					// prompt-cache analytics and durable separation. The live-array
+					// separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
+					// tailResult above) is sufficient for the prompt-construction path;
+					// these DB writes persist the split for post-hoc analysis, dashboard
+					// queries, and future readers. Non-fatal — failure here never breaks
+					// the agent loop (PREVENT-PI-004: zero network, local SQLite only).
+					{
+						const sid = runtime.rt.sessionId;
+						const turn = runtime.currentTurn;
+						const now = Date.now();
+						const threadStmt = db.prepare(
+							"INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
+						);
+						const toolStmt = db.prepare(
+							"INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
+						);
+						for (const m of messages) {
+							const role = m.role;
+							const content = messageContentText(m);
+							if (role === "user" || role === "assistant") {
+								threadStmt.run(sid, role, content, turn, now);
+							} else if (role === "toolResult" || role === "bashExecution") {
+								toolStmt.run(sid, role, content, turn, now);
+							}
+						}
+					}
 				} catch (e) {
 					runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
 				}
@@ -232,7 +293,7 @@ export function registerContextHandler(
 				runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
 				runtime.trimCache.cut <= messages.length
 			) {
-				const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002
+				const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
 				runtime.diagLiveTrimFires++;
 				runtime.diagLiveTrimReplays++;
 				runtime.snapshot(ctx);
@@ -474,6 +535,13 @@ export function registerContextHandler(
 				compactedFrom: ran.result.compactedFrom,
 				summary: ran.result.summary,
 				anchorUserMessages,
+				// CRITICAL-OVER ESCAPE HATCH: when context is at/over ~90% of the
+				// window, relief takes priority over the anchor floor. Without this,
+				// computeLiveTrimCut bails to null (can't satisfy the floor) and the
+				// model is fed a raw overflow that errors every turn — the
+				// "Already compacted" + overflow death-spiral (2026-08-01 incident).
+				// A thin anchor is recoverable; an overflowed session is not.
+				criticalOver: (pct ?? 0) >= 90,
 			});
 			if (cut === null) {
 				runtime.diagCtxCutNull++;
@@ -482,6 +550,7 @@ export function registerContextHandler(
 					compactedFrom: ran.result.compactedFrom,
 					viewLen: view.length,
 					anchorUserMessages,
+					criticalOver: (pct ?? 0) >= 90,
 				});
 				return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
 			}

package/extensions/mega-runtime/snapshot.ts

@@ -225,7 +225,7 @@ export function computeMegaSnapshot(p: SnapshotInput): SnapshotResult {
 		cachePct,
 		megaCacheFlare: p.megaCacheFlare,
 		megaCacheFlarePct: p.megaCacheFlarePct,
-		perTurnCacheHitPct: p.perTurnCacheHitPct ?? cachePct,
+		perTurnCacheHitPct: p.perTurnCacheHitPct,
 		levelUpFlare: p.levelUpFlare,
 		achievementFlare: p.achievementFlare,
 		achievementFlareTitles: p.achievementFlareTitles,

package/extensions/mega-trim.ts

@@ -25,6 +25,13 @@ export interface BuildLiveTrimViewOpts {
   summary: string;
   /** Min recent user messages to keep as the anchor (PREVENT-PI-001). */
   anchorUserMessages: number;
+  /** CRITICAL-OVER ESCAPE HATCH: when true, context is over ~90% of the window
+   *  and relief takes priority over the anchor floor. computeLiveTrimCut returns
+   *  the boundary-safe cut even if it keeps fewer than `anchorUserMessages` user
+   *  messages, rather than bailing to null (which would feed the model a raw
+   *  overflow). A compacted session with a thin anchor is recoverable; an
+   *  overflowed session that errors every turn is not. Default false. */
+  criticalOver?: boolean;
 }
 
 /**
@@ -68,9 +75,17 @@ export function computeLiveTrimCut(view: EngineMessage[], opts: BuildLiveTrimVie
     if (cut > 1) {
       const finalRecent = view.slice(cut);
       if (finalRecent.filter((m) => m.role === "user").length < opts.anchorUserMessages) {
+        // CRITICAL-OVER ESCAPE HATCH: when context is critically over the window,
+        // return the boundary-safe cut even though it's below the anchor floor.
+        // Bailing to null here would feed the model the raw overflow and trap the
+        // session in an unrecoverable error loop ("Already compacted" + overflow).
+        // A thin anchor is recoverable; an overflowed session is not.
+        if (opts.criticalOver && cut > 0) return cut;
         return null; // cannot satisfy the floor without dropping too much — retry next call
       }
     } else {
+      // Same escape hatch for the cut > 1 false branch.
+      if (opts.criticalOver && cut > 0) return cut;
       return null;
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.11.8",
+  "version": "0.11.10",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/dedup.ts

@@ -66,6 +66,7 @@ export interface DedupConfigShape {
   RAPTOR_MULTILEVEL_ENABLED: boolean;
   RAPTOR_LEVEL_WEIGHTS: number[]; // per-level weights, index 0 = leaves (uncalibrated)
   RAPTOR_LEAF_EXPANSION: boolean;
+  RAPTOR_INCREMENTAL: boolean;
   RAPTOR_MAX_LEAF_EXPANSION: number; // uncalibrated
   RAPTOR_FRESHNESS_HOURS: number; // S42D: skip rebuild when tree is fresh (uncalibrated)
   /** S25 Phase-2: inject top-level RAPTOR summary nodes (root + level-1
@@ -109,6 +110,7 @@ export function loadDedupConfig(): DedupConfigShape {
     RAPTOR_MULTILEVEL_ENABLED: envBool("MEGACOMPACT_RAPTOR_MULTILEVEL", true),
     RAPTOR_LEVEL_WEIGHTS: envNumArray("MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS", [1.0, 0.9, 0.8, 0.7, 0.5]),
     RAPTOR_LEAF_EXPANSION: envBool("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", true),
+    RAPTOR_INCREMENTAL: envBool("MEGACOMPACT_RAPTOR_INCREMENTAL", true),
     RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
     RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
     RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),

package/src/dedup/raptor/incremental.ts

@@ -0,0 +1,339 @@
+/**
+ * incremental.ts — incremental RAPTOR tree update (Sprint 26, #7).
+ *
+ * Instead of rebuilding the full tree (~2s per compaction), insert only the new
+ * checkpoints since the last build and recompute only the affected cluster
+ * assignments up the tree. Falls back to a full rebuild when the tree is
+ * missing, corrupted, or >50% of nodes are new.
+ *
+ * Non-fatal: every failure logs and returns null (caller falls back to full
+ * rebuild). PREVENT-PI-004: zero network; all operations are local SQLite + CPU.
+ */
+
+import type { Embedder, Vector } from "../../embedder.js";
+import { cosineSimilarity } from "../../embedder.js";
+import { defaultEmbedder as getDefaultEmbedder } from "../../embedder.js";
+import { buildRaptorTree, type Leaf, type RaptorTree } from "./tree.js";
+import type { QualityMarker } from "./guardrails.js";
+import { meanVector } from "./kmeans.js";
+import { summarizeCluster } from "./summarizer.js";
+import type { Logger } from "../../log.js";
+import {
+  saveRaptorTree,
+  listRaptorNodes,
+  type StoredRaptorNode,
+} from "../../store/sqlite.js";
+
+/** When >FRESH_THRESHOLD of existing leaves are new, a full rebuild is cheaper. */
+const FRESH_THRESHOLD = 0.5;
+
+/**
+ * Incrementally update a persisted RAPTOR tree with new leaves.
+ *
+ * 1. Reads the existing tree from the store.
+ * 2. Filters `newLeaves` to those NOT already covered by the tree.
+ * 3. If no new leaves → early-return (existing tree).
+ * 4. If no existing tree or >50% new → full rebuild.
+ * 5. For each new leaf, finds the best-matching Level-0 cluster node,
+ *    reassigns (inserts into its children), updates that node's summary
+ *    and centroid embedding.
+ * 6. Propagates changes up through parent levels to the root.
+ * 7. Persists the updated tree (full delete + reinsert in a tx).
+ *
+ * Never throws. Returns the updated tree, or null on failure/fallback.
+ */
+export function incrementRaptorTree(
+  existingLeaves: Leaf[],
+  newLeaves: Leaf[],
+  opts: {
+    embedder?: Embedder;
+    stateDir: string;
+    sessionId: string;
+    budgetMs?: number;
+    clustersPerLevel?: number;
+    consistencyThreshold?: number;
+    logger?: Logger;
+    builtAt?: number;
+  },
+): RaptorTree | null {
+  const logger = opts.logger;
+  const builtAt = opts.builtAt ?? Date.now();
+
+  try {
+    // 1. Read existing tree.
+    const storedNodes = listRaptorNodes(opts.sessionId, opts.stateDir);
+    if (storedNodes.length === 0) {
+      logger?.info("raptor_incremental_no_tree", { sessionId: opts.sessionId });
+      return null; // no existing tree — caller falls back to full buildRaptorTree
+    }
+
+    // 2. Find new leaf ids not already in the tree.
+    const coveredLeafIds = new Set<string>();
+    for (const n of storedNodes) {
+      for (const cid of n.children) coveredLeafIds.add(cid);
+    }
+    const reallyNewLeaves = newLeaves.filter(
+      (l) => !coveredLeafIds.has(l.id),
+    );
+    const totalCovered = coveredLeafIds.size;
+
+    if (reallyNewLeaves.length === 0) {
+      logger?.info("raptor_incremental_no_new", { sessionId: opts.sessionId });
+      return rehydrateFromStored(storedNodes);
+    }
+
+    // 4. Fallback guard: >50% new → full rebuild.
+    if (
+      totalCovered > 0 &&
+      reallyNewLeaves.length / totalCovered > FRESH_THRESHOLD
+    ) {
+      logger?.info("raptor_incremental_fallback_ratio", {
+        sessionId: opts.sessionId,
+        coveredNodes: totalCovered,
+        newLeaves: reallyNewLeaves.length,
+        threshold: FRESH_THRESHOLD,
+      });
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // 5. Build the in-memory tree from stored nodes.
+    const tree = rehydrateFromStored(storedNodes);
+    if (!tree || tree.nodes.size === 0) {
+      logger?.info("raptor_incremental_rehydrate_failed", {
+        sessionId: opts.sessionId,
+      });
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // Group stored nodes by level.
+    const levelGroups = new Map<number, StoredRaptorNode[]>();
+    for (const n of storedNodes) {
+      const g = levelGroups.get(n.level);
+      if (g) g.push(n);
+      else levelGroups.set(n.level, [n]);
+    }
+    // Level-0 covers leaf ids directly.
+    const level0Nodes =
+      levelGroups.get(0) ??
+      storedNodes.filter((n) => {
+        // A node is level 0 if its level is the minimum.
+        const minLevel = Math.min(...storedNodes.map((x) => x.level));
+        return n.level === minLevel;
+      });
+    if (level0Nodes.length === 0) {
+      // Single-root tree with no hierarchy: cannot increment, fall back.
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // For each new leaf, assign to the nearest Level-0 cluster node.
+    const leafEmbeddings = new Map<string, Vector>();
+    for (const l of reallyNewLeaves) leafEmbeddings.set(l.id, l.embedding);
+
+    // Track which Level-0 nodes changed (by their stored id).
+    const affectedNodeIds = new Set<string>();
+
+    for (const leaf of reallyNewLeaves) {
+      const emb = leafEmbeddings.get(leaf.id);
+      if (!emb || emb.length === 0) continue;
+
+      // Find best matching Level-0 node.
+      let bestNode: StoredRaptorNode | null = null;
+      let bestSim = -Infinity;
+      for (const cn of level0Nodes) {
+        const sim = cosineSimilarity(emb, cn.embedding);
+        if (sim > bestSim) {
+          bestSim = sim;
+          bestNode = cn;
+        }
+      }
+      if (!bestNode) continue;
+
+      // Add the leaf id to this node's children.
+      const updated = tree.nodes.get(bestNode.id);
+      if (updated) {
+        if (!updated.children.includes(leaf.id)) {
+          updated.children.push(leaf.id);
+        }
+        affectedNodeIds.add(bestNode.id);
+      }
+    }
+
+    if (affectedNodeIds.size === 0) {
+      // No matches found — fall back to full rebuild.
+      logger?.info("raptor_incremental_no_matches", {
+        sessionId: opts.sessionId,
+      });
+      return fullRebuild(existingLeaves, reallyNewLeaves, opts, logger);
+    }
+
+    // 6. Recompute embeddings and summaries for affected nodes, bottom-up.
+    // Build a map: leafId → leaf info for summarization.
+    const leafMap = new Map<string, Leaf>();
+    for (const l of existingLeaves) leafMap.set(l.id, l);
+    for (const l of reallyNewLeaves) leafMap.set(l.id, l);
+
+    // Recompute affected Level-0 nodes.
+    for (const nid of affectedNodeIds) {
+      const node = tree.nodes.get(nid);
+      if (!node) continue;
+      const coveredLeaves = node.children
+        .map((cid) => leafMap.get(cid))
+        .filter((l): l is Leaf => !!l);
+      if (coveredLeaves.length > 0) {
+        node.embedding = meanVector(
+          coveredLeaves.map((l) => l.embedding),
+        );
+        // Summarize: flatten messages from all covered leaves.
+        const messages = coveredLeaves.flatMap((l) => l.messages);
+        if (messages.length > 0) {
+          const cs = summarizeCluster(messages);
+          node.summary = cs.summary;
+          node.tokenEstimate = cs.tokenEstimate;
+        }
+      }
+    }
+
+    // Propagate up: for each higher level, recompute nodes whose leaf set
+    // includes any newly inserted leaf. Since `children` stores flattened leaf
+    // ids (not node ids), we check intersection with the new leaf id set.
+    const newLeafIds = new Set(reallyNewLeaves.map((l) => l.id));
+    const sortedLevels = [...levelGroups.keys()].sort((a, b) => a - b);
+    for (const level of sortedLevels.slice(1)) {
+      const levelNodes = levelGroups.get(level) ?? [];
+      for (const stored of levelNodes) {
+        const node = tree.nodes.get(stored.id);
+        if (!node) continue;
+
+        if (node.children.some((cid) => newLeafIds.has(cid))) {
+          // Recompute embedding as centroid of covered leaves.
+          const covered = node.children
+            .map((cid) => leafMap.get(cid))
+            .filter((l): l is Leaf => !!l);
+          if (covered.length > 0) {
+            node.embedding = meanVector(
+              covered.map((l) => l.embedding),
+            );
+            const messages = covered.flatMap((l) => l.messages);
+            if (messages.length > 0) {
+              const cs = summarizeCluster(messages);
+              node.summary = cs.summary;
+              node.tokenEstimate = cs.tokenEstimate;
+            }
+          }
+        }
+      }
+    }
+
+    // 7. Persist the full updated tree (delete + reinsert in a tx).
+    tree.builtAt = builtAt;
+    saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+
+    logger?.info("raptor_incremental_success", {
+      sessionId: opts.sessionId,
+      nodeCount: tree.nodes.size,
+      newLeaves: reallyNewLeaves.length,
+      affectedNodes: affectedNodeIds.size,
+    });
+
+    return tree;
+  } catch (e) {
+    logger?.error("raptor_incremental_failed", {
+      sessionId: opts.sessionId,
+      error: String(e instanceof Error ? e.message : e),
+    });
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Full rebuild fallback: merges existingLeaves + newLeaves, builds from scratch.
+ */
+function fullRebuild(
+  existingLeaves: Leaf[],
+  newLeaves: Leaf[],
+  opts: {
+    embedder?: Embedder;
+    budgetMs?: number;
+    clustersPerLevel?: number;
+    consistencyThreshold?: number;
+    stateDir: string;
+    sessionId: string;
+    logger?: Logger;
+    builtAt?: number;
+  },
+  logger?: Logger,
+): RaptorTree | null {
+  logger?.info("raptor_incremental_full_rebuild", {
+    sessionId: opts.sessionId,
+  });
+  try {
+    // Merge: deduplicate by leaf id.
+    const seen = new Set<string>();
+    const allLeaves: Leaf[] = [];
+    for (const l of existingLeaves) {
+      if (!seen.has(l.id)) {
+        seen.add(l.id);
+        allLeaves.push(l);
+      }
+    }
+    for (const l of newLeaves) {
+      if (!seen.has(l.id)) {
+        seen.add(l.id);
+        allLeaves.push(l);
+      }
+    }
+    const tree = buildRaptorTree(allLeaves, {
+      embedder: opts.embedder ?? getDefaultEmbedder(),
+      budgetMs: opts.budgetMs,
+      clustersPerLevel: opts.clustersPerLevel,
+      consistencyThreshold: opts.consistencyThreshold,
+    });
+    const builtAt = opts.builtAt ?? Date.now();
+    saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+    return tree;
+  } catch (e2) {
+    logger?.error("raptor_incremental_full_rebuild_failed", {
+      sessionId: opts.sessionId,
+      error: String(e2 instanceof Error ? e2.message : e2),
+    });
+    return null;
+  }
+}
+
+/** Rebuild the in-memory RaptorTree from stored nodes. */
+function rehydrateFromStored(
+  nodes: StoredRaptorNode[],
+): RaptorTree | null {
+  if (nodes.length === 0) return null;
+  const root = nodes.reduce<StoredRaptorNode | null>(
+    (best, n) => (!best || n.level > (best?.level ?? -1) ? n : best),
+    null,
+  );
+  const tree: RaptorTree = {
+    nodes: new Map(
+      nodes.map((n) => [
+        n.id,
+        {
+          id: n.id,
+          level: n.level,
+          parentId: n.parentId,
+          children: n.children,
+          summary: n.summary,
+          embedding: n.embedding,
+          qualityMarker: n.qualityMarker as QualityMarker,
+          tokenEstimate: n.tokenEstimate,
+        },
+      ]),
+    ),
+    rootId: root?.id ?? null,
+    levels: Math.max(1, ...nodes.map((n) => n.level + 1)),
+    timedOut: root != null && root.level >= 99,
+    builtAt: nodes.reduce((max, n) => Math.max(max, n.builtAt), 0),
+  };
+  return tree;
+}
+

package/src/dedup/raptor/index.ts

@@ -13,10 +13,12 @@
 import type { Embedder } from "../../embedder.js";
 import { defaultEmbedder } from "../../embedder.js";
 import { buildRaptorTree, type Leaf, type RaptorTree } from "./tree.js";
+import { incrementRaptorTree } from "./incremental.js";
 import { stagedExpansion } from "./retrieval.js";
 import { Logger } from "../../log.js";
 import { saveRaptorTree, listRaptorNodes } from "../../store/sqlite.js";
 import { insertBuildHistory, computeCoherenceScore } from "./buildHistory.js";
+import { loadDedupConfig } from "../../config/dedup.js";
 
 /** Shadow mode is on by default; set RAPTOR_SHADOW_MODE=false to serve live. */
 export function isShadowMode(): boolean {
@@ -51,15 +53,38 @@ export function runRaptor(
   const embedder = opts.embedder ?? defaultEmbedder();
   const logger = opts.logger;
   const startedAt = opts.builtAt ?? Date.now();
+  const builtAt = startedAt;
   try {
-    const tree = buildRaptorTree(leaves, {
-      embedder,
-      budgetMs: opts.budgetMs,
-      clustersPerLevel: opts.clustersPerLevel,
-      consistencyThreshold: opts.consistencyThreshold,
-    });
-    const builtAt = opts.builtAt ?? Date.now();
-    saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+    // Try incremental update first when the flag is ON (Sprint 26, #7).
+    // The incremental path reads the existing tree, diffs by leaf id, and
+    // inserts only new leaves into the nearest clusters. Falls back to a full
+    // rebuild when the tree is missing, corrupted, or >50% of nodes are new.
+    const cfg = loadDedupConfig();
+    let tree: RaptorTree | null = null;
+    if (cfg.RAPTOR_INCREMENTAL) {
+      tree = incrementRaptorTree(leaves, leaves, {
+        embedder,
+        stateDir: opts.stateDir,
+        sessionId: opts.sessionId,
+        budgetMs: opts.budgetMs,
+        clustersPerLevel: opts.clustersPerLevel,
+        consistencyThreshold: opts.consistencyThreshold,
+        logger,
+        builtAt,
+      });
+    }
+
+    // Fall back to full rebuild when incremental returned null (not attempted,
+    // tree missing, ratio exceeded, or internal error).
+    if (!tree) {
+      tree = buildRaptorTree(leaves, {
+        embedder,
+        budgetMs: opts.budgetMs,
+        clustersPerLevel: opts.clustersPerLevel,
+        consistencyThreshold: opts.consistencyThreshold,
+      });
+      saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir);
+    }
     logger?.info("raptor_build", {
       sessionId: opts.sessionId,
       nodes: tree.nodes.size,

package/src/dedup/raptor/summarizer.ts

@@ -24,7 +24,7 @@ export interface ClusterSummary {
 function ollamaEndpoint(): { url: string; model: string } | null {
   const model = process.env.MEGACOMPACT_RAPTOR_MODEL;
   if (!model) return null;
-  const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434";
+  const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434"; // guardrails-allow PREVENT-PI-004: loopback-only Ollama endpoint for local RAPTOR summarization
   // Guard: only loopback is permitted (remote Ollama would violate PREVENT-PI-004).
   if (!/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/.test(base)) {
     throw new Error(

package/src/httpEmbedder.ts

@@ -28,7 +28,7 @@
 import type { Embedder, Vector } from "./embedder.js";
 import { l2Normalize } from "./embedder.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
-import { isIP } from "node:net";
+import { isIP } from "node:net"; // guardrails-allow PREVENT-PI-004: localhost-only loopback address validation in BYO embedding server
 
 export interface HttpEmbedderOptions {
   url: string;

package/extensions/dashboard-client/dist/assets/CacheTab-CUIiUgh3.js

@@ -1,2 +0,0 @@
-import{j as e,u as x,r as p,k as T,l as R,i as $,c as y}from"./index-PRkilPXw.js";import{C as L,T as P}from"./TimeSavedCard-z4M8TDLm.js";import{b as d,c as h,d as H,e as M,f as i}from"./format-COjDIVGj.js";import{l as I}from"./pricing-CvE4NllQ.js";function f(a){return a>=80?"ov-stat-value-green":a>=50?"ov-stat-value-yellow":"ov-stat-value-red"}function k(a){return a==null?"":a>=0?"ov-stat-value-green":"ov-stat-value-red"}function r({label:a,value:t,className:l,title:s}){return e.jsxs("div",{className:"ov-stat-row",children:[e.jsx("span",{className:"ov-stat-label",title:s,children:a}),e.jsx("span",{className:`ov-stat-value${l?` ${l}`:""}`,children:t})]})}function E(a){return e.jsxs("div",{className:"card provider-cache-card",children:[e.jsx(r,{label:"Cache Hit Rate",value:`${a.hitPct.toFixed(1)}%`,className:f(a.hitPct)}),e.jsx(r,{label:"Turns Recorded",value:String(a.turnCount)}),e.jsx(r,{label:"Cache Read Tokens",value:d(a.totalCacheRead)}),e.jsx(r,{label:"Cache Write Tokens",value:d(a.totalCacheWrite)}),e.jsx(r,{label:"Input Tokens",value:d(a.totalInput)}),e.jsx(r,{label:"$ Saved (reads)",value:h(a.cacheReadSaved)}),e.jsx(r,{label:"Write Investment",value:h(a.cacheWriteCost)}),e.jsx(r,{label:"Net Saved",value:h(a.netSaved),className:k(a.netSaved)}),e.jsx(r,{label:"Model",value:a.modelLabel??"—"}),e.jsx(r,{label:"Tracked Since",value:H(a.firstTurnAt)}),e.jsx(r,{label:"Last Updated",value:M(a.latestTurnAt)}),a.byModel.length>0&&e.jsxs(e.Fragment,{children:[e.jsx("div",{className:"ov-stat-section-label",children:"Per Model"}),a.byModel.map(t=>{const l=I(t.model),s=l!=null?t.totalCacheRead/1e6*l:null,c=l!=null?t.totalCacheWrite/1e6*l:null;return e.jsxs("div",{className:"ov-stat-group",children:[e.jsx(r,{label:"Model",value:t.model}),e.jsx(r,{label:"Hit Rate",value:`${t.hitPct.toFixed(1)}%`,className:f(t.hitPct)}),e.jsx(r,{label:"Cache Read",value:d(t.totalCacheRead)}),e.jsx(r,{label:"Cache Write",value:d(t.totalCacheWrite)}),e.jsx(r,{label:"Samples",value:String(t.sampleCount)}),e.jsx(r,{label:"$ Saved (reads)",value:s!=null?h(s):"—"}),e.jsx(r,{label:"Write Investment",value:c!=null?h(c):"—"})]},t.model)})]})]})}const W=["ov-bar-green","ov-bar-blue","ov-bar-teal","ov-bar-yellow","ov-bar-gray"],w=["Permanent","Epoch","Topic","Thread","Volatile"],A={good:"ov-text-green",fair:"ov-text-yellow",degraded:"ov-text-orange",poor:"ov-text-red"};function D({data:a}){const t=Math.max(...a.buckets.map(s=>s.count),1),l=A[a.health.label]??"ov-text-gray";return e.jsxs("div",{className:"ov-card",children:[e.jsx("h3",{className:"ov-card-title",children:"Cache Stripe Distribution"}),e.jsxs("div",{className:"ov-stripe-health-summary",children:[e.jsx("span",{className:`ov-stripe-health-badge ${l}`,children:a.health.label}),e.jsx("span",{className:"ov-stripe-health-score",children:i(a.health.score)}),e.jsxs("span",{className:"ov-stripe-churn",children:["Churn: ",i(a.health.churnRate)]}),e.jsxs("span",{className:"ov-stripe-total",children:[a.totalChunks," total chunks"]})]}),e.jsx("div",{className:"ov-stripe-bars",children:a.buckets.map(s=>{const c=t>0?Math.max(0,Math.min(100,s.count/t*100)):0,o=W[s.stripe]??"ov-bar-gray",n=s.label||(w[s.stripe]??`Stripe ${s.stripe}`);return e.jsxs("div",{className:"ov-stripe-row",children:[e.jsx("span",{className:"ov-stripe-label",title:n,children:n}),e.jsx("div",{className:"ov-bar-track",children:e.jsx("div",{className:`ov-bar-fill ${o}`,style:{width:`${c}%`}})}),e.jsx("span",{className:"ov-stripe-count",children:s.count}),e.jsx("span",{className:"ov-stripe-stability",title:`min ${i(s.minStability)} / max ${i(s.maxStability)}`,children:i(s.avgStability)})]},s.stripe)})}),e.jsxs("div",{className:"ov-stripe-footer",children:["Epoch: ",a.epochId??"—","· Dominant tier: ",i(a.health.dominantTier)]})]})}function v(a){return`${a.toFixed(1)}%`}function F(a){return a>=80?"ov-bar-green":a>=60?"ov-bar-teal":a>=40?"ov-bar-yellow":a>=20?"ov-bar-orange":"ov-bar-red"}const _=({perf:a})=>{const{cache_hit_pct:t}=a,l=t.samples&&t.samples.length>0?t.samples:[];return e.jsxs("div",{className:"ov-card cache-trend-card",children:[e.jsx("h3",{className:"ov-card-title",children:"Cache Hit-Rate Trend"}),e.jsxs("div",{className:"ov-perf-summary",children:[e.jsxs("div",{className:"ov-perf-stat",children:["Avg: ",e.jsx("strong",{children:v(t.avg)})]}),e.jsxs("div",{className:"ov-perf-stat",children:["Latest: ",e.jsx("strong",{children:v(t.latest)})]}),e.jsxs("div",{className:"ov-perf-stat",children:["Samples: ",e.jsx("strong",{children:t.n})]})]}),l.length>0?e.jsxs(e.Fragment,{children:[e.jsx("div",{className:"ov-sparkline",role:"img","aria-label":`Cache hit rate trend: ${t.n} samples, average ${v(t.avg)}, latest ${v(t.latest)}`,children:l.map((s,c)=>{const o=Math.max(2,Math.round(s.pct*.6)),n=F(s.pct),b=c===l.length-1;return e.jsx("div",{className:`ov-sparkline-bar ${n}${b?" ov-bar-latest":""}`,style:{height:`${o}px`},title:`${v(s.pct)} at ${new Date(s.ts).toLocaleTimeString()}`},c)})}),e.jsxs("div",{className:"ov-card-footer",children:[l.length," sample",l.length!==1?"s":""," over"," ",l.length>1?`${Math.round((l[l.length-1].ts-l[0].ts)/1e3)}s`:"latest turn"]})]}):e.jsx("div",{className:"ov-empty-state",children:"No cache-hit samples yet"})]})};function q(){const{data:a,loading:t,error:l}=x(p.useCallback(()=>T(),[]),{pollInterval:5e3}),{data:s,loading:c,error:o}=x(p.useCallback(()=>R(),[]),{pollInterval:15e3}),{data:n,loading:b,error:j}=x(p.useCallback(()=>$({minutes:30}),[]),{pollInterval:15e3}),{data:g,loading:N,error:C}=x(p.useCallback(()=>y(),[]),{pollInterval:5e3});return e.jsxs("div",{className:"cache-tab",children:[e.jsx("h2",{children:"Provider Prompt Cache"}),t?e.jsx("p",{className:"tab-stub",children:"Loading..."}):l?e.jsxs("p",{className:"tab-stub",children:["Provider cache unavailable: ",l.message]}):a?e.jsx(E,{hitPct:a.cache.avgHitPct,turnCount:a.cache.turnCount,totalCacheRead:a.cache.totalCacheRead,totalCacheWrite:a.cache.totalCacheWrite,totalInput:a.cache.totalInput,cacheReadSaved:a.savings?.cacheReadSaved??null,cacheWriteCost:a.savings?.cacheWriteCost??null,netSaved:a.savings?.netSaved??null,modelLabel:a.savings?.model??null,firstTurnAt:a.cache.firstTurnAt,latestTurnAt:a.cache.latestTurnAt,byModel:a.cache.byModel??[]}):null,e.jsx("h2",{children:"Cache Stripe Distribution"}),c?e.jsx("p",{className:"tab-stub",children:"Loading..."}):o?e.jsxs("p",{className:"tab-stub",children:["Cache stripe data unavailable: ",o.message]}):s?e.jsx(D,{data:s}):null,e.jsx("h2",{children:"Cache Hit-Rate Trend"}),b?e.jsx("p",{className:"tab-stub",children:"Loading..."}):j?e.jsxs("p",{className:"tab-stub",children:["Hit-rate trend unavailable: ",j.message]}):n?e.jsx(_,{perf:n}):null,e.jsx("h2",{children:"Mega-Compact Dedup Cache"}),N?e.jsx("p",{className:"tab-stub",children:"Loading..."}):C?e.jsxs("p",{className:"tab-stub",children:["Snapshot unavailable: ",C.message]}):g&&(()=>{const{cacheHits:u,compacts:S,timeSaved:m}=g;return e.jsxs("div",{className:"card-grid overview-card-grid",children:[e.jsx(L,{cacheHitsSession:u.session,cacheHitsTotal:u.total,tokensSavedSession:u.sessionTokensSaved,tokensSavedTotal:u.totalTokensSaved,compactionsSession:S.session,compactionsTotal:S.total}),e.jsx(P,{compactSessionSec:m.compact.sessionSec,compactTotalSec:m.compact.totalSec,cacheHitSessionSec:m.cacheHit.sessionSec,cacheHitTotalSec:m.cacheHit.totalSec})]})})()]})}export{q as default};
-//# sourceMappingURL=CacheTab-CUIiUgh3.js.map