pi-mega-compact 0.20.23 → 0.20.24

package/dist/src/cache-stripe-score.js

@@ -0,0 +1,115 @@
+/**
+ * cache-stripe-score.ts — stability scoring + pure helpers for cache-striping.
+ *
+ * Extracted from cache-stripe-impl.ts (delegate-shell split) so the DB-touching
+ * refreshStripeAssignments lives apart from pure scoring math. No SQL, no pi
+ * runtime types (PREVENT-PI-004, PREVENT-002).
+ */
+// ─── Constants ───────────────────────────────────────────────────────────────
+const WEIGHT_SEMANTIC = 0.5;
+const WEIGHT_RECENCY = 0.3;
+const WEIGHT_FREQUENCY = 0.2;
+const STRIPE_THRESHOLDS = [
+    { minStability: 0.90, stripe: 0 },
+    { minStability: 0.70, stripe: 1 },
+    { minStability: 0.50, stripe: 2 },
+    { minStability: 0.30, stripe: 3 },
+    { minStability: -Infinity, stripe: 4 },
+];
+// ─── Embedding helpers (no external dep) ─────────────────────────────────────
+/** FNV-1a 32-bit hash for the content-based embedding fallback. */
+function fnv1a(text) {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < text.length; i++) {
+        hash ^= text.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193);
+    }
+    return (hash >>> 0) / 0x100000000;
+}
+/** Crude 128-dim hashed n-gram embedding fallback. Used when no embedder is
+ *  injected (e.g. tests); production should pass TrigramEmbedder. */
+export function fallbackEmbed(text) {
+    const dim = 128;
+    const vec = new Array(dim).fill(0);
+    const norm = text.toLowerCase().replace(/\s+/g, " ");
+    if (norm.length === 0)
+        return vec;
+    vec[Math.floor(fnv1a(norm) * dim)] += 1;
+    for (const word of norm.split(" ")) {
+        if (word.length === 0)
+            continue;
+        vec[Math.floor(fnv1a(word) * dim)] += 0.5;
+        for (let i = 0; i < Math.max(1, word.length - 1); i++) {
+            const trigram = word.slice(i, i + 3);
+            if (trigram.length === 3) {
+                vec[Math.floor(fnv1a(trigram) * dim)] += 0.25;
+            }
+        }
+    }
+    return l2Normalize(vec);
+}
+export function l2Normalize(v) {
+    let sumSq = 0;
+    for (let i = 0; i < v.length; i++)
+        sumSq += v[i] * v[i];
+    if (sumSq === 0)
+        return v;
+    const norm = Math.sqrt(sumSq);
+    for (let i = 0; i < v.length; i++)
+        v[i] /= norm;
+    return v;
+}
+/** Compute cosine similarity between two vectors of equal length. */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length || a.length === 0)
+        return 0;
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    const denom = Math.sqrt(na) * Math.sqrt(nb);
+    return denom === 0 ? 0 : dot / denom;
+}
+// ─── Stability Scoring ───────────────────────────────────────────────────────
+/**
+ * Composite stability = 0.5*semantic + 0.3*recency + 0.2*frequency.
+ * Recency/frequency fall back to neutral scores when the epoch lacks
+ * access-tracking data (the current cache_stripes schema).
+ */
+export function computeStabilityScore(chunk, allChunks, embedder, sessionEmbed) {
+    const emb = embedder
+        ? embedder.embed(chunk.content)
+        : fallbackEmbed(chunk.content);
+    // No session embedding → self-similarity baseline.
+    const sem = cosineSimilarity(emb, sessionEmbed ?? emb);
+    const semanticScore = isNaN(sem) ? 0 : sem;
+    let recencyScore = 0.5;
+    const accessed = allChunks
+        .map((c) => c.lastAccessedAt)
+        .filter((t) => t > 0);
+    if (accessed.length > 1) {
+        const minT = Math.min(...accessed);
+        const maxT = Math.max(...accessed);
+        const range = maxT - minT;
+        recencyScore = range > 0 ? (chunk.lastAccessedAt - minT) / range : 1.0;
+    }
+    const counts = allChunks.map((c) => c.accessCount);
+    const maxCount = Math.max(...counts, 1);
+    const freqScore = maxCount > 0 ? chunk.accessCount / maxCount : 0;
+    const stability = WEIGHT_SEMANTIC * semanticScore +
+        WEIGHT_RECENCY * recencyScore +
+        WEIGHT_FREQUENCY * freqScore;
+    return Math.max(0, Math.min(1, stability));
+}
+/** Map a stability score to its stripe (layer). */
+export function stabilityToStripe(stability) {
+    for (const t of STRIPE_THRESHOLDS) {
+        if (stability >= t.minStability)
+            return t.stripe;
+    }
+    return 4;
+}

package/extensions/dashboard-server/routes-rag-settings-helpers.ts

@@ -122,6 +122,18 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
 				"Tailwind + shadcn visual design (sidebar, glass panels)",
 				true,
 			),
+			boolDirect(
+				"MEGACOMPACT_MESSAGE_SEPARATION",
+				"Message Separation (P2)",
+				"PLAN_V2: split conversation thread from tool results to grow stable cache prefix",
+				false,
+			),
+			boolDirect(
+				"MEGACOMPACT_CACHE_STRIPING",
+				"Cache Striping (P3)",
+				"PLAN_V2: order stable context by stability score so durable chunks lead the prompt",
+				false,
+			),
 		],
 	},
 	{

package/extensions/mega-events/agent-handlers/turnEndHandler/cacheStripe.ts

@@ -28,12 +28,21 @@ export function cacheStripe(
 			const embedder = defaultEmbedder();
 
 			// (a) Extract text from the assistant's response for topic embedding.
+			// pi's assistant content may be a plain string OR an array of parts.
+			// Discriminate by type and only reach into `.text` after confirming the
+			// part is an object carrying a string `.text` — never assume shape.
 			let textToEmbed = "";
 			const msg = event.message;
-			if (msg.role === "assistant" && Array.isArray(msg.content)) {
-				for (const part of msg.content) {
-					if ("text" in part && typeof part.text === "string") {
-						textToEmbed += part.text + " ";
+			if (msg.role === "assistant") {
+				const c = msg.content;
+				if (typeof c === "string") {
+					textToEmbed = c;
+				} else if (Array.isArray(c)) {
+					for (const part of c) {
+						if (part && typeof part === "object" && "text" in part) {
+							const t = (part as { text?: unknown }).text;
+							if (typeof t === "string") textToEmbed += t + " ";
+						}
 					}
 				}
 			}

package/extensions/mega-events/context-handler/afterCompact.ts

@@ -50,9 +50,12 @@ export async function persistEpochAndMaintain(
 	config: MegaConfig,
 	ran: { result: CompactResult },
 ): Promise<void> {
-	// S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
-	// This makes the cache key stable across identical compactions.
-	if (config.dbMirror) {
+	// PLAN_V2 cache-striping epoch row. buildCacheOptimizedPrompt's stripe
+	// lookup reads the most-recent checkpoint_epochs row, so emit it whenever
+	// PLAN_V2 flags are live. Separate from the legacy dbMirror block below so
+	// PLAN_V2 can run with dbMirror OFF (flag-on is a genuine additive; flag-OFF
+	// = byte-identical to the predecessor, since this block no-ops then).
+	if (config.dbMirror || config.messageSeparation || config.cacheStriping) {
 		try {
 			const db = openStore(runtime.currentStateDir);
 			const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
@@ -67,6 +70,22 @@ export async function persistEpochAndMaintain(
 				createdAt: Date.now(),
 			};
 			writeCheckpointEpoch(db, epoch);
+		} catch (e) {
+			runtime.logger.warn("planv2-epoch-fail", { error: String(e) });
+		}
+	}
+
+	// S27 DB-mirror downstream: stamp turns, rebuild wiki, seed topics, dedup.
+	// Remains on `config.dbMirror` only — these are DB-mirror maintenance and
+	// must NOT run for PLAN_V2-only configurations.
+	if (config.dbMirror) {
+		try {
+			const db = openStore(runtime.currentStateDir);
+			const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
+			// epoch + writeCheckpointEpoch moved to the PLAN_V2 block above (it
+			// gates the wider set of flags); reuse epochIdFor(cpId) here so the
+			// stamps below land in the same epoch row.
+			const epochId = epochIdFor(cpId);
 			// S50B: link this session's turns to the epoch that just compacted
 			// them (compression-by-conversation-epoch metrics). Isolated-store
 			// only; best-effort + non-fatal.
@@ -74,7 +93,7 @@ export async function persistEpochAndMaintain(
 				stampTurnsEpochFor(
 					config,
 					runtime.rt.sessionId,
-					epoch.epochId,
+					epochId,
 					runtime.currentStateDir,
 				);
 			} catch {

package/extensions/mega-events/context-handler/dbMirrorAppend.ts

@@ -16,6 +16,21 @@ import type { MegaRuntime } from "../../mega-runtime.js";
 import type { MegaConfig } from "../../mega-config.js";
 import { messageContentText } from "./messageText.js";
 
+/**
+ * Best-effort tool_call_id for the tool_results insert. The toolResult variant
+ * carries a top-level toolCallId (read via an `unknown`-narrowed cast — never
+ * reach into `.content`, which is variant-specific and requires narrowing).
+ * bashExecution has no toolCallId, so fall back to a stable synthetic id keyed
+ * on (turn, index) to satisfy the NOT NULL column. No `any` (PREVENT-011).
+ */
+function toolCallIdOf(m: AgentMessage, fallback: string): string {
+	if (m.role === "toolResult") {
+		const id = (m as unknown as { toolCallId?: unknown }).toolCallId;
+		if (typeof id === "string" && id.length > 0) return id;
+	}
+	return fallback;
+}
+
 /**
  * Append incoming messages to the DB mirror (raw_transcript + thread/tool
  * tables) and the v2 ledger. Gated on config.dbMirror for the mirror; the VC1B
@@ -49,25 +64,42 @@ export function appendMirrorAndLedger(
 			// separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
 			// tailResult) is sufficient for the prompt-construction path;
 			// these DB writes persist the split for post-hoc analysis, dashboard
-			// queries, and future readers. Non-fatal — failure here never breaks
-			// the agent loop (PREVENT-PI-004: zero network, local SQLite only).
-			{
+			// queries, and future readers. Gated on (messageSeparation ||
+			// cacheStriping) so flag-OFF remains byte-identical to the
+			// predecessor — when both flags are OFF the live prompt is never
+			// separated, and growing these tables would be dead state.
+			// Non-fatal — failure here never breaks the agent loop
+			// (PREVENT-PI-004: zero network, local SQLite only).
+			if (config.messageSeparation || config.cacheStriping) {
 				const sid = runtime.rt.sessionId;
 				const turn = runtime.currentTurn;
 				const now = Date.now();
 				const threadStmt = db.prepare(
 					"INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
 				);
+				// Schema (plan-v2.ts) is (conversation_id, tool_call_id,
+				// tool_result, turn_index, timestamp) — NOT role/content.
 				const toolStmt = db.prepare(
-					"INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
+					"INSERT OR IGNORE INTO tool_results (conversation_id, tool_call_id, tool_result, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
+				);
+				const toolHas = db.prepare(
+					"SELECT 1 FROM tool_results WHERE conversation_id = ? AND turn_index = ? AND tool_call_id = ? AND tool_result = ? LIMIT 1",
+				);
+				const threadHas = db.prepare(
+					"SELECT 1 FROM conversation_thread WHERE conversation_id = ? AND turn_index = ? AND role = ? AND content = ? LIMIT 1",
 				);
-				for (const m of messages) {
+				for (const [idx, m] of messages.entries()) {
 					const role = m.role;
 					const content = messageContentText(m);
 					if (role === "user" || role === "assistant") {
-						threadStmt.run(sid, role, content, turn, now);
+						if (threadHas.get(sid, turn, role, content) == null) {
+							threadStmt.run(sid, role, content, turn, now);
+						}
 					} else if (role === "toolResult" || role === "bashExecution") {
-						toolStmt.run(sid, role, content, turn, now);
+						const toolCallId = toolCallIdOf(m, `bash:${turn}:${idx}`);
+						if (toolHas.get(sid, turn, toolCallId, content) == null) {
+							toolStmt.run(sid, toolCallId, content, turn, now);
+						}
 					}
 				}
 			}

package/extensions/mega-events/context-handler/tailResult.ts

@@ -12,6 +12,16 @@ import type { MegaConfig } from "../../mega-config.js";
 import { stagedForTail, withRecallTail } from "../recall-tail.js";
 import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "../separated-prompt.js";
 import { messageContentText } from "./messageText.js";
+import { computeContentDigest } from "../../../src/dedup/digest.js";
+
+// P2.5: per-runtime cross-turn prompt-prefix footprints for stable-prefix
+// measurement. WeakMap keyed by MegaRuntime so the entry dies with the runtime
+// (same lifecycle as a session boundary in practice — a new session re-creates
+// the pipeline over the same runtime, but the prior turn's fingerprints must
+// still be visible for cross-turn compare; a fresh runtime has an empty map and
+// the sessionId guard below blocks cross-session compare).
+type TurnPrefix = { sessionId: string; turn: number; fingerprints: string[] };
+const lastTurnPrefix = new WeakMap<MegaRuntime, TurnPrefix>();
 
 /**
  * Build the tail injection closure. Returns undefined when nothing is staged
@@ -39,23 +49,54 @@ export function buildTailResult(
 		} else if (config.messageSeparation) {
 			result = buildSeparatedPrompt(result);
 		}
-		// P2.5: log prefix stability (fire-and-forget, non-fatal).
-		// tailResult is sync, so use .then().catch() on the dynamic import.
+		// P2.5: log cross-turn stable-prefix length (cache-hit proxy). After the
+		// prompt is built (separated or cache-optimized), fingerprint each leading
+		// message and count how many are byte-identical to the previous turn's
+		// prompt, in order. A high stablePrefix = the provider KV-cache prefix is
+		// re-used (cache hit). Fire-and-forget + non-fatal.
+		//
+		// tailResult may run several times per turn (gate return / replay /
+		// debounce / live-trim), so the sessionId+turn guard measures once per
+		// turn — compare against the PREVIOUS turn's stored footprint, then store
+		// this turn's for the next comparison. Cross-session compares are skipped
+		// via the stored sessionId check (belt-and-suspenders; the WeakMap entry
+		// is scoped to this runtime, which owns the session).
 		if (result.length > 1) {
-			import("../../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
-				const stableScore = computeStabilityScore(
-					{ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
-					result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
-				);
-				runtime.logger.info("prefix_stability", {
-					stableScore: Number.isFinite(stableScore) ? stableScore : 0,
-					prefixMessages: result.length,
-					separation: config.messageSeparation ? "v2" : "off",
-					striping: config.cacheStriping ? "v3" : "off",
-				});
-			}).catch(() => {
+			try {
+				const sessionId = runtime.rt.sessionId;
+				const turn = runtime.currentTurn;
+				const prev = lastTurnPrefix.get(runtime);
+				const isNewTurn =
+					!prev || prev.sessionId !== sessionId || prev.turn !== turn;
+				if (isNewTurn) {
+					const fingerprints = result.map((m) =>
+						// Role prefix disambiguates identical text across variants
+						// (roles are a fixed enum — no realistic fingerprint collision).
+						computeContentDigest(`${m.role}|${messageContentText(m)}`).contentHash,
+					);
+					let stablePrefix = 0;
+					if (prev && prev.sessionId === sessionId) {
+						for (let i = 0; i < fingerprints.length; i++) {
+							if (
+								i >= prev.fingerprints.length ||
+								prev.fingerprints[i] !== fingerprints[i]
+							) {
+								break;
+							}
+							stablePrefix++;
+						}
+					}
+					lastTurnPrefix.set(runtime, { sessionId, turn, fingerprints });
+					runtime.logger.info("prefix_stability", {
+						stablePrefix,
+						totalMessages: result.length,
+						separation: config.messageSeparation ? "v2" : "off",
+						striping: config.cacheStriping ? "v3" : "off",
+					});
+				}
+			} catch {
 				// Non-fatal: stability logging is best-effort.
-			});
+			}
 		}
 		return { messages: result };
 	};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.20.23",
+  "version": "0.20.24",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",