pi-mega-compact 0.11.7 → 0.11.9

package/extensions/mega-events/context-handler.ts

@@ -29,6 +29,8 @@ import {
 	createTopicStore,
 	bumpWikiCompactCounter,
 } from "../../src/topics/index.js";
+import { TrigramEmbedder } from "../../src/embedder.js";
+import type { EmbeddedChunk } from "../../src/topics/types.js";
 import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
 import {
 	pressureFromPct,
@@ -37,7 +39,21 @@ import {
 } from "../mega-config.js";
 import { appendMirrorMessages } from "./mirror-append.js";
 import { stagedForTail, withRecallTail } from "./recall-tail.js";
-import { buildSeparatedPrompt } from "./separated-prompt.js";
+import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "./separated-prompt.js";
+
+/** Best-effort text extraction from an AgentMessage for analytics/logging.
+ *  AgentMessage is a discriminated union; .content exists only on some
+ *  variants (user/assistant/toolResult/custom). Narrow by role, never assume. */
+function messageContentText(m: AgentMessage): string {
+	const c = (m as { content?: unknown }).content;
+	if (typeof c === "string") return c;
+	if (Array.isArray(c)) {
+		return c
+			.map((b) => (b && typeof b === "object" && "text" in b ? String((b as { text?: string }).text ?? "") : ""))
+			.join(" ");
+	}
+	return "";
+}
 
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(
@@ -71,13 +87,33 @@ export function registerContextHandler(
 		// reaches raw_transcript (PREVENT-PI: append-only, view-only).
 		const tailResult = (msgs?: readonly AgentMessage[]) => {
 			const base = msgs ?? messages;
-			if (!stagedForTail(runtime, config) && !config.messageSeparation) return undefined;
+			if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping) return undefined;
 			let result: AgentMessage[] = stagedForTail(runtime, config)
 				? withRecallTail(base, runtime, config)
 				: (base as AgentMessage[]);
-			if (config.messageSeparation) {
+			if (config.cacheStriping) {
+				result = buildCacheOptimizedPrompt(result);
+			} else if (config.messageSeparation) {
 				result = buildSeparatedPrompt(result);
 			}
+			// P2.5: log prefix stability (fire-and-forget, non-fatal).
+			// tailResult is sync, so use .then().catch() on the dynamic import.
+			if (result.length > 1) {
+				import("../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
+					const stableScore = computeStabilityScore(
+						{ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
+						result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
+					);
+					runtime.logger.info("prefix_stability", {
+						stableScore: Number.isFinite(stableScore) ? stableScore : 0,
+						prefixMessages: result.length,
+						separation: config.messageSeparation ? "v2" : "off",
+						striping: config.cacheStriping ? "v3" : "off",
+					});
+				}).catch(() => {
+					// Non-fatal: stability logging is best-effort.
+				});
+			}
 			return { messages: result };
 		};
 		// Always track context for the dashboard/widget, even when auto is off.
@@ -127,6 +163,33 @@ export function registerContextHandler(
 						epochIdFor(runtime.rt.sessionId),
 						runtime.currentTurn,
 					);
+					// P2.2: populate conversation_thread + tool_results tables for
+					// prompt-cache analytics and durable separation. The live-array
+					// separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
+					// tailResult above) is sufficient for the prompt-construction path;
+					// these DB writes persist the split for post-hoc analysis, dashboard
+					// queries, and future readers. Non-fatal — failure here never breaks
+					// the agent loop (PREVENT-PI-004: zero network, local SQLite only).
+					{
+						const sid = runtime.rt.sessionId;
+						const turn = runtime.currentTurn;
+						const now = Date.now();
+						const threadStmt = db.prepare(
+							"INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
+						);
+						const toolStmt = db.prepare(
+							"INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
+						);
+						for (const m of messages) {
+							const role = m.role;
+							const content = messageContentText(m);
+							if (role === "user" || role === "assistant") {
+								threadStmt.run(sid, role, content, turn, now);
+							} else if (role === "toolResult" || role === "bashExecution") {
+								toolStmt.run(sid, role, content, turn, now);
+							}
+						}
+					}
 				} catch (e) {
 					runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
 				}
@@ -306,6 +369,84 @@ export function registerContextHandler(
 						error: String(wikiErr),
 					});
 				}
+
+				// D1: seed the topic model from raw_transcript when context_chunks is
+				// thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
+				// Seeds buildTopicModel with on-the-fly trigram embeddings from
+				// recent raw_transcript rows for the current session.
+				try {
+					if (
+						config.autoWikiEnabled &&
+						config.turnsDbEnabled &&
+						TurnsConfig.WIKI_SEED_FROM_TURNS
+					) {
+						const floor = 50;
+						const countRow = db
+							.prepare(
+								`SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
+							)
+							.get(runtime.rt.sessionId) as { cnt: number } | undefined;
+						const chunkCount = countRow?.cnt ?? 0;
+						if (chunkCount < floor) {
+							const transcriptRows = db
+								.prepare(
+									`SELECT DISTINCT content_bytes
+			       FROM raw_transcript
+			       WHERE session_id = ?
+			         AND length(content_bytes) > 10
+			       ORDER BY seq ASC
+			       LIMIT 200`,
+								)
+								.all(runtime.rt.sessionId) as Array<{
+								content_bytes: string;
+							}>;
+							if (transcriptRows.length > 0) {
+								const embedder = new TrigramEmbedder();
+								const seedChunks: EmbeddedChunk[] = [];
+								for (let i = 0; i < transcriptRows.length; i++) {
+									const text = transcriptRows[i].content_bytes.trim();
+									if (text.length === 0) continue;
+									const vec = embedder.embed(text);
+									seedChunks.push({
+										chunkId: `seed_transcript_${i}`,
+										sessionId: runtime.rt.sessionId,
+										vec,
+										text,
+									});
+								}
+								if (seedChunks.length > 0) {
+									const model = buildTopicModel(
+										db,
+										{
+											kRange: [
+												TurnsConfig.WIKI_K_MIN,
+												TurnsConfig.WIKI_K_MAX,
+											],
+											labelTopTerms:
+												TurnsConfig.WIKI_LABEL_TOP_TERMS,
+											restarts: 5,
+											seed: 0x9e3779b9,
+										},
+										seedChunks,
+									);
+									createTopicStore(
+										runtime.currentStateDir,
+									).replaceTopicModel(model);
+									runtime.logger.info("wiki_seed", {
+										clusterCount: model.k,
+										sourceChunks: seedChunks.length,
+										totalChunks: model.totalChunks,
+										method: "kmeans+tfidf",
+									});
+								}
+							}
+						}
+					}
+				} catch (seedErr) {
+					runtime.logger.warn("wiki_seed_failed", {
+						error: String(seedErr),
+					});
+				}
 				// S27 Task 6: Fire-and-forget dedup pipeline.
 				// Deduplicates raw_transcript rows for the compacted range.
 				try {
@@ -394,6 +535,13 @@ export function registerContextHandler(
 				compactedFrom: ran.result.compactedFrom,
 				summary: ran.result.summary,
 				anchorUserMessages,
+				// CRITICAL-OVER ESCAPE HATCH: when context is at/over ~90% of the
+				// window, relief takes priority over the anchor floor. Without this,
+				// computeLiveTrimCut bails to null (can't satisfy the floor) and the
+				// model is fed a raw overflow that errors every turn — the
+				// "Already compacted" + overflow death-spiral (2026-08-01 incident).
+				// A thin anchor is recoverable; an overflowed session is not.
+				criticalOver: (pct ?? 0) >= 90,
 			});
 			if (cut === null) {
 				runtime.diagCtxCutNull++;
@@ -402,6 +550,7 @@ export function registerContextHandler(
 					compactedFrom: ran.result.compactedFrom,
 					viewLen: view.length,
 					anchorUserMessages,
+					criticalOver: (pct ?? 0) >= 90,
 				});
 				return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
 			}

package/extensions/mega-runtime/snapshot.ts

@@ -225,7 +225,7 @@ export function computeMegaSnapshot(p: SnapshotInput): SnapshotResult {
 		cachePct,
 		megaCacheFlare: p.megaCacheFlare,
 		megaCacheFlarePct: p.megaCacheFlarePct,
-		perTurnCacheHitPct: p.perTurnCacheHitPct ?? cachePct,
+		perTurnCacheHitPct: p.perTurnCacheHitPct,
 		levelUpFlare: p.levelUpFlare,
 		achievementFlare: p.achievementFlare,
 		achievementFlareTitles: p.achievementFlareTitles,

package/extensions/mega-trim.ts

@@ -25,6 +25,13 @@ export interface BuildLiveTrimViewOpts {
   summary: string;
   /** Min recent user messages to keep as the anchor (PREVENT-PI-001). */
   anchorUserMessages: number;
+  /** CRITICAL-OVER ESCAPE HATCH: when true, context is over ~90% of the window
+   *  and relief takes priority over the anchor floor. computeLiveTrimCut returns
+   *  the boundary-safe cut even if it keeps fewer than `anchorUserMessages` user
+   *  messages, rather than bailing to null (which would feed the model a raw
+   *  overflow). A compacted session with a thin anchor is recoverable; an
+   *  overflowed session that errors every turn is not. Default false. */
+  criticalOver?: boolean;
 }
 
 /**
@@ -68,9 +75,17 @@ export function computeLiveTrimCut(view: EngineMessage[], opts: BuildLiveTrimVie
     if (cut > 1) {
       const finalRecent = view.slice(cut);
       if (finalRecent.filter((m) => m.role === "user").length < opts.anchorUserMessages) {
+        // CRITICAL-OVER ESCAPE HATCH: when context is critically over the window,
+        // return the boundary-safe cut even though it's below the anchor floor.
+        // Bailing to null here would feed the model the raw overflow and trap the
+        // session in an unrecoverable error loop ("Already compacted" + overflow).
+        // A thin anchor is recoverable; an overflowed session is not.
+        if (opts.criticalOver && cut > 0) return cut;
         return null; // cannot satisfy the floor without dropping too much — retry next call
       }
     } else {
+      // Same escape hatch for the cut > 1 false branch.
+      if (opts.criticalOver && cut > 0) return cut;
       return null;
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.11.7",
+  "version": "0.11.9",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/turns.ts

@@ -37,8 +37,10 @@ export interface TurnsConfigShape {
 	WIKI_K_MAX: number;
 	/** S51: how many TF-IDF terms form a topic label (default 5). */
 	WIKI_LABEL_TOP_TERMS: number;
-	/** S51: rebuild the topic model every Nth compaction (default 10). */
+	/** S51: rebuild the topic model every Nth compaction (default 3). */
 	WIKI_REBUILD_EVERY_N_COMPACTS: number;
+	/** D1: seed the topic model from raw_transcript when context_chunks is thin (default true). */
+	WIKI_SEED_FROM_TURNS: boolean;
 }
 
 function envKRange(defMin: number, defMax: number): [number, number] {
@@ -62,7 +64,8 @@ export function loadTurnsConfig(): TurnsConfigShape {
 		WIKI_K_MIN: kMin,
 		WIKI_K_MAX: kMax,
 		WIKI_LABEL_TOP_TERMS: envNum("MEGACOMPACT_WIKI_LABEL_TOP_TERMS", 5),
-		WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 10),
+		WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 3),
+		WIKI_SEED_FROM_TURNS: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
 	};
 }
 

package/src/memoryGraph/embedding.ts

@@ -0,0 +1,63 @@
+/**
+ * embedding.ts — D3 embedding helpers for memoryGraph source builders.
+ *
+ * Uses TrigramEmbedder with an in-memory SQLite cache table.
+ * Isolated here so the main module stays under the 500-line limit.
+ */
+import type { DatabaseSync } from "node:sqlite";
+import { TrigramEmbedder } from "../embedder.js";
+
+let _trigramEmbedder: TrigramEmbedder | null = null;
+
+function lazyEmbedder(): TrigramEmbedder {
+  if (!_trigramEmbedder) {
+    _trigramEmbedder = new TrigramEmbedder();
+  }
+  return _trigramEmbedder;
+}
+
+function simpleHash(s: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h.toString(36);
+}
+
+/** Compute or retrieve a cached embedding vector for a text string.
+ *  Uses the embedding_cache table (created by the A2 migration) for
+ *  persistence. Falls back to uncached TrigramEmbedder on table error. */
+export function getOrComputeEmbedding(db: DatabaseSync, content: string): number[] {
+  try {
+    const hash = simpleHash(content);
+    const cached = db
+      .prepare("SELECT embedding FROM embedding_cache WHERE content_hash = ?")
+      .get(hash) as { embedding: string } | undefined;
+    if (cached && typeof cached.embedding === "string") {
+      return JSON.parse(cached.embedding) as number[];
+    }
+    const vec = lazyEmbedder().embed(content);
+    db.prepare(
+      "INSERT OR REPLACE INTO embedding_cache (content_hash, content, embedding) VALUES (?, ?, ?)",
+    ).run(hash, content, JSON.stringify(vec));
+    return vec;
+  } catch {
+    return lazyEmbedder().embed(content);
+  }
+}
+
+/** Cosine similarity between two equal-length vectors. */
+export function cosineSimilarity(a: number[], b: number[]): number {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  const len = Math.min(a.length, b.length);
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom > 0 ? dot / denom : 0;
+}