pi-mega-compact 0.11.6 → 0.11.8

package/extensions/mega-commands.ts

@@ -11,13 +11,13 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
 import { normalizeSessionId } from "../src/store.js";
 import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
 import { decompressSmart } from "../src/store/compression.js";
-import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
+import { loadMetrics, fpRate, p95, defaultMetricsPath, defaultEventsPath } from "../src/monitoring.js";
 import { type MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
 import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
 import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
 import type { MegaConfig } from "./mega-config.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
-import { mkdirSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
 import path from "node:path";
 import os from "node:os";
 import { createRequire } from "node:module";
@@ -280,6 +280,52 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
     },
   });
 
+  /**
+   * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
+   * Best-effort, non-fatal — returns true if a low-quality signal was found.
+   */
+  /**
+   * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
+   * Best-effort, non-fatal — returns true if a low-quality signal was found.
+   */
+  function checkRecallQuality(stateDir: string): boolean {
+    const eventPaths = [
+      defaultEventsPath(stateDir),
+      path.join(stateDir, "mega-compact.log"),
+    ];
+    for (const fp of eventPaths) {
+      try {
+        if (!existsSync(fp)) continue;
+        const buf = readFileSync(fp, { encoding: "utf-8" });
+        // Only scan last 64 KiB of the file
+        const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
+        const lines = tail.split("\n").filter(Boolean);
+        for (const line of lines) {
+          try {
+            // guardrails-allow PREVENT-001: JSON.parse with null check
+            const parsed = JSON.parse(line);
+            if (typeof parsed !== "object" || parsed === null) continue;
+            // Check for recall_metrics_low_quality event (latest format)
+            if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
+              if (typeof parsed.score === "number" && parsed.score < 0.4) return true;
+            }
+            // Check for recall_metrics with low relevanceScore
+            if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
+              if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4) return true;
+            }
+            // Check for RecallQualityEvent format from monitoring.ts
+            if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4) return true;
+          } catch {
+            // skip unparseable lines
+          }
+        }
+      } catch {
+        // non-fatal: skip if can't read
+      }
+    }
+    return false;
+  }
+
   pi.registerCommand("mega-setup", {
     description:
       "Local embedding setup wizard — detect available embedders and configure mega-compact.",
@@ -355,6 +401,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
         // require error
       }
 
+      // --- 2.5 Quality check ---
+      // Scan recent events for low recall quality (best-effort, non-fatal)
+      const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
+      const lowRecallQuality = checkRecallQuality(stateDir);
+
       // --- 3. Build choices ---
       interface Choice {
         label: string;
@@ -402,6 +453,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
 
       // --- 4. Prompt user ---
       log(`Current embedder: ${currentEmbedder}`);
+      if (lowRecallQuality) {
+        log(
+          "Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.",
+        );
+      }
       let selected = "trigram";
       if (choices.length > 1) {
         try {
@@ -435,7 +491,6 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
         const envContent = lines.join("\n") + "\n";
 
         // Write .mega-compact.env to state dir
-        const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
         try {
           mkdirSync(stateDir, { recursive: true });
           writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -457,8 +512,6 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
           `export MEGACOMPACT_EMBEDDING_URL="${url}"`,
         ];
         const envContent = lines.join("\n") + "\n";
-
-        const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
         try {
           mkdirSync(stateDir, { recursive: true });
           writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -477,7 +530,11 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
           "ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.",
         );
       } else {
-        log("Keeping TrigramEmbedder (default). No configuration needed.");
+        const qualityTip =
+          lowRecallQuality
+            ? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
+            : "";
+        log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
       }
      } catch (e) {
        ctx.ui.notify(`[mega-compact] /mega-setup failed: ${String(e)}`);

package/extensions/mega-config.ts

@@ -183,6 +183,18 @@ export interface MegaConfig {
 	ragRecallMetrics: boolean;
 	/** S57 B4: Memory graph traversal (dashboard-oriented). */
 	ragMemoryGraph: boolean;
+	/** D1: Seed initial wiki topic model from live turns when no context_chunks exist yet. */
+	wikiSeedFromTurns: boolean;
+	/** D3 Source A: Include structural turn nodes in the memory graph (metadata only, no content). */
+	memoryGraphSeedTurns: boolean;
+	/** D3 Source B: Include raw_transcript content nodes in the memory graph (requires dbMirror). */
+	memoryGraphSeedTurnContent: boolean;
+	/** D3 Source C: Include memory review nodes in the memory graph. */
+	memoryGraphSeedMemories: boolean;
+	/** D3 edges: Stricter cosine floor for cross-type edges (e.g. turn↔checkpoint). */
+	memoryGraphCrossTypeThreshold: number;
+	/** D3 edges: Cosine floor for within-type semantic edges. */
+	memoryGraphWithinTypeThreshold: number;
 }
 
 function envFlag(name: string, fallback: number): number {
@@ -350,6 +362,12 @@ export function loadConfig(): MegaConfig {
 		ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),
 		ragRecallMetrics: envBool("MEGACOMPACT_RECALL_METRICS", false),
 		ragMemoryGraph: envBool("MEGACOMPACT_MEMORY_GRAPH", false),
+		wikiSeedFromTurns: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
+		memoryGraphSeedTurns: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true),
+		memoryGraphSeedTurnContent: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true),
+		memoryGraphSeedMemories: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true),
+		memoryGraphCrossTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85),
+		memoryGraphWithinTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7),
 		// D.1: env-overridable recompact delta (minimum context growth % before
 		// re-compacting instead of replaying the cached live trim). Default 50.
 		recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),

package/extensions/mega-events/context-handler.ts

@@ -29,6 +29,8 @@ import {
 	createTopicStore,
 	bumpWikiCompactCounter,
 } from "../../src/topics/index.js";
+import { TrigramEmbedder } from "../../src/embedder.js";
+import type { EmbeddedChunk } from "../../src/topics/types.js";
 import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
 import {
 	pressureFromPct,
@@ -306,6 +308,84 @@ export function registerContextHandler(
 						error: String(wikiErr),
 					});
 				}
+
+				// D1: seed the topic model from raw_transcript when context_chunks is
+				// thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
+				// Seeds buildTopicModel with on-the-fly trigram embeddings from
+				// recent raw_transcript rows for the current session.
+				try {
+					if (
+						config.autoWikiEnabled &&
+						config.turnsDbEnabled &&
+						TurnsConfig.WIKI_SEED_FROM_TURNS
+					) {
+						const floor = 50;
+						const countRow = db
+							.prepare(
+								`SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
+							)
+							.get(runtime.rt.sessionId) as { cnt: number } | undefined;
+						const chunkCount = countRow?.cnt ?? 0;
+						if (chunkCount < floor) {
+							const transcriptRows = db
+								.prepare(
+									`SELECT DISTINCT content_bytes
+			       FROM raw_transcript
+			       WHERE session_id = ?
+			         AND length(content_bytes) > 10
+			       ORDER BY seq ASC
+			       LIMIT 200`,
+								)
+								.all(runtime.rt.sessionId) as Array<{
+								content_bytes: string;
+							}>;
+							if (transcriptRows.length > 0) {
+								const embedder = new TrigramEmbedder();
+								const seedChunks: EmbeddedChunk[] = [];
+								for (let i = 0; i < transcriptRows.length; i++) {
+									const text = transcriptRows[i].content_bytes.trim();
+									if (text.length === 0) continue;
+									const vec = embedder.embed(text);
+									seedChunks.push({
+										chunkId: `seed_transcript_${i}`,
+										sessionId: runtime.rt.sessionId,
+										vec,
+										text,
+									});
+								}
+								if (seedChunks.length > 0) {
+									const model = buildTopicModel(
+										db,
+										{
+											kRange: [
+												TurnsConfig.WIKI_K_MIN,
+												TurnsConfig.WIKI_K_MAX,
+											],
+											labelTopTerms:
+												TurnsConfig.WIKI_LABEL_TOP_TERMS,
+											restarts: 5,
+											seed: 0x9e3779b9,
+										},
+										seedChunks,
+									);
+									createTopicStore(
+										runtime.currentStateDir,
+									).replaceTopicModel(model);
+									runtime.logger.info("wiki_seed", {
+										clusterCount: model.k,
+										sourceChunks: seedChunks.length,
+										totalChunks: model.totalChunks,
+										method: "kmeans+tfidf",
+									});
+								}
+							}
+						}
+					}
+				} catch (seedErr) {
+					runtime.logger.warn("wiki_seed_failed", {
+						error: String(seedErr),
+					});
+				}
 				// S27 Task 6: Fire-and-forget dedup pipeline.
 				// Deduplicates raw_transcript rows for the compacted range.
 				try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.11.6",
+  "version": "0.11.8",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/turns.ts

@@ -37,8 +37,10 @@ export interface TurnsConfigShape {
 	WIKI_K_MAX: number;
 	/** S51: how many TF-IDF terms form a topic label (default 5). */
 	WIKI_LABEL_TOP_TERMS: number;
-	/** S51: rebuild the topic model every Nth compaction (default 10). */
+	/** S51: rebuild the topic model every Nth compaction (default 3). */
 	WIKI_REBUILD_EVERY_N_COMPACTS: number;
+	/** D1: seed the topic model from raw_transcript when context_chunks is thin (default true). */
+	WIKI_SEED_FROM_TURNS: boolean;
 }
 
 function envKRange(defMin: number, defMax: number): [number, number] {
@@ -62,7 +64,8 @@ export function loadTurnsConfig(): TurnsConfigShape {
 		WIKI_K_MIN: kMin,
 		WIKI_K_MAX: kMax,
 		WIKI_LABEL_TOP_TERMS: envNum("MEGACOMPACT_WIKI_LABEL_TOP_TERMS", 5),
-		WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 10),
+		WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 3),
+		WIKI_SEED_FROM_TURNS: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
 	};
 }
 

package/src/memoryGraph/embedding.ts

@@ -0,0 +1,63 @@
+/**
+ * embedding.ts — D3 embedding helpers for memoryGraph source builders.
+ *
+ * Uses TrigramEmbedder with an in-memory SQLite cache table.
+ * Isolated here so the main module stays under the 500-line limit.
+ */
+import type { DatabaseSync } from "node:sqlite";
+import { TrigramEmbedder } from "../embedder.js";
+
+let _trigramEmbedder: TrigramEmbedder | null = null;
+
+function lazyEmbedder(): TrigramEmbedder {
+  if (!_trigramEmbedder) {
+    _trigramEmbedder = new TrigramEmbedder();
+  }
+  return _trigramEmbedder;
+}
+
+function simpleHash(s: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h.toString(36);
+}
+
+/** Compute or retrieve a cached embedding vector for a text string.
+ *  Uses the embedding_cache table (created by the A2 migration) for
+ *  persistence. Falls back to uncached TrigramEmbedder on table error. */
+export function getOrComputeEmbedding(db: DatabaseSync, content: string): number[] {
+  try {
+    const hash = simpleHash(content);
+    const cached = db
+      .prepare("SELECT embedding FROM embedding_cache WHERE content_hash = ?")
+      .get(hash) as { embedding: string } | undefined;
+    if (cached && typeof cached.embedding === "string") {
+      return JSON.parse(cached.embedding) as number[];
+    }
+    const vec = lazyEmbedder().embed(content);
+    db.prepare(
+      "INSERT OR REPLACE INTO embedding_cache (content_hash, content, embedding) VALUES (?, ?, ?)",
+    ).run(hash, content, JSON.stringify(vec));
+    return vec;
+  } catch {
+    return lazyEmbedder().embed(content);
+  }
+}
+
+/** Cosine similarity between two equal-length vectors. */
+export function cosineSimilarity(a: number[], b: number[]): number {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  const len = Math.min(a.length, b.length);
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom > 0 ? dot / denom : 0;
+}