pi-mega-compact 0.11.7 → 0.11.8

package/dist/extensions/dashboard-server/routes-memory-map.js

@@ -1,7 +1,8 @@
 /**
- * routes-memory-map.ts — GET /api/memory-map route handler (S46).
+ * routes-memory-map.ts — GET /api/memory-map route handler (S46/D3).
  *
  * Builds a memory graph from the local SQLite store and returns it as JSON.
+ * D3: surfaces nodeType on each node and the GraphValidationReport.
  * Uses an inline require to the pi-agnostic src/memoryGraph.ts builder.
  */
 import { createRequire } from "node:module";
@@ -16,19 +17,8 @@ export function handleMemoryMap(req, res, ctx) {
         return false;
     // guardrails-allow PREVENT-PI-004: local loopback-only dashboard server — reads the local SQLite store for graph data.
     const { buildMemoryGraph } = _req("../../src/memoryGraph.js");
-    const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : undefined;
-    const threshold = typeof parsed.query.threshold === "string" ? Number.parseFloat(parsed.query.threshold) : undefined;
-    const maxEdges = typeof parsed.query.maxEdgesPerNode === "string" ? Number.parseInt(parsed.query.maxEdgesPerNode, 10) : undefined;
-    const raptorEdges = parsed.query.includeRaptorEdges === "false" ? false : true;
-    const temporalEdges = parsed.query.includeTemporalEdges === "false" ? false : true;
-    const graph = buildMemoryGraph({
-        sessionId,
-        similarityThreshold: threshold,
-        maxEdgesPerNode: maxEdges,
-        stateDir: ctx.stateDir,
-        includeRaptorEdges: raptorEdges,
-        includeTemporalEdges: temporalEdges,
-    });
+    const sessionId = typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
+    const graph = buildMemoryGraph(sessionId, ctx.stateDir);
     const nodes = graph.nodes.map((n) => ({
         id: n.id,
         sessionId: n.sessionId,
@@ -41,9 +31,12 @@ export function handleMemoryMap(req, res, ctx) {
         topicSummary: n.topicSummary,
         decisionCount: n.decisionCount,
         textSnippet: n.textSnippet,
+        // D3: pass through nodeType for UI node-shape encoding
+        nodeType: n.nodeType,
     }));
     const edges = graph.edges;
-    const response = { nodes, edges, metadata: graph.metadata };
+    // D3: surface the validation report (optional in the response type for backward compat)
+    const response = { nodes, edges, metadata: graph.metadata, validation: graph.validation };
     res.writeHead(200, { "Content-Type": "application/json" });
     res.end(JSON.stringify(response));
     return true;

package/dist/extensions/mega-commands.js

@@ -9,12 +9,12 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
 import { normalizeSessionId } from "../src/store.js";
 import { listCheckpoints, latestModelSnapshot, countInjectedGlobal, listRepoRegistry } from "../src/store/sqlite.js";
 import { decompressSmart } from "../src/store/compression.js";
-import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
+import { loadMetrics, fpRate, p95, defaultMetricsPath, defaultEventsPath } from "../src/monitoring.js";
 import { C, recentUserQuery } from "./mega-runtime.js";
 import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
 import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
-import { mkdirSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
 import path from "node:path";
 import os from "node:os";
 import { createRequire } from "node:module";
@@ -260,6 +260,58 @@ export function registerCommands(pi, runtime, config) {
                 `• data safety — every compressed region is kept verbatim; nothing is permanently deleted. /mega-restore brings any of it back.`);
         },
     });
+    /**
+     * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
+     * Best-effort, non-fatal — returns true if a low-quality signal was found.
+     */
+    /**
+     * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
+     * Best-effort, non-fatal — returns true if a low-quality signal was found.
+     */
+    function checkRecallQuality(stateDir) {
+        const eventPaths = [
+            defaultEventsPath(stateDir),
+            path.join(stateDir, "mega-compact.log"),
+        ];
+        for (const fp of eventPaths) {
+            try {
+                if (!existsSync(fp))
+                    continue;
+                const buf = readFileSync(fp, { encoding: "utf-8" });
+                // Only scan last 64 KiB of the file
+                const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
+                const lines = tail.split("\n").filter(Boolean);
+                for (const line of lines) {
+                    try {
+                        // guardrails-allow PREVENT-001: JSON.parse with null check
+                        const parsed = JSON.parse(line);
+                        if (typeof parsed !== "object" || parsed === null)
+                            continue;
+                        // Check for recall_metrics_low_quality event (latest format)
+                        if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
+                            if (typeof parsed.score === "number" && parsed.score < 0.4)
+                                return true;
+                        }
+                        // Check for recall_metrics with low relevanceScore
+                        if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
+                            if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4)
+                                return true;
+                        }
+                        // Check for RecallQualityEvent format from monitoring.ts
+                        if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4)
+                            return true;
+                    }
+                    catch {
+                        // skip unparseable lines
+                    }
+                }
+            }
+            catch {
+                // non-fatal: skip if can't read
+            }
+        }
+        return false;
+    }
     pi.registerCommand("mega-setup", {
         description: "Local embedding setup wizard — detect available embedders and configure mega-compact.",
         handler: async (_args, ctx) => {
@@ -335,6 +387,10 @@ export function registerCommands(pi, runtime, config) {
                 catch {
                     // require error
                 }
+                // --- 2.5 Quality check ---
+                // Scan recent events for low recall quality (best-effort, non-fatal)
+                const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
+                const lowRecallQuality = checkRecallQuality(stateDir);
                 const choices = [];
                 if (ollamaDetected && ollamaRunning) {
                     const model = ollamaModels.find((m) => m.includes("nomic-embed") ||
@@ -370,6 +426,9 @@ export function registerCommands(pi, runtime, config) {
                 });
                 // --- 4. Prompt user ---
                 log(`Current embedder: ${currentEmbedder}`);
+                if (lowRecallQuality) {
+                    log("Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.");
+                }
                 let selected = "trigram";
                 if (choices.length > 1) {
                     try {
@@ -403,7 +462,6 @@ export function registerCommands(pi, runtime, config) {
                     ];
                     const envContent = lines.join("\n") + "\n";
                     // Write .mega-compact.env to state dir
-                    const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
                     try {
                         mkdirSync(stateDir, { recursive: true });
                         writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -421,7 +479,6 @@ export function registerCommands(pi, runtime, config) {
                         `export MEGACOMPACT_EMBEDDING_URL="${url}"`,
                     ];
                     const envContent = lines.join("\n") + "\n";
-                    const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
                     try {
                         mkdirSync(stateDir, { recursive: true });
                         writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
@@ -435,7 +492,10 @@ export function registerCommands(pi, runtime, config) {
                     log("ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.");
                 }
                 else {
-                    log("Keeping TrigramEmbedder (default). No configuration needed.");
+                    const qualityTip = lowRecallQuality
+                        ? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
+                        : "";
+                    log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
                 }
             }
             catch (e) {

package/dist/extensions/mega-config.js

@@ -173,6 +173,12 @@ export function loadConfig() {
         ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),
         ragRecallMetrics: envBool("MEGACOMPACT_RECALL_METRICS", false),
         ragMemoryGraph: envBool("MEGACOMPACT_MEMORY_GRAPH", false),
+        wikiSeedFromTurns: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
+        memoryGraphSeedTurns: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true),
+        memoryGraphSeedTurnContent: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true),
+        memoryGraphSeedMemories: envBool("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true),
+        memoryGraphCrossTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85),
+        memoryGraphWithinTypeThreshold: envFlag("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7),
         // D.1: env-overridable recompact delta (minimum context growth % before
         // re-compacting instead of replaying the cached live trim). Default 50.
         recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),

package/dist/extensions/mega-events/context-handler.js

@@ -7,6 +7,7 @@ import { stampTurnsEpochFor } from "../mega-turn-store.js";
 import { TurnsConfig } from "../../src/config/turns.js";
 import { openTurnStore } from "../../src/store/turns/connection.js";
 import { buildTopicModel, createTopicStore, bumpWikiCompactCounter, } from "../../src/topics/index.js";
+import { TrigramEmbedder } from "../../src/embedder.js";
 import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
 import { pressureFromPct, pressureRatio, } from "../mega-config.js";
 import { appendMirrorMessages } from "./mirror-append.js";
@@ -250,6 +251,70 @@ export function registerContextHandler(pi, runtime, config) {
                         error: String(wikiErr),
                     });
                 }
+                // D1: seed the topic model from raw_transcript when context_chunks is
+                // thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
+                // Seeds buildTopicModel with on-the-fly trigram embeddings from
+                // recent raw_transcript rows for the current session.
+                try {
+                    if (config.autoWikiEnabled &&
+                        config.turnsDbEnabled &&
+                        TurnsConfig.WIKI_SEED_FROM_TURNS) {
+                        const floor = 50;
+                        const countRow = db
+                            .prepare(`SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`)
+                            .get(runtime.rt.sessionId);
+                        const chunkCount = countRow?.cnt ?? 0;
+                        if (chunkCount < floor) {
+                            const transcriptRows = db
+                                .prepare(`SELECT DISTINCT content_bytes
+			       FROM raw_transcript
+			       WHERE session_id = ?
+			         AND length(content_bytes) > 10
+			       ORDER BY seq ASC
+			       LIMIT 200`)
+                                .all(runtime.rt.sessionId);
+                            if (transcriptRows.length > 0) {
+                                const embedder = new TrigramEmbedder();
+                                const seedChunks = [];
+                                for (let i = 0; i < transcriptRows.length; i++) {
+                                    const text = transcriptRows[i].content_bytes.trim();
+                                    if (text.length === 0)
+                                        continue;
+                                    const vec = embedder.embed(text);
+                                    seedChunks.push({
+                                        chunkId: `seed_transcript_${i}`,
+                                        sessionId: runtime.rt.sessionId,
+                                        vec,
+                                        text,
+                                    });
+                                }
+                                if (seedChunks.length > 0) {
+                                    const model = buildTopicModel(db, {
+                                        kRange: [
+                                            TurnsConfig.WIKI_K_MIN,
+                                            TurnsConfig.WIKI_K_MAX,
+                                        ],
+                                        labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
+                                        restarts: 5,
+                                        seed: 0x9e3779b9,
+                                    }, seedChunks);
+                                    createTopicStore(runtime.currentStateDir).replaceTopicModel(model);
+                                    runtime.logger.info("wiki_seed", {
+                                        clusterCount: model.k,
+                                        sourceChunks: seedChunks.length,
+                                        totalChunks: model.totalChunks,
+                                        method: "kmeans+tfidf",
+                                    });
+                                }
+                            }
+                        }
+                    }
+                }
+                catch (seedErr) {
+                    runtime.logger.warn("wiki_seed_failed", {
+                        error: String(seedErr),
+                    });
+                }
                 // S27 Task 6: Fire-and-forget dedup pipeline.
                 // Deduplicates raw_transcript rows for the compacted range.
                 try {

package/dist/src/config/turns.js

@@ -42,7 +42,8 @@ export function loadTurnsConfig() {
         WIKI_K_MIN: kMin,
         WIKI_K_MAX: kMax,
         WIKI_LABEL_TOP_TERMS: envNum("MEGACOMPACT_WIKI_LABEL_TOP_TERMS", 5),
-        WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 10),
+        WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 3),
+        WIKI_SEED_FROM_TURNS: envBool("MEGACOMPACT_WIKI_SEED_FROM_TURNS", true),
     };
 }
 export const TurnsConfig = loadTurnsConfig();

package/dist/src/memoryGraph/embedding.js

@@ -0,0 +1,50 @@
+import { TrigramEmbedder } from "../embedder.js";
+let _trigramEmbedder = null;
+function lazyEmbedder() {
+    if (!_trigramEmbedder) {
+        _trigramEmbedder = new TrigramEmbedder();
+    }
+    return _trigramEmbedder;
+}
+function simpleHash(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h.toString(36);
+}
+/** Compute or retrieve a cached embedding vector for a text string.
+ *  Uses the embedding_cache table (created by the A2 migration) for
+ *  persistence. Falls back to uncached TrigramEmbedder on table error. */
+export function getOrComputeEmbedding(db, content) {
+    try {
+        const hash = simpleHash(content);
+        const cached = db
+            .prepare("SELECT embedding FROM embedding_cache WHERE content_hash = ?")
+            .get(hash);
+        if (cached && typeof cached.embedding === "string") {
+            return JSON.parse(cached.embedding);
+        }
+        const vec = lazyEmbedder().embed(content);
+        db.prepare("INSERT OR REPLACE INTO embedding_cache (content_hash, content, embedding) VALUES (?, ?, ?)").run(hash, content, JSON.stringify(vec));
+        return vec;
+    }
+    catch {
+        return lazyEmbedder().embed(content);
+    }
+}
+/** Cosine similarity between two equal-length vectors. */
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    const len = Math.min(a.length, b.length);
+    for (let i = 0; i < len; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    const denom = Math.sqrt(na) * Math.sqrt(nb);
+    return denom > 0 ? dot / denom : 0;
+}