pi-mega-compact 0.11.6 → 0.11.8

package/dist/src/memoryGraph.js

@@ -1,299 +1,173 @@
 /**
- * memoryGraph.ts — S46 visual memory map: build a graph of graphs.
+ * memoryGraph.ts — S46/D3 visual memory map: build a graph of graphs.
  *
- * Nodes = checkpoints (with optional RAPTOR level annotations).
- * Edges = temporal adjacency, semantic similarity (cosine above threshold), and
- *         RAPTOR parent–child links.
+ * Delegated-shell pattern: implementation lives in memoryGraph/gates.ts,
+ * memoryGraph/sources.ts, and memoryGraph/embedding.ts. This file owns
+ * the public types, the entry point (buildMemoryGraph), and exports.
  *
  * pi-agnostic (no pi runtime types). Zero network calls (PREVENT-PI-004).
- * Synchronous read-only interface — non-fatal on store errors.
+ * Synchronous read-only interface -- non-fatal on store errors.
  *
  * Re-exports the public types as both MemoryGraph* and shorter aliases
  * (GraphNode, GraphEdge, GraphMetadata) so dashboard api-contracts can import
  * from the canonical source.
  */
-import { getStateDir, normalizeSessionId } from "./store.js";
-import { openStore, decodeEmbedding } from "./store/sqlite/utils.js";
-import { listRaptorNodes } from "./store/sqlite/raptor.js";
+import { getStateDir } from "./store.js";
+import { openStore } from "./store/sqlite/utils.js";
 import { Logger } from "./log.js";
+import { runValidationPipeline } from "./memoryGraph/gates.js";
+import { buildCheckpointNodes, buildTurnNodes, buildTurnContentNodes, buildMemoryNodes, addRaptorAnnotations, deduplicateEdges, areTurnsEnabled, isTurnContentEnabled, isTurnContentFlaggedOn, areMemoriesEnabled, } from "./memoryGraph/sources.js";
 const log = new Logger();
 // ---------------------------------------------------------------------------
-// Defaults
-// ---------------------------------------------------------------------------
-const DEFAULT_SIMILARITY_THRESHOLD = 0.7;
-const DEFAULT_MAX_EDGES_PER_NODE = 4;
-// ---------------------------------------------------------------------------
-// Public API
+// Public entry point
 // ---------------------------------------------------------------------------
 /**
- * Build a memory graph from the store.
+ * Build a memory graph from three sources (checkpoints / turns / memories)
+ * wired through a 9-gate validation pipeline.
  *
- * Opens the SQLite store, reads all (or session-filtered) checkpoints,
- * computes cosine similarity edges, temporal edges, and RAPTOR parent edges.
- * Returns a MemoryGraph or an empty graph on failure (non-fatal).
+ * Returns an empty graph (byte-identical pre-change for checkpoint-only) when
+ * no sources are available.
  */
-export function buildMemoryGraph(opts = {}) {
-    const { similarityThreshold = DEFAULT_SIMILARITY_THRESHOLD, maxEdgesPerNode = DEFAULT_MAX_EDGES_PER_NODE, sessionId, stateDir = getStateDir(), includeRaptorEdges = true, includeTemporalEdges = true, } = opts;
-    const builtAt = Date.now();
-    try {
-        const nodes = loadNodes(sessionId, stateDir);
-        const nodeIndex = buildNodeIndex(nodes);
-        const edges = [];
-        // Temporal edges: consecutive checkpoints within the same session
-        if (includeTemporalEdges) {
-            edges.push(...buildTemporalEdges(nodes));
-        }
-        // Semantic edges: cosine similarity above threshold
-        edges.push(...buildSemanticEdges(nodes, nodeIndex, similarityThreshold, maxEdgesPerNode));
-        // RAPTOR parent–child edges
-        if (includeRaptorEdges) {
-            edges.push(...buildRaptorEdges(nodes, sessionId, stateDir, nodeIndex));
-        }
-        // Deduplicate edges (keep highest weight for each source-target pair)
-        const dedupedEdges = deduplicateEdges(edges);
-        // Count sessions
-        const sessionSet = new Set();
-        for (const n of nodes) {
-            sessionSet.add(n.sessionId);
-        }
-        const meta = {
-            sessionCount: sessionSet.size,
-            totalCheckpoints: nodes.length,
-            totalEdges: dedupedEdges.length,
-            semanticEdgeCount: dedupedEdges.filter((e) => e.type === "semantic").length,
-            temporalEdgeCount: dedupedEdges.filter((e) => e.type === "temporal").length,
-            raptorEdgeCount: dedupedEdges.filter((e) => e.type === "raptor_parent").length,
-            similarityThresholdUsed: similarityThreshold,
-            builtAt,
-        };
-        return { nodes, edges: dedupedEdges, metadata: meta };
+export function buildMemoryGraph(sessionId, stateDir = getStateDir()) {
+    const { nodes, edges } = buildUnvalidatedGraph(sessionId, stateDir);
+    // Gate pipeline -- filters, deduplicates, and validates
+    const { ws, stats } = runValidationPipeline({ nodes, edges });
+    const nodeTypeBreakdown = {};
+    const edgeTypeBreakdown = {};
+    for (const n of ws.nodes) {
+        nodeTypeBreakdown[n.nodeType] = (nodeTypeBreakdown[n.nodeType] ?? 0) + 1;
     }
-    catch (err) {
-        log.warn("buildMemoryGraph failed", { error: String(err) });
-        return {
-            nodes: [],
-            edges: [],
-            metadata: {
-                sessionCount: 0,
-                totalCheckpoints: 0,
-                totalEdges: 0,
-                semanticEdgeCount: 0,
-                temporalEdgeCount: 0,
-                raptorEdgeCount: 0,
-                similarityThresholdUsed: similarityThreshold,
-                builtAt,
-            },
-        };
+    for (const e of ws.edges) {
+        edgeTypeBreakdown[e.type] = (edgeTypeBreakdown[e.type] ?? 0) + 1;
     }
-}
-// ---------------------------------------------------------------------------
-// Node loading
-// ---------------------------------------------------------------------------
-function loadNodes(sessionId, stateDir) {
-    const db = openStore(stateDir);
-    const sql = sessionId
-        ? "SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC"
-        : "SELECT * FROM context_chunks ORDER BY session_id, id ASC";
-    const rows = sessionId
-        ? db.prepare(sql).all(normalizeSessionId(sessionId))
-        : db.prepare(sql).all();
-    return rows.map(rowToGraphNode);
-}
-function rowToGraphNode(row) {
-    const summary = row.summary ?? "";
+    const avgWeight = ws.edges.length > 0
+        ? ws.edges.reduce((s, e) => s + e.weight, 0) / ws.edges.length
+        : 0;
+    const sources = countSourcesByType(ws.nodes);
+    const validation = {
+        gatesRun: stats.gatesRun.length,
+        gatesPassed: stats.gatesPassed.length,
+        dropped: stats.dropped,
+        warnings: stats.warnings.map((w) => ({
+            gate: w.gate,
+            code: w.code,
+            count: w.count,
+        })),
+        sources,
+        builtAt: Date.now(),
+    };
     return {
-        id: row.id,
-        sessionId: row.session_id,
-        label: row.id,
-        summaryTruncated: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
-        tokenEstimate: row.token_estimate ?? 0,
-        timestamp: row.created_at ?? Date.now(),
-        dedupStatus: row.dedup_status ?? undefined,
-        raptorLevel: 0, // enriched below if RAPTOR data exists
-        topicSummary: row.topic_summary ?? undefined,
-        decisionCount: row.key_decisions
-            ? safeJsonCount(row.key_decisions)
-            : 0,
-        textSnippet: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
+        nodes: ws.nodes,
+        edges: ws.edges,
+        metadata: {
+            totalNodes: ws.nodes.length,
+            totalEdges: ws.edges.length,
+            avgWeight,
+            nodeTypeBreakdown,
+            edgeTypeBreakdown,
+        },
+        validation,
     };
 }
-/** Parse a JSON array stored in a text column and return its length. */
-function safeJsonCount(raw) {
-    if (!raw)
-        return 0;
-    try {
-        const arr = JSON.parse(raw);
-        return Array.isArray(arr) ? arr.length : 0;
-    }
-    catch {
-        return 0;
-    }
-}
 // ---------------------------------------------------------------------------
-// Node index building
+// Source orchestrator
 // ---------------------------------------------------------------------------
-function buildNodeIndex(nodes) {
-    const idx = new Map();
-    nodes.forEach((n, i) => idx.set(n.id, i));
-    return idx;
-}
-// ---------------------------------------------------------------------------
-// Edges: temporal
-// ---------------------------------------------------------------------------
-function buildTemporalEdges(nodes) {
+function buildUnvalidatedGraph(sessionId, stateDir) {
+    const nodes = [];
     const edges = [];
-    if (nodes.length < 2)
-        return edges;
-    let prev = nodes[0];
-    for (let i = 1; i < nodes.length; i++) {
-        const cur = nodes[i];
-        if (cur.sessionId === prev.sessionId) {
-            edges.push({
-                source: prev.id,
-                target: cur.id,
-                weight: 1.0,
-                type: "temporal",
-            });
-        }
-        prev = cur;
+    let db;
+    try {
+        db = openStore(stateDir);
     }
-    return edges;
-}
-// ---------------------------------------------------------------------------
-// Edges: semantic (cosine similarity)
-// ---------------------------------------------------------------------------
-function buildSemanticEdges(nodes, _nodeIndex, threshold, maxPerNode) {
-    if (nodes.length < 2)
-        return [];
-    const db = openStore();
-    // Retrieve embeddings from DB for each node
-    const embeddings = new Map();
-    const placeholders = nodes.map(() => "?").join(",");
-    const ids = nodes.map((n) => n.id);
-    const rows = db
-        .prepare(`SELECT id, embedding_blob FROM context_chunks WHERE id IN (${placeholders})`)
-        .all(...ids);
-    for (const row of rows) {
-        const emb = decodeEmbedding(row.embedding_blob);
-        if (emb.length > 0) {
-            embeddings.set(row.id, emb);
-        }
+    catch {
+        log.warn("graph_source_unavailable", { detail: "Cannot open store" });
+        return { nodes: [], edges: [] };
     }
-    const edges = [];
-    for (const node of nodes) {
-        const aId = node.id;
-        const aEmb = embeddings.get(aId);
-        if (!aEmb)
-            continue;
-        const candidates = [];
-        for (const other of nodes) {
-            if (other.id === aId)
-                continue;
-            // Skip if already connected via temporal or raptor edge
-            const bEmb = embeddings.get(other.id);
-            if (!bEmb)
-                continue;
-            const sim = cosineSimilarity(aEmb, bEmb);
-            if (sim >= threshold) {
-                candidates.push({ target: other.id, weight: sim });
+    try {
+        // Source: checkpoints (always on)
+        buildCheckpointNodes(db, sessionId, nodes, edges);
+        // Source A: turns (structural, metadata-only)
+        if (areTurnsEnabled()) {
+            try {
+                buildTurnNodes(db, sessionId, nodes, edges);
+            }
+            catch (err) {
+                log.warn("graph_source_unavailable", {
+                    source: "turns",
+                    detail: String(err),
+                });
             }
         }
-        // Keep top-K highest-weight semantic neighbors
-        candidates.sort((a, b) => b.weight - a.weight);
-        const top = candidates.slice(0, maxPerNode);
-        for (const c of top) {
-            edges.push({
-                source: aId,
-                target: c.target,
-                weight: c.weight,
-                type: "semantic",
-            });
+        // Source B: turn content (raw_transcript join, embedding-based)
+        if (isTurnContentEnabled()) {
+            try {
+                buildTurnContentNodes(db, sessionId, nodes, edges);
+            }
+            catch (err) {
+                log.warn("graph_source_unavailable", {
+                    source: "turn-content",
+                    detail: String(err),
+                });
+            }
         }
-    }
-    return edges;
-}
-// ---------------------------------------------------------------------------
-// Edges: RAPTOR parent–child
-// ---------------------------------------------------------------------------
-function buildRaptorEdges(nodes, sessionId, stateDir, nodeIndex) {
-    const edges = [];
-    try {
-        if (sessionId) {
-            enrichRaptorForSession(sessionId, stateDir, nodeIndex, nodes, edges);
+        else if (isTurnContentFlaggedOn() && !isTurnContentEnabled()) {
+            log.warn("graph_source_unavailable", {
+                source: "turn-content",
+                detail: "dbMirror is OFF -- no raw_transcript available",
+            });
         }
-        else {
-            // Load all sessions from DB
-            const db = openStore(stateDir);
-            const sessionRows = db
-                .prepare("SELECT DISTINCT session_id FROM raptor_nodes ORDER BY session_id")
-                .all();
-            for (const row of sessionRows) {
-                enrichRaptorForSession(row.session_id, stateDir, nodeIndex, nodes, edges);
+        // Source C: memories
+        if (areMemoriesEnabled()) {
+            try {
+                buildMemoryNodes(db, sessionId, nodes, edges);
+            }
+            catch (err) {
+                log.warn("graph_source_unavailable", {
+                    source: "memories",
+                    detail: String(err),
+                });
             }
         }
-    }
-    catch (err) {
-        log.warn("RAPTOR edge build failed (non-fatal)", { error: String(err) });
-    }
-    return edges;
-}
-function enrichRaptorForSession(sessId, stateDir, nodeIndex, nodes, edges) {
-    const raptorNodes = listRaptorNodes(sessId, stateDir);
-    for (const rn of raptorNodes) {
-        const idx = nodeIndex.get(rn.id);
-        if (idx !== undefined) {
-            // Annotate the graph node with the RAPTOR level
-            nodes[idx].raptorLevel = rn.level;
+        // RAPTOR annotations on checkpoint nodes
+        try {
+            addRaptorAnnotations(sessionId, stateDir, nodes, edges);
         }
-        if (!rn.parentId)
-            continue;
-        // Only create edge if both nodes are in the graph
-        if (nodeIndex.has(rn.id) && nodeIndex.has(rn.parentId)) {
-            edges.push({
-                source: rn.id,
-                target: rn.parentId,
-                weight: Math.max(0.1, 1.0 - rn.level * 0.1), // higher levels = weaker link
-                type: "raptor_parent",
-            });
+        catch {
+            // non-fatal
         }
     }
-}
-// ---------------------------------------------------------------------------
-// Edge deduplication
-// ---------------------------------------------------------------------------
-function deduplicateEdges(edges) {
-    const seen = new Map();
-    for (const e of edges) {
-        const keyA = `${e.source}|${e.target}`;
-        const keyB = `${e.target}|${e.source}`;
-        const existing = seen.get(keyA) ?? seen.get(keyB);
-        if (!existing) {
-            seen.set(keyA, e);
-        }
-        else if (e.weight > existing.weight) {
-            // Replace with higher-weight edge
-            seen.delete(keyA);
-            seen.delete(keyB);
-            seen.set(keyA, e);
-        }
+    finally {
+        // DatabaseSync has no close in Node 22 -- leave GC to clean up
     }
-    return Array.from(seen.values());
+    // Pre-gate edge deduplication
+    const deduped = deduplicateEdges(edges);
+    edges.length = 0;
+    edges.push(...deduped);
+    return { nodes, edges };
 }
 // ---------------------------------------------------------------------------
-// Cosine similarity
-// ---------------------------------------------------------------------------
-function cosineSimilarity(a, b) {
-    const len = Math.min(a.length, b.length);
-    if (len === 0)
-        return 0;
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < len; i++) {
-        dot += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
+// Helpers
+// ---------------------------------------------------------------------------
+function countSourcesByType(nodes) {
+    let checkpoint = 0;
+    let turn = 0;
+    let turnContent = 0;
+    let memory = 0;
+    for (const n of nodes) {
+        switch (n.nodeType) {
+            case "checkpoint":
+                checkpoint++;
+                break;
+            case "turn":
+                turn++;
+                break;
+            case "turn-content":
+                turnContent++;
+                break;
+            case "memory":
+                memory++;
+                break;
+        }
     }
-    const denom = Math.sqrt(normA) * Math.sqrt(normB);
-    return denom === 0 ? 0 : dot / denom;
+    return { checkpoint, turn, turnContent, memory };
 }

package/dist/src/recall.js

@@ -20,6 +20,19 @@ import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
 import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
 import { maxCheckpointTimestamp } from "./store/sqlite.js";
 import { normalizeSessionId } from "./store.js";
+// ---------------------------------------------------------------------------
+// S57 RAG suite feature flags + module imports. Each flag defaults OFF — the
+// flag-OFF path below is byte-identical to the pre-S57 recall path (recallQuery
+// === opts.query, the single searchRecall call), so the helpers are only reached
+// when their flag is ON. Wiring is additive; the proven v0.11.5 behavior is the
+// default.
+// ---------------------------------------------------------------------------
+import { RAG_QUERY_REFORMULATION, RAG_TIERED_ROUTER, RAG_RECALL_METRICS } from "./config.js";
+import { reformulateQuery, isVagueQuery } from "./queryReformulation.js";
+import { VAGUE_MIN_WORDS, VAGUE_VERY_SHORT_WORDS } from "./queryReformulation/cache.js";
+import { getTieredRouter } from "./tieredRouter.js";
+import { computeRecallMetrics } from "./recallMetrics.js";
+import { Logger } from "./log.js";
 /** Wrap a recall block so the model reads it as restored compacted context. */
 export function formatRecallBlock(hits) {
     if (hits.length === 0)
@@ -72,6 +85,107 @@ export function formatRaptorBlock(nodes) {
         "session so far. Use it as a map of what has been covered:\n\n" +
         parts.join("\n"));
 }
+// ---------------------------------------------------------------------------
+// S57 RAG suite helpers (B1/B2/B3). Each is only called when its feature flag
+// is ON; flag-OFF never reaches them. All degrade non-fatally to the standard
+// path on any error.
+// ---------------------------------------------------------------------------
+/**
+ * B1: If the query is vague, reformulate it via embedding-neighbor TF-IDF
+ * expansion. Returns the original query when reformulation is not supported
+ * or the query is already specific. Non-fatal: any error → original query.
+ */
+function reformulateRecallQuery(query, store, sessionId) {
+    try {
+        if (!isVagueQuery(query, {
+            vagueMinWords: VAGUE_MIN_WORDS,
+            vagueVeryShortWords: VAGUE_VERY_SHORT_WORDS,
+        }))
+            return query;
+        // Pre-compute neighbor candidates once. reformulateQuery embeds the query
+        // and calls `scan` internally; we return these pre-computed neighbors (same
+        // query) so the store needs no by-embedding lookup.
+        const { hits } = searchRecall({ sessionId, query, limit: 10, skipInjected: false }, store);
+        if (hits.length < 2)
+            return query; // too few neighbors for TF-IDF
+        const neighbors = hits.map((h) => ({
+            id: h.checkpoint.checkpointId,
+            score: h.score,
+        }));
+        const textById = new Map();
+        for (const h of hits)
+            textById.set(h.checkpoint.checkpointId, h.checkpoint.summary);
+        const scan = () => neighbors;
+        const corpus = {
+            totalDocs: neighbors.length,
+            docFreq: (term) => {
+                let df = 0;
+                for (const text of textById.values()) {
+                    if (text.toLowerCase().includes(term.toLowerCase()))
+                        df++;
+                }
+                return Math.max(df, 1);
+            },
+        };
+        const neighborTexts = (ids) => ids.map((id) => ({ id, text: textById.get(id) ?? "" }));
+        const { result } = reformulateQuery(query, defaultEmbedder(), scan, corpus, neighborTexts);
+        return result.expanded.length > query.length ? result.expanded : query;
+    }
+    catch {
+        return query; // non-fatal: fall back to original query
+    }
+}
+/**
+ * B2: Synchronous tiered recall. The TieredRouter.route() is async, so the
+ * sync path can only peek the L0 in-memory cache; on miss it falls through to
+ * the standard sync vector scan. The router's L1/L2 value is realized on the
+ * async path (recallAndInlineAsync). Non-fatal.
+ */
+function runTieredRecall(query, sessionId, limit, skip, store) {
+    try {
+        const router = getTieredRouter();
+        if (!router) {
+            const result = searchRecall({ sessionId, query, limit, skipInjected: skip }, store);
+            return { newHits: result.newHits, tier: "off" };
+        }
+        // Peek the L0 in-memory cache synchronously; fall through on miss.
+        const cached = router.peekCache(sessionId, query, limit);
+        if (cached && cached.length > 0)
+            return { newHits: cached, tier: "L0-cache" };
+        const result = searchRecall({ sessionId, query, limit, skipInjected: skip }, store);
+        return { newHits: result.newHits, tier: "sync-fallback" };
+    }
+    catch {
+        const result = searchRecall({ sessionId, query, limit, skipInjected: skip }, store);
+        return { newHits: result.newHits, tier: "fallback-error" };
+    }
+}
+/**
+ * B3: Compute recall-quality metrics on the injected hits and log them.
+ * Only called when RAG_RECALL_METRICS() is ON. Non-fatal.
+ */
+function scoreAndLogRecallMetrics(query, toInject) {
+    try {
+        const logger = new Logger();
+        const metrics = computeRecallMetrics(query, toInject);
+        logger.info("recall_metrics", {
+            hitCount: toInject.length,
+            score: metrics.score,
+            pass: metrics.pass,
+        });
+        if (!metrics.pass && toInject.length > 0) {
+            logger.info("recall_metrics_low_quality", {
+                score: metrics.score,
+                relevance: metrics.breakdown.relevance,
+                coverage: metrics.breakdown.coverage,
+                diversity: metrics.breakdown.diversity,
+            });
+        }
+    }
+    catch {
+        /* non-fatal: metrics never break recall */
+    }
+}
 /**
  * Run the unified recall+dudupe+prepare-inject pipeline. Does NOT touch pi;
  * it records injections via `markInjected` so the next call dedupes. The
@@ -123,7 +237,22 @@ export function recallAndInline(opts, store) {
     // F4: thread skipInjected through to searchRecall instead of hardcoding false
     // and re-implementing the filter here. newHits is already deduped when skip
     // is true (default); equals hits when skip is false (openclaw command path).
-    const { newHits } = searchRecall({ sessionId: opts.sessionId, query: opts.query, limit, skipInjected: skip }, store);
+    //
+    // S57 B1: optionally expand a vague query via TF-IDF neighbor terms.
+    // S57 B2: optionally route via the TieredRouter's sync L0 cache peek.
+    // Both flags default OFF — recallQuery === opts.query and the single
+    // searchRecall call reproduce the pre-S57 byte-identical path.
+    const recallQuery = RAG_QUERY_REFORMULATION()
+        ? reformulateRecallQuery(opts.query, store, opts.sessionId)
+        : opts.query;
+    let newHits;
+    if (RAG_TIERED_ROUTER()) {
+        newHits = runTieredRecall(recallQuery, opts.sessionId, limit, skip, store).newHits;
+    }
+    else {
+        const result = searchRecall({ sessionId: opts.sessionId, query: recallQuery, limit, skipInjected: skip }, store);
+        newHits = result.newHits;
+    }
     // F1: hoist one embedder instance for inline dedupe (matches the async path).
     // defaultEmbedder() is deterministic but creating it per hit wastes allocations.
     const embedder = defaultEmbedder();
@@ -154,6 +283,12 @@ export function recallAndInline(opts, store) {
         blockTokens += partTokens;
         vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
     }
+    // S57 B3: optionally score + log recall quality metrics (flag-OFF: skipped,
+    // byte-identical). Scores the injected hits against the ORIGINAL query so the
+    // metric measures whether the (possibly expanded) search results stay
+    // relevant to what the user actually asked.
+    if (RAG_RECALL_METRICS())
+        scoreAndLogRecallMetrics(opts.query, toInject);
     // F3: format once — one preamble, correct [1..n] numbering.
     const recallBlock = toInject.length > 0 ? formatRecallBlock(toInject) : "";
     const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);