pi-mega-compact 0.13.0 → 0.13.2

package/dist/src/hyde.js

@@ -0,0 +1,146 @@
+/**
+ * hyde.ts — Hypothetical Document Embeddings for LLM-backed recall (S43 re-plan).
+ *
+ * When an HttpEmbedder (localhost Ollama) is active and RAG_HYDE_ENABLED, we
+ * generate a hypothetical answer to the recall query via the Ollama /api/chat
+ * endpoint, embed it, and RRF-fuse the resulting search hits with the raw-query
+ * hits. This gives a stronger recall signal than TF-IDF term expansion (S57 B1)
+ * when an LLM is available. Non-fatal: any error → returns "" (caller uses raw-only).
+ *
+ * PREVENT-PI-004: reuses HttpEmbedder's loopback-child-process pattern (spawnSync
+ * with its own event loop). The remote opt-in (MEGACOMPACT_ALLOW_REMOTE_EMBEDDER)
+ * governs this too — derived chat URL is on the same host as the embedding URL.
+ */
+import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only LLM generation — reuses HttpEmbedder's loopback exception; governed by MEGACOMPACT_ALLOW_REMOTE_EMBEDDER
+import { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+import { Logger } from "./log.js";
+// Inline worker: POSTs to the derived Ollama /api/chat endpoint in a child
+// process (its own event loop, so spawnSync blocks without a fetch deadlock).
+const CHAT_WORKER = String.raw `
+const u = process.env.MC_HYDE_URL, b = process.env.MC_HYDE_BODY;
+try {
+  const r = await fetch(u, { method: "POST", headers: { "content-type": "application/json" }, body: b }); // guardrails-allow PREVENT-PI-004: localhost-only /api/chat — same host as BYO embedding endpoint; remote only via MEGACOMPACT_ALLOW_REMOTE_EMBEDDER
+  const out = JSON.stringify({ status: r.status, ok: r.ok, text: await r.text() });
+  process.stdout.write(out);
+} catch (e) {
+  process.stdout.write(JSON.stringify({ error: String(e && e.message ? e.message : e) }));
+}
+`;
+/**
+ * Generate a hypothetical answer document for a recall query via the LLM-hosted
+ * /api/chat endpoint (HttpEmbedder only). Returns "" when the embedder has no
+ * LLM chat surface (TrigramEmbedder) or on ANY error. Non-fatal — the caller
+ * falls back to raw-query recall only.
+ */
+/** Best-effort structured failure log. Never throws (PyDE is non-fatal). */
+function logHydeFailure(reason) {
+    try {
+        new Logger().warn("hyde_failed", { reason });
+    }
+    catch {
+        /* ignore — logging must never break recall */
+    }
+}
+export function generateHypotheticalDoc(query, embedder) {
+    if (embedder.kind !== "http")
+        return ""; // TrigramEmbedder: no LLM surface
+    const url = embedder.chatUrl;
+    if (!url) {
+        logHydeFailure("no chatUrl");
+        return "";
+    }
+    const model = process.env.MEGACOMPACT_HYDE_MODEL ?? "llama3.2";
+    const content = "Answer this question about the codebase concisely (2-3 sentences): " + query;
+    const body = JSON.stringify({
+        model,
+        messages: [{ role: "user", content }],
+        stream: false,
+    });
+    let res;
+    try {
+        res = spawnSync(process.execPath, ["-e", CHAT_WORKER], {
+            encoding: "utf8",
+            env: { ...process.env, MC_HYDE_URL: url, MC_HYDE_BODY: body },
+            timeout: 20000,
+        });
+    }
+    catch {
+        logHydeFailure("spawn-error");
+        return "";
+    }
+    if (res.error ||
+        typeof res.stdout !== "string" ||
+        res.stdout.length === 0 ||
+        res.status !== 0) {
+        logHydeFailure("child-error");
+        return "";
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(res.stdout);
+    }
+    catch {
+        logHydeFailure("non-json");
+        return "";
+    }
+    if (parsed.error || !parsed.ok || (parsed.status ?? 0) >= 400) {
+        logHydeFailure("chat-http");
+        return "";
+    }
+    // Ollama /api/chat (stream:false) returns { message: { role, content } }.
+    let messageBody;
+    try {
+        if (typeof parsed.text === "string")
+            messageBody = JSON.parse(parsed.text);
+        else {
+            logHydeFailure("empty-body");
+            return "";
+        }
+    }
+    catch {
+        logHydeFailure("non-json-body");
+        return "";
+    }
+    if (!messageBody || typeof messageBody !== "object") {
+        logHydeFailure("bad-body");
+        return "";
+    }
+    const msg = messageBody.message;
+    if (!msg || typeof msg !== "object") {
+        logHydeFailure("no-message");
+        return "";
+    }
+    const contentOut = msg.content;
+    if (typeof contentOut !== "string" || contentOut.length === 0) {
+        logHydeFailure("empty-content");
+        return "";
+    }
+    return contentOut;
+}
+/**
+ * RRF-fuse two recall hit lists (raw-query and hypothetical-doc) into a single
+ * deduplicated ranking, sliced to `limit`. Order is by fused RRF score.
+ */
+export function fuseRecallHits(raw, hyde, limit) {
+    const byId = new Map();
+    const rawRanked = raw.map((h, i) => {
+        byId.set(h.checkpoint.checkpointId, h);
+        return { id: h.checkpoint.checkpointId, rank: i + 1 };
+    });
+    const hydeRanked = hyde.map((h, i) => {
+        if (!byId.has(h.checkpoint.checkpointId)) {
+            byId.set(h.checkpoint.checkpointId, h);
+        }
+        return { id: h.checkpoint.checkpointId, rank: i + 1 };
+    });
+    const fused = reciprocalRankFusion([rawRanked, hydeRanked]);
+    const out = [];
+    for (const item of fused) {
+        const hit = byId.get(item.id);
+        if (hit)
+            out.push(hit);
+        if (out.length >= limit)
+            break;
+    }
+    return out;
+}

package/dist/src/importance.test/_helpers.js

@@ -0,0 +1,5 @@
+/**
+ * Shared helpers for the importance.test split files.
+ * Extracted from src/importance.test.ts: the fixed NOW epoch constant.
+ */
+export const NOW = 1_700_000_000_000; // fixed epoch ms for determinism

package/dist/src/memoryGraph/sources/checkpoints.js

@@ -0,0 +1,78 @@
+import { decodeEmbedding } from "../../store/sqlite/utils.js";
+import { cosineSimilarity } from "../embedding.js";
+import { safeJsonCount } from "./helpers.js";
+export function buildCheckpointNodes(db, sessionId, nodes, edges) {
+    const rows = sessionId
+        ? db
+            .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
+                  dedup_status, topic_summary, key_decisions,
+                  normalized_text, embedding_blob
+           FROM context_chunks
+           WHERE session_id = ?
+           ORDER BY timestamp ASC`)
+            .all(sessionId)
+        : db
+            .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
+                  dedup_status, topic_summary, key_decisions,
+                  normalized_text, embedding_blob
+           FROM context_chunks
+           ORDER BY timestamp ASC`)
+            .all();
+    if (rows.length === 0)
+        return;
+    let prevId = null;
+    for (const row of rows) {
+        const id = String(row.id);
+        const rowSessionId = sessionId || String(row.session_id ?? "");
+        const summary = String(row.summary ?? "");
+        const node = {
+            id,
+            sessionId: rowSessionId,
+            label: id,
+            summaryTruncated: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
+            tokenEstimate: Number(row.token_estimate ?? 0),
+            timestamp: Number(row.timestamp ?? Date.now()),
+            dedupStatus: row.dedup_status ? String(row.dedup_status) : undefined,
+            raptorLevel: 0,
+            topicSummary: row.topic_summary ? String(row.topic_summary) : undefined,
+            decisionCount: row.key_decisions ? safeJsonCount(String(row.key_decisions)) : 0,
+            textSnippet: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
+            nodeType: "checkpoint",
+        };
+        nodes.push(node);
+        if (prevId !== null) {
+            edges.push({ source: prevId, target: id, weight: 1.0, type: "temporal" });
+        }
+        prevId = id;
+    }
+    addCheckpointSemanticEdges(rows, edges);
+}
+function addCheckpointSemanticEdges(rows, edges) {
+    const embeddings = [];
+    for (const row of rows) {
+        const blob = row.embedding_blob;
+        embeddings.push({
+            id: String(row.id),
+            blob: blob instanceof Buffer ? blob : null,
+        });
+    }
+    for (let i = 0; i < embeddings.length; i++) {
+        if (!embeddings[i].blob)
+            continue;
+        const a = decodeEmbedding(embeddings[i].blob);
+        for (let j = i + 1; j < embeddings.length; j++) {
+            if (!embeddings[j].blob)
+                continue;
+            const b = decodeEmbedding(embeddings[j].blob);
+            const sim = cosineSimilarity(a, b);
+            if (sim > 0) {
+                edges.push({
+                    source: embeddings[i].id,
+                    target: embeddings[j].id,
+                    weight: sim,
+                    type: "semantic",
+                });
+            }
+        }
+    }
+}

package/dist/src/memoryGraph/sources/edges.js

@@ -0,0 +1,17 @@
+export function deduplicateEdges(edges) {
+    const seen = new Map();
+    for (const e of edges) {
+        const keyA = `${e.source}|${e.target}`;
+        const keyB = `${e.target}|${e.source}`;
+        const existing = seen.get(keyA) ?? seen.get(keyB);
+        if (existing) {
+            if (e.weight > existing.weight) {
+                seen.set(keyA, e);
+            }
+        }
+        else {
+            seen.set(keyA, e);
+        }
+    }
+    return [...seen.values()];
+}

package/dist/src/memoryGraph/sources/flags.js

@@ -0,0 +1,22 @@
+/**
+ * flags.ts — memory-graph feature-flag / env helpers (extracted from sources.ts).
+ */
+export function flagEnabled(name, def) {
+    const v = process.env[name];
+    if (v === undefined)
+        return def;
+    return v === "true" || v === "1";
+}
+export function areTurnsEnabled() {
+    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true);
+}
+export function isTurnContentEnabled() {
+    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true) &&
+        flagEnabled("MEGACOMPACT_DB_MIRROR", false);
+}
+export function isTurnContentFlaggedOn() {
+    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true);
+}
+export function areMemoriesEnabled() {
+    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true);
+}

package/dist/src/memoryGraph/sources/helpers.js

@@ -0,0 +1,14 @@
+/**
+ * helpers.ts — shared micro-helpers (extracted from sources.ts).
+ */
+export function safeJsonCount(raw) {
+    if (!raw || typeof raw !== "string")
+        return 0;
+    try {
+        const arr = JSON.parse(raw);
+        return Array.isArray(arr) ? arr.length : 0;
+    }
+    catch {
+        return 0;
+    }
+}

package/dist/src/memoryGraph/sources/memories.js

@@ -0,0 +1,88 @@
+import { getOrComputeEmbedding, cosineSimilarity } from "../embedding.js";
+export function buildMemoryNodes(db, sessionId, nodes, edges) {
+    const existingIds = new Set(nodes.map((n) => n.id));
+    const rows = db
+        .prepare(`SELECT id, content, source_turn, created_at
+       FROM memories
+       ORDER BY created_at ASC`)
+        .all();
+    if (rows.length === 0)
+        return;
+    const memoryNodes = [];
+    for (const row of rows) {
+        const id = `mem:${String(row.id)}`;
+        if (existingIds.has(id))
+            continue;
+        existingIds.add(id);
+        const content = row.content ? String(row.content) : "";
+        const snippet = content.length > 200 ? content.slice(0, 197) + "..." : content;
+        const node = {
+            id,
+            sessionId,
+            label: `Memory ${String(row.id)}`,
+            summaryTruncated: snippet,
+            tokenEstimate: content.length,
+            timestamp: row.created_at ? Number(row.created_at) : Date.now(),
+            dedupStatus: undefined,
+            raptorLevel: 0,
+            topicSummary: undefined,
+            decisionCount: 0,
+            textSnippet: snippet,
+            nodeType: "memory",
+            epochId: undefined,
+        };
+        memoryNodes.push(node);
+    }
+    if (memoryNodes.length > 1) {
+        const embeddings = memoryNodes.map((n) => {
+            try {
+                return { id: n.id, vec: getOrComputeEmbedding(db, n.textSnippet) };
+            }
+            catch {
+                return { id: n.id, vec: null };
+            }
+        });
+        for (let i = 0; i < embeddings.length; i++) {
+            if (!embeddings[i].vec)
+                continue;
+            for (let j = i + 1; j < embeddings.length; j++) {
+                if (!embeddings[j].vec)
+                    continue;
+                const sim = cosineSimilarity(embeddings[i].vec, embeddings[j].vec);
+                if (sim > 0) {
+                    edges.push({
+                        source: embeddings[i].id,
+                        target: embeddings[j].id,
+                        weight: sim,
+                        type: "semantic",
+                    });
+                }
+            }
+        }
+        crossLinkMemoryToCheckpoints(memoryNodes, nodes, edges);
+    }
+    nodes.push(...memoryNodes);
+}
+function crossLinkMemoryToCheckpoints(memoryNodes, allNodes, edges) {
+    for (const mn of memoryNodes) {
+        let bestNeighbor = null;
+        for (const n of allNodes) {
+            if (n.nodeType !== "checkpoint" && n.nodeType !== "turn-content")
+                continue;
+            if (n.id === mn.id)
+                continue;
+            const diff = Math.abs(mn.timestamp - n.timestamp);
+            if (bestNeighbor === null || diff < bestNeighbor.diff) {
+                bestNeighbor = { id: n.id, diff };
+            }
+        }
+        if (bestNeighbor) {
+            edges.push({
+                source: mn.id,
+                target: bestNeighbor.id,
+                weight: 0.85,
+                type: "semantic",
+            });
+        }
+    }
+}

package/dist/src/memoryGraph/sources/raptor.js

@@ -0,0 +1,26 @@
+import { listRaptorNodes } from "../../store/sqlite/raptor.js";
+export function addRaptorAnnotations(sessionId, stateDir, nodes, edges) {
+    const raptorNodes = listRaptorNodes(sessionId, stateDir);
+    if (!raptorNodes || raptorNodes.length === 0)
+        return;
+    const nodeIndex = new Map();
+    for (let i = 0; i < nodes.length; i++) {
+        nodeIndex.set(nodes[i].id, i);
+    }
+    for (const rn of raptorNodes) {
+        const idx = nodeIndex.get(rn.id);
+        if (idx !== undefined) {
+            nodes[idx].raptorLevel = rn.level;
+        }
+        if (!rn.parentId)
+            continue;
+        if (nodeIndex.has(rn.id) && nodeIndex.has(rn.parentId)) {
+            edges.push({
+                source: rn.id,
+                target: rn.parentId,
+                weight: Math.max(0.1, 1.0 - rn.level * 0.1),
+                type: "raptor_parent",
+            });
+        }
+    }
+}

package/dist/src/memoryGraph/sources/turns.js

@@ -0,0 +1,176 @@
+import { getOrComputeEmbedding, cosineSimilarity } from "../embedding.js";
+export function buildTurnNodes(db, sessionId, nodes, edges) {
+    const existingIds = new Set(nodes.map((n) => n.id));
+    const rows = sessionId
+        ? db
+            .prepare(`SELECT turn_index, session_id, role, pressure_band, ctx_tokens, ctx_percent,
+                  epoch_id, ended_at
+           FROM turns
+           WHERE session_id = ?
+           ORDER BY turn_index ASC`)
+            .all(sessionId)
+        : db
+            .prepare(`SELECT turn_index, session_id, role, pressure_band, ctx_tokens, ctx_percent,
+                  epoch_id, ended_at
+           FROM turns
+           ORDER BY turn_index ASC`)
+            .all();
+    if (rows.length === 0)
+        return;
+    let prevId = null;
+    for (const row of rows) {
+        const ti = Number(row.turn_index);
+        const rowSessionId = sessionId || String(row.session_id ?? "");
+        const nodeId = `turn:${rowSessionId}:${ti}`;
+        if (existingIds.has(nodeId))
+            continue;
+        existingIds.add(nodeId);
+        const role = row.role ? String(row.role) : "unknown";
+        const endedAt = row.ended_at ? Number(row.ended_at) : Date.now();
+        const node = {
+            id: nodeId,
+            sessionId: rowSessionId,
+            label: `Turn ${ti}`,
+            summaryTruncated: `Turn ${ti}: ${role}${row.epoch_id ? ` (epoch: ${String(row.epoch_id)})` : ""}`,
+            tokenEstimate: row.ctx_tokens ? Number(row.ctx_tokens) : 0,
+            timestamp: endedAt,
+            dedupStatus: undefined,
+            raptorLevel: 0,
+            topicSummary: undefined,
+            decisionCount: 0,
+            textSnippet: `Turn ${ti}: ${role} — ${String(row.ctx_tokens ?? "?")} tokens`,
+            nodeType: "turn",
+            epochId: row.epoch_id ? String(row.epoch_id) : undefined,
+        };
+        nodes.push(node);
+        if (prevId !== null) {
+            edges.push({ source: prevId, target: nodeId, weight: 1.0, type: "temporal" });
+        }
+        prevId = nodeId;
+    }
+}
+export function buildTurnContentNodes(turnDb, mainDb, sessionId, nodes, edges) {
+    // Local dedup only — DON'T skip ids from Source A (turn-structural). Source A
+    // and Source B share the same node id (turn:<session>:<idx>); Gate 2
+    // (identity_merge) merges them with the richest nodeType winning. Using the
+    // global nodes array here would skip every node (Source A already added them).
+    const seen = new Set();
+    const turnRows = sessionId
+        ? turnDb
+            .prepare(`SELECT turn_index, role, ended_at
+           FROM turns
+           WHERE session_id = ?
+           ORDER BY turn_index ASC`)
+            .all(sessionId)
+        : turnDb
+            .prepare(`SELECT turn_index, role, ended_at
+           FROM turns
+           ORDER BY turn_index ASC`)
+            .all();
+    // Build a map of (session_id, turn_index) → content from raw_transcript.
+    const contentMap = new Map();
+    try {
+        const transcriptRows = sessionId
+            ? mainDb
+                .prepare(`SELECT turn_index, content_bytes FROM raw_transcript WHERE session_id = ?`)
+                .all(sessionId)
+            : mainDb
+                .prepare(`SELECT turn_index, content_bytes FROM raw_transcript`)
+                .all();
+        for (const r of transcriptRows) {
+            contentMap.set(String(r.turn_index), String(r.content_bytes ?? ""));
+        }
+    }
+    catch {
+        // raw_transcript may not exist in the main db (dbMirror off) — return empty.
+        return;
+    }
+    // Merge: only produce nodes where both a turn row AND transcript content exist.
+    const rows = turnRows
+        .filter((t) => contentMap.has(String(t.turn_index)))
+        .map((t) => ({
+        turn_index: t.turn_index,
+        role: t.role,
+        ended_at: t.ended_at,
+        content_bytes: contentMap.get(String(t.turn_index)) ?? "",
+    }));
+    if (rows.length === 0)
+        return;
+    const turnContentNodes = [];
+    for (const row of rows) {
+        const ti = Number(row.turn_index);
+        const nodeId = `turn:${sessionId}:${ti}`;
+        if (seen.has(nodeId))
+            continue;
+        seen.add(nodeId);
+        const contentBytes = row.content_bytes ? String(row.content_bytes) : "";
+        const snippet = contentBytes.length > 200 ? contentBytes.slice(0, 197) + "..." : contentBytes;
+        const node = {
+            id: nodeId,
+            sessionId,
+            label: `Turn ${ti}`,
+            summaryTruncated: snippet,
+            tokenEstimate: contentBytes.length,
+            timestamp: row.ended_at ? Number(row.ended_at) : Date.now(),
+            dedupStatus: undefined,
+            raptorLevel: 0,
+            topicSummary: undefined,
+            decisionCount: 0,
+            textSnippet: snippet,
+            nodeType: "turn-content",
+            epochId: undefined,
+        };
+        turnContentNodes.push(node);
+    }
+    if (turnContentNodes.length > 1) {
+        const embeddings = turnContentNodes.map((n) => {
+            try {
+                return { id: n.id, vec: getOrComputeEmbedding(mainDb, n.textSnippet) };
+            }
+            catch {
+                return { id: n.id, vec: null };
+            }
+        });
+        for (let i = 0; i < embeddings.length; i++) {
+            if (!embeddings[i].vec)
+                continue;
+            for (let j = i + 1; j < embeddings.length; j++) {
+                if (!embeddings[j].vec)
+                    continue;
+                const sim = cosineSimilarity(embeddings[i].vec, embeddings[j].vec);
+                if (sim > 0) {
+                    edges.push({
+                        source: embeddings[i].id,
+                        target: embeddings[j].id,
+                        weight: sim,
+                        type: "semantic",
+                    });
+                }
+            }
+        }
+        linkTurnToCheckpointEdges(mainDb, sessionId, turnContentNodes, edges);
+    }
+    nodes.push(...turnContentNodes);
+}
+function linkTurnToCheckpointEdges(db, sessionId, turnNodes, edges) {
+    const checkpoints = db
+        .prepare(`SELECT id, timestamp FROM context_chunks WHERE session_id = ? ORDER BY timestamp ASC`)
+        .all(sessionId);
+    for (const tn of turnNodes) {
+        let bestCp = null;
+        for (const cp of checkpoints) {
+            const diff = Math.abs(tn.timestamp - cp.timestamp);
+            if (bestCp === null || diff < bestCp.diff) {
+                bestCp = { id: cp.id, diff };
+            }
+        }
+        if (bestCp) {
+            edges.push({
+                source: tn.id,
+                target: bestCp.id,
+                weight: 1.0,
+                type: "temporal",
+            });
+        }
+    }
+}