pi-mega-compact 0.13.0 → 0.13.2

package/dist/src/memoryGraph/sources.js

@@ -1,441 +1,15 @@
-import { listRaptorNodes } from "../store/sqlite/raptor.js";
-import { decodeEmbedding } from "../store/sqlite/utils.js";
-import { getOrComputeEmbedding, cosineSimilarity } from "./embedding.js";
-// ---------------------------------------------------------------------------
-// Feature flags and env helpers
-// ---------------------------------------------------------------------------
-function flagEnabled(name, def) {
-    const v = process.env[name];
-    if (v === undefined)
-        return def;
-    return v === "true" || v === "1";
-}
-export function areTurnsEnabled() {
-    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS", true);
-}
-export function isTurnContentEnabled() {
-    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true) &&
-        flagEnabled("MEGACOMPACT_DB_MIRROR", false);
-}
-export function isTurnContentFlaggedOn() {
-    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_TURN_CONTENT", true);
-}
-export function areMemoriesEnabled() {
-    return flagEnabled("MEGACOMPACT_MEMORY_GRAPH_SEED_MEMORIES", true);
-}
-// ---------------------------------------------------------------------------
-// Source: checkpoints (from context_chunks table)
-// ---------------------------------------------------------------------------
-export function buildCheckpointNodes(db, sessionId, nodes, edges) {
-    const rows = sessionId
-        ? db
-            .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
-                  dedup_status, topic_summary, key_decisions,
-                  normalized_text, embedding_blob
-           FROM context_chunks
-           WHERE session_id = ?
-           ORDER BY timestamp ASC`)
-            .all(sessionId)
-        : db
-            .prepare(`SELECT id, session_id, summary, token_estimate, timestamp,
-                  dedup_status, topic_summary, key_decisions,
-                  normalized_text, embedding_blob
-           FROM context_chunks
-           ORDER BY timestamp ASC`)
-            .all();
-    if (rows.length === 0)
-        return;
-    let prevId = null;
-    for (const row of rows) {
-        const id = String(row.id);
-        const rowSessionId = sessionId || String(row.session_id ?? "");
-        const summary = String(row.summary ?? "");
-        const node = {
-            id,
-            sessionId: rowSessionId,
-            label: id,
-            summaryTruncated: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
-            tokenEstimate: Number(row.token_estimate ?? 0),
-            timestamp: Number(row.timestamp ?? Date.now()),
-            dedupStatus: row.dedup_status ? String(row.dedup_status) : undefined,
-            raptorLevel: 0,
-            topicSummary: row.topic_summary ? String(row.topic_summary) : undefined,
-            decisionCount: row.key_decisions ? safeJsonCount(String(row.key_decisions)) : 0,
-            textSnippet: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
-            nodeType: "checkpoint",
-        };
-        nodes.push(node);
-        if (prevId !== null) {
-            edges.push({ source: prevId, target: id, weight: 1.0, type: "temporal" });
-        }
-        prevId = id;
-    }
-    addCheckpointSemanticEdges(rows, edges);
-}
-function addCheckpointSemanticEdges(rows, edges) {
-    const embeddings = [];
-    for (const row of rows) {
-        const blob = row.embedding_blob;
-        embeddings.push({
-            id: String(row.id),
-            blob: blob instanceof Buffer ? blob : null,
-        });
-    }
-    for (let i = 0; i < embeddings.length; i++) {
-        if (!embeddings[i].blob)
-            continue;
-        const a = decodeEmbedding(embeddings[i].blob);
-        for (let j = i + 1; j < embeddings.length; j++) {
-            if (!embeddings[j].blob)
-                continue;
-            const b = decodeEmbedding(embeddings[j].blob);
-            const sim = cosineSimilarity(a, b);
-            if (sim > 0) {
-                edges.push({
-                    source: embeddings[i].id,
-                    target: embeddings[j].id,
-                    weight: sim,
-                    type: "semantic",
-                });
-            }
-        }
-    }
-}
-// ---------------------------------------------------------------------------
-// Source A: turn structural nodes (from turns table, metadata-only)
-// ---------------------------------------------------------------------------
-export function buildTurnNodes(db, sessionId, nodes, edges) {
-    const existingIds = new Set(nodes.map((n) => n.id));
-    const rows = sessionId
-        ? db
-            .prepare(`SELECT turn_index, session_id, role, pressure_band, ctx_tokens, ctx_percent,
-                  epoch_id, ended_at
-           FROM turns
-           WHERE session_id = ?
-           ORDER BY turn_index ASC`)
-            .all(sessionId)
-        : db
-            .prepare(`SELECT turn_index, session_id, role, pressure_band, ctx_tokens, ctx_percent,
-                  epoch_id, ended_at
-           FROM turns
-           ORDER BY turn_index ASC`)
-            .all();
-    if (rows.length === 0)
-        return;
-    let prevId = null;
-    for (const row of rows) {
-        const ti = Number(row.turn_index);
-        const rowSessionId = sessionId || String(row.session_id ?? "");
-        const nodeId = `turn:${rowSessionId}:${ti}`;
-        if (existingIds.has(nodeId))
-            continue;
-        existingIds.add(nodeId);
-        const role = row.role ? String(row.role) : "unknown";
-        const endedAt = row.ended_at ? Number(row.ended_at) : Date.now();
-        const node = {
-            id: nodeId,
-            sessionId: rowSessionId,
-            label: `Turn ${ti}`,
-            summaryTruncated: `Turn ${ti}: ${role}${row.epoch_id ? ` (epoch: ${String(row.epoch_id)})` : ""}`,
-            tokenEstimate: row.ctx_tokens ? Number(row.ctx_tokens) : 0,
-            timestamp: endedAt,
-            dedupStatus: undefined,
-            raptorLevel: 0,
-            topicSummary: undefined,
-            decisionCount: 0,
-            textSnippet: `Turn ${ti}: ${role} — ${String(row.ctx_tokens ?? "?")} tokens`,
-            nodeType: "turn",
-            epochId: row.epoch_id ? String(row.epoch_id) : undefined,
-        };
-        nodes.push(node);
-        if (prevId !== null) {
-            edges.push({ source: prevId, target: nodeId, weight: 1.0, type: "temporal" });
-        }
-        prevId = nodeId;
-    }
-}
-// ---------------------------------------------------------------------------
-// Source B: turn-content nodes (from turns JOIN raw_transcript)
-// ---------------------------------------------------------------------------
-export function buildTurnContentNodes(turnDb, mainDb, sessionId, nodes, edges) {
-    // Local dedup only — DON'T skip ids from Source A (turn-structural). Source A
-    // and Source B share the same node id (turn:<session>:<idx>); Gate 2
-    // (identity_merge) merges them with the richest nodeType winning. Using the
-    // global nodes array here would skip every node (Source A already added them).
-    const seen = new Set();
-    // turns live in turns.db, raw_transcript in the main db — can't JOIN across
-    // databases, so query separately and merge in JS.
-    const turnRows = sessionId
-        ? turnDb
-            .prepare(`SELECT turn_index, role, ended_at
-           FROM turns
-           WHERE session_id = ?
-           ORDER BY turn_index ASC`)
-            .all(sessionId)
-        : turnDb
-            .prepare(`SELECT turn_index, role, ended_at
-           FROM turns
-           ORDER BY turn_index ASC`)
-            .all();
-    // Build a map of (session_id, turn_index) → content from raw_transcript.
-    const contentMap = new Map();
-    try {
-        const transcriptRows = sessionId
-            ? mainDb
-                .prepare(`SELECT turn_index, content_bytes FROM raw_transcript WHERE session_id = ?`)
-                .all(sessionId)
-            : mainDb
-                .prepare(`SELECT turn_index, content_bytes FROM raw_transcript`)
-                .all();
-        for (const r of transcriptRows) {
-            contentMap.set(String(r.turn_index), String(r.content_bytes ?? ""));
-        }
-    }
-    catch {
-        // raw_transcript may not exist in the main db (dbMirror off) — return empty.
-        return;
-    }
-    // Merge: only produce nodes where both a turn row AND transcript content exist.
-    const rows = turnRows
-        .filter((t) => contentMap.has(String(t.turn_index)))
-        .map((t) => ({
-        turn_index: t.turn_index,
-        role: t.role,
-        ended_at: t.ended_at,
-        content_bytes: contentMap.get(String(t.turn_index)) ?? "",
-    }));
-    if (rows.length === 0)
-        return;
-    const turnContentNodes = [];
-    for (const row of rows) {
-        const ti = Number(row.turn_index);
-        const nodeId = `turn:${sessionId}:${ti}`;
-        if (seen.has(nodeId))
-            continue;
-        seen.add(nodeId);
-        const contentBytes = row.content_bytes ? String(row.content_bytes) : "";
-        const snippet = contentBytes.length > 200 ? contentBytes.slice(0, 197) + "..." : contentBytes;
-        const node = {
-            id: nodeId,
-            sessionId,
-            label: `Turn ${ti}`,
-            summaryTruncated: snippet,
-            tokenEstimate: contentBytes.length,
-            timestamp: row.ended_at ? Number(row.ended_at) : Date.now(),
-            dedupStatus: undefined,
-            raptorLevel: 0,
-            topicSummary: undefined,
-            decisionCount: 0,
-            textSnippet: snippet,
-            nodeType: "turn-content",
-            epochId: undefined,
-        };
-        turnContentNodes.push(node);
-    }
-    if (turnContentNodes.length > 1) {
-        const embeddings = turnContentNodes.map((n) => {
-            try {
-                return { id: n.id, vec: getOrComputeEmbedding(mainDb, n.textSnippet) };
-            }
-            catch {
-                return { id: n.id, vec: null };
-            }
-        });
-        for (let i = 0; i < embeddings.length; i++) {
-            if (!embeddings[i].vec)
-                continue;
-            for (let j = i + 1; j < embeddings.length; j++) {
-                if (!embeddings[j].vec)
-                    continue;
-                const sim = cosineSimilarity(embeddings[i].vec, embeddings[j].vec);
-                if (sim > 0) {
-                    edges.push({
-                        source: embeddings[i].id,
-                        target: embeddings[j].id,
-                        weight: sim,
-                        type: "semantic",
-                    });
-                }
-            }
-        }
-        linkTurnToCheckpointEdges(mainDb, sessionId, turnContentNodes, edges);
-    }
-    nodes.push(...turnContentNodes);
-}
-// ---------------------------------------------------------------------------
-// Source C: memory nodes
-// ---------------------------------------------------------------------------
-export function buildMemoryNodes(db, sessionId, nodes, edges) {
-    const existingIds = new Set(nodes.map((n) => n.id));
-    const rows = db
-        .prepare(`SELECT id, content, source_turn, created_at
-       FROM memories
-       ORDER BY created_at ASC`)
-        .all();
-    if (rows.length === 0)
-        return;
-    const memoryNodes = [];
-    for (const row of rows) {
-        const id = `mem:${String(row.id)}`;
-        if (existingIds.has(id))
-            continue;
-        existingIds.add(id);
-        const content = row.content ? String(row.content) : "";
-        const snippet = content.length > 200 ? content.slice(0, 197) + "..." : content;
-        const node = {
-            id,
-            sessionId,
-            label: `Memory ${String(row.id)}`,
-            summaryTruncated: snippet,
-            tokenEstimate: content.length,
-            timestamp: row.created_at ? Number(row.created_at) : Date.now(),
-            dedupStatus: undefined,
-            raptorLevel: 0,
-            topicSummary: undefined,
-            decisionCount: 0,
-            textSnippet: snippet,
-            nodeType: "memory",
-            epochId: undefined,
-        };
-        memoryNodes.push(node);
-    }
-    if (memoryNodes.length > 1) {
-        const embeddings = memoryNodes.map((n) => {
-            try {
-                return { id: n.id, vec: getOrComputeEmbedding(db, n.textSnippet) };
-            }
-            catch {
-                return { id: n.id, vec: null };
-            }
-        });
-        for (let i = 0; i < embeddings.length; i++) {
-            if (!embeddings[i].vec)
-                continue;
-            for (let j = i + 1; j < embeddings.length; j++) {
-                if (!embeddings[j].vec)
-                    continue;
-                const sim = cosineSimilarity(embeddings[i].vec, embeddings[j].vec);
-                if (sim > 0) {
-                    edges.push({
-                        source: embeddings[i].id,
-                        target: embeddings[j].id,
-                        weight: sim,
-                        type: "semantic",
-                    });
-                }
-            }
-        }
-        crossLinkMemoryToCheckpoints(memoryNodes, nodes, edges);
-    }
-    nodes.push(...memoryNodes);
-}
-// ---------------------------------------------------------------------------
-// RAPTOR annotations
-// ---------------------------------------------------------------------------
-export function addRaptorAnnotations(sessionId, stateDir, nodes, edges) {
-    const raptorNodes = listRaptorNodes(sessionId, stateDir);
-    if (!raptorNodes || raptorNodes.length === 0)
-        return;
-    const nodeIndex = new Map();
-    for (let i = 0; i < nodes.length; i++) {
-        nodeIndex.set(nodes[i].id, i);
-    }
-    for (const rn of raptorNodes) {
-        const idx = nodeIndex.get(rn.id);
-        if (idx !== undefined) {
-            nodes[idx].raptorLevel = rn.level;
-        }
-        if (!rn.parentId)
-            continue;
-        if (nodeIndex.has(rn.id) && nodeIndex.has(rn.parentId)) {
-            edges.push({
-                source: rn.id,
-                target: rn.parentId,
-                weight: Math.max(0.1, 1.0 - rn.level * 0.1),
-                type: "raptor_parent",
-            });
-        }
-    }
-}
-// ---------------------------------------------------------------------------
-// Cross-source edge linking
-// ---------------------------------------------------------------------------
-function linkTurnToCheckpointEdges(db, sessionId, turnNodes, edges) {
-    const checkpoints = db
-        .prepare(`SELECT id, timestamp FROM context_chunks WHERE session_id = ? ORDER BY timestamp ASC`)
-        .all(sessionId);
-    for (const tn of turnNodes) {
-        let bestCp = null;
-        for (const cp of checkpoints) {
-            const diff = Math.abs(tn.timestamp - cp.timestamp);
-            if (bestCp === null || diff < bestCp.diff) {
-                bestCp = { id: cp.id, diff };
-            }
-        }
-        if (bestCp) {
-            edges.push({
-                source: tn.id,
-                target: bestCp.id,
-                weight: 1.0,
-                type: "temporal",
-            });
-        }
-    }
-}
-function crossLinkMemoryToCheckpoints(memoryNodes, allNodes, edges) {
-    for (const mn of memoryNodes) {
-        let bestNeighbor = null;
-        for (const n of allNodes) {
-            if (n.nodeType !== "checkpoint" && n.nodeType !== "turn-content")
-                continue;
-            if (n.id === mn.id)
-                continue;
-            const diff = Math.abs(mn.timestamp - n.timestamp);
-            if (bestNeighbor === null || diff < bestNeighbor.diff) {
-                bestNeighbor = { id: n.id, diff };
-            }
-        }
-        if (bestNeighbor) {
-            edges.push({
-                source: mn.id,
-                target: bestNeighbor.id,
-                weight: 0.85,
-                type: "semantic",
-            });
-        }
-    }
-}
-// ---------------------------------------------------------------------------
-// Edge deduplication
-// ---------------------------------------------------------------------------
-export function deduplicateEdges(edges) {
-    const seen = new Map();
-    for (const e of edges) {
-        const keyA = `${e.source}|${e.target}`;
-        const keyB = `${e.target}|${e.source}`;
-        const existing = seen.get(keyA) ?? seen.get(keyB);
-        if (existing) {
-            if (e.weight > existing.weight) {
-                seen.set(keyA, e);
-            }
-        }
-        else {
-            seen.set(keyA, e);
-        }
-    }
-    return [...seen.values()];
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function safeJsonCount(raw) {
-    if (!raw || typeof raw !== "string")
-        return 0;
-    try {
-        const arr = JSON.parse(raw);
-        return Array.isArray(arr) ? arr.length : 0;
-    }
-    catch {
-        return 0;
-    }
-}
+/**
+ * sources.ts — D3 data-source builders for the memory graph.
+ *
+ * Thin pointer: each source builder (checkpoint, turn, turn-content, memory) plus
+ * RAPTOR annotation, edge dedup, and feature flags lives in ./sources/* to keep
+ * this file (and every impl file) under the 500-line limit. The full public API
+ * is re-exported here so existing callers importing "./memoryGraph/sources.js"
+ * are unchanged.
+ */
+export { areTurnsEnabled, isTurnContentEnabled, isTurnContentFlaggedOn, areMemoriesEnabled, } from "./sources/flags.js";
+export { buildCheckpointNodes } from "./sources/checkpoints.js";
+export { buildTurnNodes, buildTurnContentNodes, } from "./sources/turns.js";
+export { buildMemoryNodes } from "./sources/memories.js";
+export { addRaptorAnnotations } from "./sources/raptor.js";
+export { deduplicateEdges } from "./sources/edges.js";

package/dist/src/memoryGraph-gates.test/_helpers.js

@@ -0,0 +1,120 @@
+/**
+ * Shared helpers for the memoryGraph-gates split files.
+ * Extracted from src/memoryGraph-gates.test.ts: temp dirs, node/edge/ws
+ * builders, withEnv, and populateStore (real SQLite seeding).
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openStore } from "../store/sqlite/utils.js";
+import { upsertCheckpoint } from "../store/sqlite/checkpoints.js";
+import { recordTurn, newConversationId } from "../store/sqlite/turns.js";
+import { appendRawTranscript } from "../store/sqlite/raw-transcript.js";
+export const baseTmp = mkdtempSync(join(tmpdir(), "mc-mgraph-gates-"));
+let counter = 0;
+export function tmpDir() {
+    return join(baseTmp, `run-${counter++}`);
+}
+export function n(id, overrides = {}) {
+    return {
+        id,
+        sessionId: "sess_test",
+        label: id,
+        summaryTruncated: "",
+        tokenEstimate: 0,
+        timestamp: 1000,
+        dedupStatus: undefined,
+        raptorLevel: 0,
+        topicSummary: undefined,
+        decisionCount: 0,
+        textSnippet: "",
+        nodeType: "checkpoint",
+        epochId: undefined,
+        ...overrides,
+    };
+}
+export function e(source, target, overrides = {}) {
+    return { source, target, weight: 1.0, type: "temporal", ...overrides };
+}
+export function ws(nodes, edges = []) {
+    return { nodes, edges };
+}
+/** Save and restore env vars around a test. */
+export function withEnv(env, fn) {
+    const saved = {};
+    for (const k of Object.keys(env)) {
+        saved[k] = process.env[k];
+        if (env[k] === undefined) {
+            delete process.env[k];
+        }
+        else {
+            process.env[k] = env[k];
+        }
+    }
+    try {
+        fn();
+    }
+    finally {
+        for (const k of Object.keys(env)) {
+            if (saved[k] === undefined) {
+                delete process.env[k];
+            }
+            else {
+                process.env[k] = saved[k];
+            }
+        }
+    }
+}
+export function populateStore(dir, sessionId, sources) {
+    const db = openStore(dir);
+    if (sources.checkpoint) {
+        upsertCheckpoint({
+            checkpointId: "chkpt_001",
+            sessionId,
+            regionHash: "abc",
+            contentHash: "def",
+            normalizedText: "checkpoint one",
+            summary: "Checkpoint One",
+            tokenEstimate: 50,
+            timestamp: 1000,
+            embedding: [],
+            keyDecisions: [],
+            nextSteps: [],
+            filesModified: [],
+        }, dir);
+        upsertCheckpoint({
+            checkpointId: "chkpt_002",
+            sessionId,
+            regionHash: "ghi",
+            contentHash: "jkl",
+            normalizedText: "checkpoint two",
+            summary: "Checkpoint Two",
+            tokenEstimate: 60,
+            timestamp: 2000,
+            embedding: [],
+            keyDecisions: [],
+            nextSteps: [],
+            filesModified: [],
+        }, dir);
+    }
+    if (sources.turn) {
+        const convId = newConversationId();
+        recordTurn({ conversationId: convId, sessionId, turnIndex: 0, role: "user", endedAt: 1500, ctxTokens: 100 }, dir);
+        recordTurn({ conversationId: convId, sessionId, turnIndex: 1, role: "assistant", endedAt: 2500, ctxTokens: 200 }, dir);
+    }
+    if (sources.turnContent) {
+        const rows = [
+            { sessionId, seq: 0, role: "user", contentBytes: "hello from the user", contentHash: "hash_0", toolName: null, messageTimestamp: 1500, checkpointEpoch: "", turnIndex: 0 },
+            { sessionId, seq: 1, role: "assistant", contentBytes: "response from assistant", contentHash: "hash_1", toolName: null, messageTimestamp: 2500, checkpointEpoch: "", turnIndex: 1 },
+        ];
+        for (const row of rows) {
+            appendRawTranscript(db, row);
+        }
+    }
+    if (sources.memory) {
+        db.prepare(`INSERT OR IGNORE INTO memories (content, kind, created_at, source_turn)
+       VALUES (?, ?, ?, ?)`).run("remembered fact", "fact", 2000, 0);
+        db.prepare(`INSERT OR IGNORE INTO memories (content, kind, created_at, source_turn)
+       VALUES (?, ?, ?, ?)`).run("user preference", "preference", 3000, 1);
+    }
+}