pi-mega-compact 0.11.7 → 0.11.8

package/dist/src/memoryGraph/gates.js

@@ -0,0 +1,379 @@
+import { Logger } from "../log.js";
+const log = new Logger();
+// ---------------------------------------------------------------------------
+// Gate 1 — Source availability guard (informational, no drop)
+// ---------------------------------------------------------------------------
+export function gateSourceAvailable(ws) {
+    if (ws.nodes.length === 0) {
+        log.warn("graph_source_unavailable", {
+            detail: "No nodes from any source — graph will be empty",
+        });
+        return { ok: false, dropped: 0, reason: "No nodes from any source" };
+    }
+    return { ok: true, dropped: 0 };
+}
+// ---------------------------------------------------------------------------
+// Gate 2 — Identity merge: duplicate nodes with same id → richest wins
+// ---------------------------------------------------------------------------
+export function gateIdentityMerge(ws) {
+    const { nodes } = ws;
+    const seen = new Map();
+    let dropped = 0;
+    for (const n of nodes) {
+        const existing = seen.get(n.id);
+        if (!existing) {
+            seen.set(n.id, n);
+            continue;
+        }
+        const existingScore = fieldScore(existing);
+        const currentScore = fieldScore(n);
+        if (currentScore > existingScore) {
+            seen.set(n.id, n);
+        }
+        dropped++;
+        log.warn("graph_identity_leak", {
+            nodeId: n.id,
+            dropped: currentScore > existingScore ? existing.id : n.id,
+        });
+    }
+    ws.nodes = [...seen.values()];
+    return { ok: dropped === 0, dropped, reason: dropped > 0 ? `${dropped} duplicate node ids merged` : undefined };
+}
+function fieldScore(n) {
+    // Base score by nodeType — richer sources win ties (spec: "richest available
+    // source wins"). checkpoint > turn-content > memory > turn-structural.
+    let s = 0;
+    switch (n.nodeType) {
+        case "checkpoint":
+            s += 4;
+            break;
+        case "turn-content":
+            s += 3;
+            break;
+        case "memory":
+            s += 2;
+            break;
+        case "turn":
+            s += 1;
+            break;
+    }
+    if (n.summaryTruncated)
+        s++;
+    if (n.textSnippet)
+        s++;
+    if (n.topicSummary)
+        s++;
+    if (n.tokenEstimate > 0)
+        s++;
+    if (n.decisionCount > 0)
+        s++;
+    if (n.dedupStatus)
+        s++;
+    return s;
+}
+// ---------------------------------------------------------------------------
+// Gate 3 — Promotion guard: suppress turns with epoch_id if checkpoint exists
+// ---------------------------------------------------------------------------
+export function gatePromotionGuard(ws) {
+    const checkpointIds = new Set();
+    for (const n of ws.nodes) {
+        if (n.nodeType === "checkpoint")
+            checkpointIds.add(n.id);
+    }
+    const keep = [];
+    const droppedIds = new Set();
+    let dropped = 0;
+    for (const n of ws.nodes) {
+        if (n.nodeType !== "turn" && n.nodeType !== "turn-content") {
+            keep.push(n);
+            continue;
+        }
+        const epochId = n.epochId;
+        if (epochId && checkpointIds.has(epochId)) {
+            droppedIds.add(n.id);
+            dropped++;
+            log.warn("graph_orphaned_epoch", {
+                nodeId: n.id,
+                epochId,
+                detail: "Turn suppressed because its checkpoint exists in the graph",
+            });
+        }
+        else {
+            if (epochId && !checkpointIds.has(epochId)) {
+                log.warn("graph_orphaned_epoch", {
+                    nodeId: n.id,
+                    epochId,
+                    detail: "Orphaned epoch — no matching checkpoint; keeping turn",
+                });
+            }
+            keep.push(n);
+        }
+    }
+    ws.nodes = keep;
+    ws.edges = ws.edges.filter((e) => !droppedIds.has(e.source) && !droppedIds.has(e.target));
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} turn nodes suppressed (checkpoint exists)` : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Gate 4 — Node type / field completeness
+// ---------------------------------------------------------------------------
+export function gateNodeCompleteness(ws) {
+    const validNodes = [];
+    let dropped = 0;
+    for (const n of ws.nodes) {
+        if (!n.nodeType || !n.sessionId) {
+            dropped++;
+            log.warn("graph_node_double", {
+                nodeId: n.id,
+                detail: "Missing nodeType or sessionId",
+            });
+            continue;
+        }
+        validNodes.push(n);
+    }
+    ws.nodes = validNodes;
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} nodes dropped for missing fields` : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Gate 5 — Dangling edge guard
+// ---------------------------------------------------------------------------
+export function gateDanglingEdges(ws) {
+    const validIds = new Set(ws.nodes.map((n) => n.id));
+    const validEdges = [];
+    let dropped = 0;
+    for (const e of ws.edges) {
+        if (validIds.has(e.source) && validIds.has(e.target)) {
+            validEdges.push(e);
+        }
+        else {
+            dropped++;
+            log.warn("graph_dangling_edge", {
+                source: e.source,
+                target: e.target,
+                detail: "Edge references non-existent node(s)",
+            });
+        }
+    }
+    ws.edges = validEdges;
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} dangling edges removed` : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Gate 6 — Edge type / threshold enforcement
+// ---------------------------------------------------------------------------
+function envThreshold(name, def) {
+    const v = process.env[name];
+    if (v === undefined)
+        return def;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : def;
+}
+export function gateEdgeThresholds(ws, options) {
+    const withinThreshold = options?.withinTypeThreshold ?? envThreshold("MEGACOMPACT_MEMORY_GRAPH_WITHIN_TYPE_THRESHOLD", 0.7);
+    const crossThreshold = options?.crossTypeThreshold ?? envThreshold("MEGACOMPACT_MEMORY_GRAPH_CROSS_TYPE_THRESHOLD", 0.85);
+    const nodeTypeMap = new Map(ws.nodes.map((n) => [n.id, n.nodeType]));
+    const validEdges = [];
+    let dropped = 0;
+    for (const e of ws.edges) {
+        if (e.type !== "semantic") {
+            validEdges.push(e);
+            continue;
+        }
+        const srcType = nodeTypeMap.get(e.source);
+        const tgtType = nodeTypeMap.get(e.target);
+        if (srcType === "turn" || tgtType === "turn") {
+            dropped++;
+            log.warn("graph_structural_semantic_edge", {
+                nodeId: srcType === "turn" ? e.source : e.target,
+                source: e.source,
+                target: e.target,
+                score: e.weight,
+                detail: "Semantic edge on structural (turn) node",
+            });
+            continue;
+        }
+        const sameType = srcType != null && srcType === tgtType;
+        const threshold = sameType ? withinThreshold : crossThreshold;
+        if (e.weight < threshold) {
+            dropped++;
+            log.warn("graph_edge_below_threshold", {
+                source: e.source,
+                target: e.target,
+                score: e.weight,
+                typePair: `${String(srcType)}↔${String(tgtType)}`,
+                threshold,
+                detail: `Edge below ${sameType ? "within-type" : "cross-type"} threshold`,
+            });
+            continue;
+        }
+        validEdges.push(e);
+    }
+    ws.edges = validEdges;
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} edges below threshold removed` : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Gate 7 — Dedup mirror cross-check
+// ---------------------------------------------------------------------------
+export function gateDedupRedundant(ws, options) {
+    const redundant = options?.redundantIds ?? new Set();
+    if (redundant.size === 0)
+        return { ok: true, dropped: 0 };
+    const keep = [];
+    const droppedIds = new Set();
+    let dropped = 0;
+    for (const n of ws.nodes) {
+        if (n.nodeType === "turn-content" && redundant.has(n.id)) {
+            dropped++;
+            droppedIds.add(n.id);
+            log.warn("graph_dedup_redundant", {
+                nodeId: n.id,
+                contentHash: n.id,
+                detail: "Turn content already collapsed into checkpoint",
+            });
+        }
+        else {
+            keep.push(n);
+        }
+    }
+    if (droppedIds.size > 0) {
+        ws.edges = ws.edges.filter((e) => !droppedIds.has(e.source) && !droppedIds.has(e.target));
+    }
+    ws.nodes = keep;
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} turn-content nodes redundant with checkpoints` : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Gate 8 — Structural vs semantic edge consistency
+// ---------------------------------------------------------------------------
+export function gateStructuralSemanticEdges(ws) {
+    const structuralTypes = new Set(["checkpoint", "turn"]);
+    const keep = [];
+    let dropped = 0;
+    for (const e of ws.edges) {
+        if (e.type !== "semantic") {
+            keep.push(e);
+            continue;
+        }
+        const srcNode = ws.nodes.find((n) => n.id === e.source);
+        const tgtNode = ws.nodes.find((n) => n.id === e.target);
+        if (srcNode && structuralTypes.has(srcNode.nodeType)) {
+            dropped++;
+            log.warn("graph_structural_semantic_edge", {
+                nodeId: srcNode.id,
+                nodeType: srcNode.nodeType,
+                detail: "Structural node has semantic edge",
+            });
+        }
+        else if (tgtNode && structuralTypes.has(tgtNode.nodeType)) {
+            dropped++;
+            log.warn("graph_structural_semantic_edge", {
+                nodeId: tgtNode.id,
+                nodeType: tgtNode.nodeType,
+                detail: "Structural node has semantic edge",
+            });
+        }
+        else {
+            keep.push(e);
+        }
+    }
+    ws.edges = keep;
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} semantic edges on structural nodes removed` : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Gate 9 — Same-type edge deduplication
+// ---------------------------------------------------------------------------
+export function gateDedupEdges(ws) {
+    const best = new Map();
+    let dropped = 0;
+    for (const e of ws.edges) {
+        const key = `${e.source}|${e.target}|${e.type}`;
+        const existing = best.get(key);
+        if (!existing || e.weight > existing.weight) {
+            if (existing)
+                dropped++;
+            best.set(key, e);
+        }
+        else {
+            dropped++;
+        }
+    }
+    if (dropped > 0) {
+        log.warn("graph_dedup_redundant", {
+            count: dropped,
+            detail: "Redundant edges deduplicated",
+        });
+    }
+    ws.edges = [...best.values()];
+    return {
+        ok: dropped === 0,
+        dropped,
+        reason: dropped > 0 ? `${dropped} redundant edges deduplicated` : undefined,
+    };
+}
+/** Run all 9 gates in order over a working set. Returns validated set + stats.
+ *  Gate 7 uses optional redundantIds from the caller. */
+export function runValidationPipeline(ws, options) {
+    const gatesRun = [];
+    const gatesPassed = [];
+    const warnings = [];
+    let totalDroppedNodes = 0;
+    let totalDroppedEdges = 0;
+    const gates = [
+        { name: "source_available", fn: () => gateSourceAvailable(ws) },
+        { name: "identity_merge", fn: () => gateIdentityMerge(ws) },
+        { name: "promotion_guard", fn: () => gatePromotionGuard(ws) },
+        { name: "node_completeness", fn: () => gateNodeCompleteness(ws) },
+        { name: "dangling_edges", fn: () => gateDanglingEdges(ws) },
+        { name: "edge_thresholds", fn: () => gateEdgeThresholds(ws, { withinTypeThreshold: options?.withinTypeThreshold, crossTypeThreshold: options?.crossTypeThreshold }) },
+        { name: "dedup_redundant", fn: () => gateDedupRedundant(ws, { redundantIds: options?.redundantIds }) },
+        { name: "structural_semantic", fn: () => gateStructuralSemanticEdges(ws) },
+        { name: "dedup_edges", fn: () => gateDedupEdges(ws) },
+    ];
+    for (const gate of gates) {
+        gatesRun.push(gate.name);
+        const r = gate.fn();
+        if (r.ok) {
+            gatesPassed.push(gate.name);
+        }
+        if (r.reason) {
+            warnings.push({ gate: gate.name, code: "gate_warning", count: r.dropped || 1 });
+        }
+        // Track by category
+        if (["identity_merge", "promotion_guard", "node_completeness"].includes(gate.name)) {
+            totalDroppedNodes += r.dropped;
+        }
+        if (["dangling_edges", "edge_thresholds", "dedup_redundant", "structural_semantic", "dedup_edges"].includes(gate.name)) {
+            totalDroppedEdges += r.dropped;
+        }
+    }
+    return {
+        ws,
+        stats: {
+            gatesRun,
+            gatesPassed,
+            dropped: { nodes: totalDroppedNodes, edges: totalDroppedEdges },
+            warnings,
+        },
+    };
+}