graph-engine-memory 0.6.1__py3-none-any.whl

graph_engine/merge.py

@@ -0,0 +1,220 @@
+"""Manual near-duplicate consolidation (`ig merge`).
+
+Consolidates two closely related nodes into one: all edges of the node
+being removed are redirected to the survivor (deduplicated, no
+self-loops), the text is appended to preserve information, and the
+removed node is marked as a tombstone (never hard-deleted — the
+graph remains an append-only history). INDEX.md is rebuilt and everything
+is written in ONE commit.
+
+Audit #3/#9/#41 fixes:
+- The deletee is tombstoned instead of being set to probation, and its file
+  is no longer deleted before the edge rewrite (half-written state
+  on a crash mid-merge).
+- Edges with intent semantics (`supersedes`, `contradicts`, `continues`,
+  `replaces`, …) are NOT blindly redirected: a deletee→X `supersedes`
+  edge says "the deletee supersedes X" — after the merge the survivor is the
+  continuation of the deletee's content, so the edge is redirected to
+  survivor→X ONLY if the direction stays semantically valid.
+  For supersedes/replaces/contradicts (deletee as source) the edge
+  is invalidated instead (valid_to set) rather than making a possibly
+  wrong statement about the survivor; extends/similar/linked edges
+  are direction-neutral enough for a redirect.
+- Merge provenance: the merge commit and the survivor node document
+  the deletee ID and timestamp; the deletee's status remains traceable
+  as a tombstone.
+
+This is the manual complement to automatic ingest dedup (cos >= 0.92):
+near-duplicates in the ~0.78–0.92 range stay below the auto threshold and
+need this consolidation. Destructive — use with care, ideally after
+a dry run on a brain copy.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .brain import Brain
+
+# Intent edges with directed semantics: a redirect deletee→X to
+# survivor→X would invert the statement (Audit #9). They are invalidated
+# instead of redirected. Direction-neutral kinds (similar/extends/same_as/…)
+# are redirected normally.
+DIRECTIONAL_KINDS = frozenset({
+    "supersedes", "ersetzt", "contradicts", "replaces", "obsoletes",
+    "korrigiert", "widerspricht",
+})
+
+
+@dataclass
+class MergeResult:
+    survivor: str
+    deletee: str
+    edges_redirected: int
+    edges_removed: int  # self-loops + duplicate pairs
+    edges_invalidated: int = 0  # directed intent edges (not redirectable)
+    redirected_ids: list[str] = field(default_factory=list)
+    invalidated_ids: list[str] = field(default_factory=list)
+
+
+def _redirect_edges(edges: list, survivor: str, deletee: str,
+                    now_iso: str) -> tuple[list, int, int, int, list[str], list[str]]:
+    """Redirect the deletee's edges to the survivor; removes self-loops + duplicates.
+
+    Directed intent edges (DIRECTIONAL_KINDS) with the deletee as source are
+    invalidated (valid_to set) instead of redirected — the survivor has not
+    necessarily inherited the statement "supersedes X" (Audit #9).
+    """
+    new_edges: list = []
+    seen: set[tuple] = set()
+    removed = 0
+    redirected = 0
+    invalidated = 0
+    redirected_ids: list[str] = []
+    invalidated_ids: list[str] = []
+    for e in edges:
+        if e.kind == "fact":
+            # Entity identities cannot be text-merged (guard below). Preserve
+            # all fact predicates, directions, validity windows and historical
+            # evidence references when unrelated semantic notes are merged.
+            new_edges.append(e)
+            continue
+        s, t = e.source, e.target
+        if s == deletee or t == deletee:
+            if e.kind in DIRECTIONAL_KINDS and s == deletee:
+                # The statement no longer holds about the survivor — historicize the edge.
+                if not e.is_invalidated:
+                    e.invalidate(now_iso)
+                    invalidated += 1
+                    invalidated_ids.append(e.id)
+                else:
+                    removed += 1
+                new_edges.append(e)
+                continue
+            if s == deletee:
+                s = survivor
+                redirected += 1
+            if t == deletee:
+                t = survivor
+                redirected += 1
+            redirected_ids.append(e.id)
+        if s == t:  # self-loop after merge
+            removed += 1
+            continue
+        key = tuple(sorted((s, t)))
+        if key in seen:  # duplicate pair (survivor already had that target)
+            removed += 1
+            continue
+        seen.add(key)
+        e.source, e.target = s, t
+        new_edges.append(e)
+    return new_edges, removed, redirected, invalidated, redirected_ids, invalidated_ids
+
+
+def _drop_vector(brain: Brain, deletee: str) -> None:
+    vec_file: Path = brain.path / "vectors.jsonl"
+    if not vec_file.exists():
+        return
+    lines = [l for l in vec_file.read_text(encoding="utf-8").splitlines() if l.strip()]
+    keep = [l for l in lines if json.loads(l).get("id") != deletee]
+    brain.write_vectors({json.loads(l)["id"]: json.loads(l)["vec"] for l in keep})
+
+
+def merge_nodes(
+    brain: Brain,
+    survivor_id: str,
+    deletee_id: str,
+    commit: bool = True,
+    embedder=None,
+) -> MergeResult:
+    """Consolidates the deletee into the survivor. Destructive; one commit.
+
+    embedder (optional): when given, the survivor vector is recomputed after
+    the text append — without it the old vector would remain and search
+    would dedupe/rank against the PRE-merge text (audit finding
+    "survivor vector stale"). The CLI always passes the engine embedder.
+    """
+    brain.authorize("admin")
+    if survivor_id == deletee_id:
+        raise ValueError("Survivor and deletee are identical.")
+    # Git mode: repo existence + fresh pull BEFORE the mutation — otherwise
+    # the merge can work on a stale state and the history diverges
+    # (audit finding "merge_nodes makes no pull()").
+    brain.ensure_ready()
+    brain.pull()
+    nodes = {n.id: n for n in brain.read_nodes()}
+    if survivor_id not in nodes:
+        raise ValueError(f"Node not found: {survivor_id}")
+    if deletee_id not in nodes:
+        raise ValueError(f"Node not found: {deletee_id}")
+    survivor, deletee = nodes[survivor_id], nodes[deletee_id]
+    if survivor.ntype == "entity" or deletee.ntype == "entity":
+        raise ValueError("Entity identities cannot be merged as near-duplicate text; use explicit aliases.")
+    now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # Redirect + deduplicate edges — BEFORE deleting the deletee file
+    # (Audit #3: previously the node file was unlinked first; a crash
+    # between unlink and edge rewrite left a half-merged state with
+    # dangling edges pointing to a no-longer-existing node).
+    edges = brain.read_edges(include_rejected=False)
+    new_edges, removed, redirected, invalidated, redirected_ids, invalidated_ids = \
+        _redirect_edges(edges, survivor_id, deletee_id, now_iso)
+    # Keep unrelated dismissed suggestions available for undo. Decisions about
+    # the deleted node cannot be restored after consolidation.
+    rejected = [e for e in brain.read_edges(include_rejected=True)
+                if e.rejected and deletee_id not in (e.source, e.target)]
+    brain.write_edges(new_edges + rejected)
+
+    # Merge text (information preservation) — with merge provenance (Audit #41)
+    survivor.text = (
+        survivor.text.rstrip()
+        + f"\n\n[consolidated from {deletee_id} on {now_iso}: {deletee.text.strip()}]"
+    )
+    brain.write_node(survivor)
+
+    # Tombstone the deletee (never hard-delete — Audit #3: previously unlink
+    # + status loss, dangling edges could point to a "living" probation node)
+    brain.tombstone_node(deletee_id)
+
+    # Remove vector
+    _drop_vector(brain, deletee_id)
+
+    # Refresh the survivor vector: the text has grown, the old vector
+    # represents the pre-merge text (audit finding: "survivor vector stale").
+    if embedder is not None:
+        normalized = " ".join(survivor.text.lower().split())
+        vecs = brain.read_vectors()
+        vecs[survivor_id] = list(embedder.embed(normalized))
+        brain.write_vectors(vecs)
+
+    # Rebuild INDEX
+    brain.rebuild_index()
+
+    if commit:
+        try:
+            brain.commit_and_push(
+                f"merge: {deletee_id} consolidated into {survivor_id} "
+                f"({redirected} redirected, {invalidated} invalidated, {removed} removed)")
+        except Exception:
+            # Audit #31: the merge mutations are on disk but uncommitted —
+            # heal the derived index and raise an actionable error.
+            try:
+                brain.rebuild_index()
+            except Exception:
+                pass
+            raise RuntimeError(
+                "merge applied but commit/push failed — changes are on disk "
+                "UNCOMMITTED. Recovery: `git -C <brain> add -A && git commit` "
+                "or retry the merge (it is idempotent against the merged state).") from None
+
+    return MergeResult(
+        survivor=survivor_id,
+        deletee=deletee_id,
+        edges_redirected=redirected,
+        edges_removed=removed,
+        edges_invalidated=invalidated,
+        redirected_ids=redirected_ids,
+        invalidated_ids=invalidated_ids,
+    )

graph_engine/model.py

@@ -0,0 +1,56 @@
+"""Core data model: nodes (ideas) and typed edges.
+
+Edges carry a type: "similar", "contradicts", "extends".
+Pending edges are suggestions from the growth loop and wait for a
+human-in-the-loop decision (accept/discard).
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field, asdict
+
+EDGE_TYPES = ("similar", "contradicts", "extends")
+
+
+def _now() -> float:
+    return time.time()
+
+
+@dataclass
+class Node:
+    text: str
+    id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
+    created: float = field(default_factory=_now)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Node":
+        return cls(text=d["text"], id=d["id"], created=d.get("created", 0.0))
+
+
+@dataclass
+class Edge:
+    source: str          # node ID (new idea)
+    target: str          # node ID (existing idea)
+    kind: str            # one of EDGE_TYPES
+    pending: bool = True  # suggestion until accepted/discarded
+    id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
+    created: float = field(default_factory=_now)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Edge":
+        return cls(
+            source=d["source"],
+            target=d["target"],
+            kind=d["kind"],
+            pending=d.get("pending", False),
+            id=d["id"],
+            created=d.get("created", 0.0),
+        )

graph_engine/namespaces.py

@@ -0,0 +1,74 @@
+"""Separate brain directories with an atomic, persisted CLI selection."""
+from __future__ import annotations
+
+from contextvars import ContextVar
+import os
+from pathlib import Path
+import re
+
+from .brain import Brain, _atomic_write
+from . import access
+
+
+request_graph: ContextVar[str | None] = ContextVar("ig_graph", default=None)
+
+
+def home() -> Path | None:
+    value = os.environ.get("IG_GRAPH_HOME", "")
+    return Path(value).expanduser().resolve() if value else None
+
+
+def validate_name(name: str) -> str:
+    if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", name):
+        raise ValueError("Graph name must contain 1-64 letters, digits, underscores or hyphens")
+    return name
+
+
+def selected() -> str:
+    if request_graph.get() is not None:
+        return request_graph.get()
+    explicit = os.environ.get("IG_GRAPH", "")
+    root = home()
+    if explicit:
+        if root is None:
+            raise ValueError("IG_GRAPH requires IG_GRAPH_HOME")
+        return validate_name(explicit)
+    if root and (root / "selected").exists():
+        return validate_name((root / "selected").read_text(encoding="utf-8").strip())
+    return "default"
+
+
+def graph_path(name: str) -> Path:
+    root = home()
+    if root is None:
+        raise ValueError("Set IG_GRAPH_HOME to use named graphs")
+    path = root / "graphs" / validate_name(name)
+    if path.resolve() != path or (root / "graphs").is_symlink():
+        raise ValueError("Graph directories must not be symlinks")
+    return path
+
+
+def list_graphs() -> list[str]:
+    root = home()
+    if root is None:
+        raise ValueError("Set IG_GRAPH_HOME to use named graphs")
+    names = sorted(p.name for p in (root / "graphs").glob("*") if p.is_dir() and not p.is_symlink())
+    return [n for n in names if not access.policy_path() or access.permitted(n, access.principal(), "read")]
+
+
+def switch_graph(name: str) -> None:
+    path = graph_path(name)
+    if access.policy_path():
+        access.require(name, access.principal(), "read")
+    if not path.is_dir():
+        raise ValueError("Graph does not exist; use ig graph create first")
+    _atomic_write(home() / "selected", name + "\n")
+
+
+def create_graph(name: str) -> None:
+    path = graph_path(name)
+    if access.policy_path():
+        # Provision roles in the trusted policy before creating a namespace.
+        access.require(name, access.principal(), "admin")
+    path.mkdir(parents=True, exist_ok=False)
+    Brain(str(path), mode=os.environ.get("IG_BRAIN_MODE", "git")).init()

graph_engine/recall.py

@@ -0,0 +1,161 @@
+"""Recall tracking: what the memory is actually asked for.
+
+A memory system that does not know which of its entries are ever retrieved
+cannot decide what to keep, promote, or let decay — OpenClaw's dreaming
+promotes a candidate only when it passed `minRecallCount` / `minUniqueQueries`,
+and GraphEngine had no such signal at all (every node was equally "fresh"
+forever, all 2120 stuck in `probation`).
+
+Two halves, deliberately split so the READ path stays cheap and git-clean:
+
+  * `record(brain, query, node_ids)` — append ONE line to the local ledger
+    `<brain>/recalls.jsonl` (gitignored). No read-modify-write of node files,
+    no commit per search: a search must not dirty the brain.
+  * `aggregate(brain)` — folds the ledger into the derived counters on the
+    nodes (`recall_count`, `recall_queries`, `last_recalled`) and writes them
+    back in ONE pass. Called by the dream pass / `ig recall --aggregate`.
+
+The counters are DERIVED data, like INDEX.md: the ledger is the source of
+truth, the node fields are the queryable projection.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import pathlib
+from collections import Counter
+
+from .brain import Brain, Node
+
+LEDGER_NAME = "recalls.jsonl"
+# Distinct query fingerprints kept per node: enough for "was this asked for in
+# many different ways?" (OpenClaw's query-diversity gate) without unbounded
+# frontmatter growth.
+MAX_QUERY_FINGERPRINTS = 25
+
+
+def ledger_path(brain: Brain) -> pathlib.Path:
+    return pathlib.Path(brain.path) / LEDGER_NAME
+
+
+def fingerprint(query: str) -> str:
+    norm = " ".join(query.lower().split())
+    return hashlib.sha1(norm.encode("utf-8")).hexdigest()[:10]
+
+
+def record(brain: Brain, query: str, node_ids: list[str], ts: str | None = None) -> int:
+    """Append one recall event. Returns the number of node ids recorded."""
+    brain.authorize("read")
+    ids = [nid for nid in node_ids if nid and brain.can_access("write", nid)]
+    if not ids:
+        return 0
+    entry = {"ts": ts or _now(), "q": fingerprint(query), "ids": ids}
+    with open(ledger_path(brain), "a", encoding="utf-8") as fh:
+        fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
+    return len(ids)
+
+
+def read_ledger(brain: Brain) -> list[dict]:
+    brain.authorize("read")
+    path = ledger_path(brain)
+    if not path.exists():
+        return []
+    out = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if line.strip():
+            try:
+                entry = json.loads(line)
+                entry["ids"] = [nid for nid in entry.get("ids", []) if brain.can_access("read", nid)]
+                if entry["ids"]:
+                    out.append(entry)
+            except json.JSONDecodeError:
+                continue
+    return out
+
+
+def aggregate(brain: Brain, *, dry_run: bool = False, commit: bool = True) -> dict:
+    """Fold the recall ledger into the node counters (one pass, one commit).
+
+    Per node: `recall_count` += occurrences, `recall_queries` = distinct query
+    fingerprints (capped), `last_recalled` = newest ledger timestamp. The ledger
+    is then MOVED to `<ledger>.processed` (audit trail) and truncated, so a
+    second run is idempotent instead of double-counting.
+    """
+    from .tiers import rebalance
+    brain.authorize("admin")
+    entries = read_ledger(brain)
+    if not entries:
+        rebalance(brain, dry_run=dry_run, commit=commit)
+        return {"nodes": 0, "recalls": 0, "ledger_entries": 0, "dry_run": dry_run}
+
+    counts: Counter = Counter()
+    queries: dict[str, set[str]] = {}
+    last: dict[str, str] = {}
+    for entry in entries:
+        ts, fp = entry.get("ts"), entry.get("q")
+        for nid in entry.get("ids", []):
+            counts[nid] += 1
+            if fp:
+                queries.setdefault(nid, set()).add(fp)
+            if ts and ts > last.get(nid, ""):
+                last[nid] = ts
+
+    nodes = {n.id: n for n in brain.read_nodes()}
+    touched = 0
+    for nid, increment in counts.items():
+        node = nodes.get(nid)
+        if node is None:
+            continue  # recalled node was deleted/merged away since the ledger write
+        node.recall_count = getattr(node, "recall_count", 0) + increment
+        seen = list(getattr(node, "recall_queries", []))
+        for fp in sorted(queries.get(nid, set())):
+            if fp not in seen:
+                seen.append(fp)
+        node.recall_queries = seen[:MAX_QUERY_FINGERPRINTS]
+        node.last_recalled = max(filter(None, [getattr(node, "last_recalled", None),
+                                               last.get(nid)]), default=None)
+        if not dry_run:
+            brain.write_node(node)
+        touched += 1
+
+    if not dry_run:
+        # Move the ledger, THEN optionally commit. The move is bookkeeping that
+        # must happen whenever the counters were written — tying it to `commit`
+        # was a real bug (found live 2026-09-18): `dream.refresh()` batches the
+        # commit with `commit=False`, so the source entries stayed in place and
+        # EVERY pass folded them again (recall_count 1 -> 2 on a single real
+        # ledger line), inflating exactly the signal the promotion gates read.
+        path = ledger_path(brain)
+        processed = pathlib.Path(brain.path) / (LEDGER_NAME + ".processed")
+        with open(processed, "a", encoding="utf-8") as fh:
+            fh.write(path.read_text(encoding="utf-8"))
+        path.write_text("", encoding="utf-8")
+        rebalance(brain, commit=False)
+        if commit:
+            brain.commit_and_push(
+                f"recall: aggregate {sum(counts.values())} recalls into {touched} nodes")
+    return {"nodes": touched, "recalls": sum(counts.values()),
+            "ledger_entries": len(entries), "dry_run": dry_run}
+
+
+def top(brain: Brain, n: int = 10) -> list[tuple[str, int]]:
+    """Most-recalled nodes (derived counters, no aggregation)."""
+    counts = Counter()
+    for node in brain.read_nodes():
+        if getattr(node, "recall_count", 0):
+            counts[node.id] = node.recall_count
+    return counts.most_common(n)
+
+
+def _now() -> str:
+    import datetime
+    return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
+
+
+def tracking_enabled(env: dict[str, str] | None = None) -> bool:
+    """Recall tracking is on unless explicitly disabled (`IG_NO_RECALL_TRACKING=1`)."""
+    raw = (env or {}).get("IG_NO_RECALL_TRACKING",
+                          os.environ.get("IG_NO_RECALL_TRACKING", ""))
+    return str(raw).lower() not in ("1", "true", "yes")