graph-engine-memory 0.6.1__py3-none-any.whl

graph_engine/feedback.py

@@ -0,0 +1,155 @@
+"""Deterministic retrieval learning from explicit user relevance judgments.
+
+The tracked feedback.json file keeps the latest judgment per normalized
+(query, node), avoiding repeated votes and allowing corrections. Channel rank
+observations train bounded RRF weights; successful queries expand similar
+queries with a small set of terms from currently visible, live target nodes.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from .brain import Brain, Node, _atomic_write, _now_iso
+
+if TYPE_CHECKING:
+    from .brain_engine import BrainEngine
+
+FEEDBACK_FILE = "feedback.json"
+MAX_EXPANSION_TERMS = 5
+STOP_WORDS = frozenset("a an and are as at be by for from in is it of on or that the this to was with".split())
+
+
+def normalize(query: str) -> str:
+    return " ".join(query.lower().split())
+
+
+def _terms(text: str) -> list[str]:
+    return [word for word in re.findall(r"\w+", text.lower())
+            if len(word) > 2 and word not in STOP_WORDS]
+
+
+def _read_feedback(brain: Brain) -> list[dict]:
+    """Read valid rows; trusted RMW preserves judgments outside caller scope."""
+    path = brain.path / FEEDBACK_FILE
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(data, dict) or data.get("version") != 1 or not isinstance(data.get("judgments"), list):
+            raise ValueError("unsupported feedback format")
+    except (json.JSONDecodeError, ValueError) as exc:
+        raise ValueError(f"Cannot read {FEEDBACK_FILE}: {exc}") from exc
+    rows = []
+    for row in data["judgments"]:
+        if (not isinstance(row, dict) or not isinstance(row.get("query"), str)
+                or not normalize(row["query"]) or not isinstance(row.get("node_id"), str)
+                or row.get("label") not in ("relevant", "irrelevant")):
+            continue
+        rows.append(row)
+    return rows
+
+
+def read_feedback(brain: Brain) -> list[dict]:
+    """Read judgments only for currently visible, live nodes."""
+    getattr(brain, "authorize", lambda **_: None)(action="read")
+    visible = {node.id for node in brain.read_nodes() if node.status != "tombstone"}
+    return [row for row in _read_feedback(brain) if row["node_id"] in visible]
+
+
+@dataclass
+class LearningProfile:
+    query: str
+    dense_weight: float = 1.0
+    bm25_weight: float = 1.0
+    expansion_terms: list[str] = field(default_factory=list)
+    adjustments: dict[str, float] = field(default_factory=dict)
+
+    @property
+    def expanded_query(self) -> str:
+        return " ".join([self.query, *self.expansion_terms])
+
+
+def learning_profile(brain: Brain, query: str, nodes: list[Node]) -> LearningProfile:
+    profile = LearningProfile(query)
+    visible = {node.id: node for node in nodes if node.status != "tombstone"}
+    # Latest row wins even in hand-edited files; a judgment cannot multiply votes.
+    getattr(brain, "authorize", lambda **_: None)(action="read")
+    judgments = {(normalize(row["query"]), row["node_id"]): row
+                 for row in _read_feedback(brain) if row["node_id"] in visible}
+    if not judgments:
+        return profile
+    dense_utility = bm25_utility = 0.0
+    expansions: Counter[str] = Counter()
+    query_terms = set(_terms(query))
+    for (prior_query, nid), row in judgments.items():
+        sign = 1 if row["label"] == "relevant" else -1
+        def utility(channel: str) -> float:
+            rank = row.get(channel + "_rank")
+            return 1.0 / rank if isinstance(rank, int) and not isinstance(rank, bool) and rank > 0 else 0.0
+        dense_utility += sign * utility("dense")
+        bm25_utility += sign * utility("bm25")
+        if prior_query == normalize(query):
+            profile.adjustments[nid] = sign / 60.0
+        prior_terms = set(_terms(prior_query))
+        similarity = len(query_terms & prior_terms) / max(1, len(query_terms | prior_terms))
+        if sign > 0 and similarity >= 0.5:
+            expansions.update(set(_terms(visible[nid].text)) - query_terms)
+    # Bounded update with a neutral prior: many duplicate votes cannot overpower
+    # the other channel, and no-feedback behavior is exactly the existing RRF.
+    delta = max(-0.5, min(0.5, (dense_utility - bm25_utility) / (2 + len(judgments))))
+    profile.dense_weight = 1.0 + delta
+    profile.bm25_weight = 1.0 - delta
+    profile.expansion_terms = sorted(expansions, key=lambda t: (-expansions[t], t))[:MAX_EXPANSION_TERMS]
+    return profile
+
+
+def record_feedback(engine: BrainEngine, query: str, node_id: str, label: str, *,
+                    commit: bool = True) -> dict:
+    if not isinstance(query, str) or not query.strip():
+        raise ValueError("query must be non-empty")
+    if label not in ("relevant", "irrelevant"):
+        raise ValueError("label must be relevant or irrelevant")
+    from .brain_engine import BRAIN_LOCK
+    from .retrieval import _retrieve_trace
+    brain = engine.brain
+    with BRAIN_LOCK:
+        getattr(brain, "authorize", lambda **_: None)(action="write", node_id=node_id)
+        brain.ensure_ready()
+        brain.pull()
+        node = brain.read_node(node_id)
+        if node is None or node.status == "tombstone":
+            raise ValueError(f"No live node with id {node_id!r}.")
+        # Measure the baseline channels, not our own previous learned boost.
+        trace = _retrieve_trace(engine, query, 30, persist=False, learn=False)
+        rows = _read_feedback(brain)
+        key = (normalize(query), node_id)
+        rows = [row for row in rows if (normalize(row["query"]), row["node_id"]) != key]
+        row = {"query": key[0], "node_id": node_id, "label": label, "at": _now_iso(),
+               "dense_rank": trace.dense_ranks.get(node_id),
+               "bm25_rank": trace.bm25_ranks.get(node_id)}
+        rows.append(row)
+        _atomic_write(brain.path / FEEDBACK_FILE,
+                      json.dumps({"version": 1, "judgments": rows}, ensure_ascii=False, indent=2) + "\n")
+        if commit:
+            brain.commit_and_push(f"feedback: {label} for {node_id}")
+        return row
+
+
+def cmd_feedback(engine: BrainEngine, args: list[str]) -> None:
+    parser = argparse.ArgumentParser(prog="ig feedback")
+    parser.add_argument("query")
+    parser.add_argument("node_id")
+    parser.add_argument("label", choices=("relevant", "irrelevant"))
+    parser.add_argument("--json", action="store_true")
+    opts = parser.parse_args(args)
+    try:
+        result = record_feedback(engine, opts.query, opts.node_id, opts.label)
+    except ValueError as exc:
+        parser.error(str(exc))
+    print(json.dumps(result, indent=2) if opts.json else
+          f"Recorded {opts.label} feedback for {opts.node_id}.")

graph_engine/gaps.py

@@ -0,0 +1,209 @@
+"""Coverage and gap analysis over the brain.
+
+Classifies each node against a topic taxonomy (area → keywords),
+counts coverage per area and identifies underrepresented areas (gaps).
+This steers targeted research instead of broad search.
+
+The taxonomy is generic and swappable via a JSON file
+(format: {"area": ["keyword", ...]}); DEFAULT_TAXONOMY is a sensible
+LLM/agent default. Nothing on the brain is modified (read-only).
+"""
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from typing import Any
+
+from .brain import Brain
+
+# Default taxonomy (LLM/agent research) — generic, no personal data.
+DEFAULT_TAXONOMY: dict[str, list[str]] = {
+    "Agent-Harness & Orchestrierung": [
+        "harness", "orchestrierung", "agent-runtime", "supervisor", "handoff",
+        "loop-kontrolle", "startup", "kontext-injektion", "resilienz",
+        "circuit-breaker", "durable", "plan-and-execute", "agent-loop",
+        "workflow", "subagent", "fanout", "delegation",
+    ],
+    "Coding-Agents & Code": [
+        "code", "programmier", "coding", "software", "repo", "debug",
+        "test-generierung", "code-review", "refactor", "swe-bench", "human-eval",
+        "legacy", "api-generierung", "sql",
+    ],
+    "Multi-Agent-Systeme": [
+        "multi-agent", "verhandlung", "auktion", "konsens", "voting",
+        "kooperation", "konkurrenz", "spieltheorie", "mechanism-design",
+        "incentive", "marktplatz", "interoperabilitaet", "blackboard",
+        "agenten-routing", "schwarm", "peer-to-peer",
+    ],
+    "LLM-Architektur & Interna": [
+        "attention", "transformer", "token", "embedding", "kv-cache", "position",
+        "rope", "mamba", "ssm", "mixture", "moe", "architektur", "logits",
+        "softmax", "layernorm", "tokenisierung", "bpe", "neuron",
+    ],
+    "Training & Alignment": [
+        "rlhf", "dpo", "grpo", "ppo", "alignment", "fine-tuning", "finetuning",
+        "lora", "peft", "qlora", "sft", "reward", "preference", "synthetic",
+        "curriculum", "gradient", "loss", "optimizer", "adamw", "training",
+    ],
+    "Evals & Benchmarks": [
+        "eval", "benchmark", "metrik", "leaderboard", "mmlu", "gsm8k", "judge",
+        "kontamination", "golden", "oracle", "pass@k", "halluzination-messung",
+        "test", "verifikation",
+    ],
+    "Retrieval & RAG": [
+        "retrieval", "rag", "rerank", "bm25", "dense", "vector",
+        "embedding-search", "chunk", "index", "graph-rag", "hybrid",
+        "semantische-suche", "cross-encoder", "late-chunking",
+    ],
+    "Memory & Kontext": [
+        "memory", "gedaechtnis", "kontext", "working-memory", "episodic",
+        "context-engineering", "token-budget", "context-window", "scratchpad",
+        "langzeit", "hierarchisches", "summarization", "kontext-kompression",
+    ],
+    "Serving & Inferenz": [
+        "serving", "inferenz", "vllm", "batching", "prefill", "decode",
+        "throughput", "latenz", "quantisierung", "gpu", "inference",
+        "speculative", "disaggregated", "ttft", "pagedattention",
+    ],
+    "Sicherheit & Governance": [
+        "sicherheit", "security", "jailbreak", "injection", "sandbox",
+        "governance", "privacy", "datenschutz", "ethik", "regulierung", "audit",
+        "compliance", "exfiltration", "autorisierung", "trust", "verantwortung",
+    ],
+    "Multimodal": [
+        "multimodal", "vision", "bild", "audio", "speech", "tts", "video",
+        "clip", "image", "sprach-synthese", "geste", "cross-modal",
+        "text-to-image",
+    ],
+    "Konversations-KI": [
+        "konversation", "dialog", "chatbot", "persona", "intent", "slot",
+        "multi-turn", "chat", "gespraech", "engagement", "turn-qualitaet",
+        "empathie",
+    ],
+    "Interpretability & Mechanistik": [
+        "interpretab", "mechanistic", "sae", "sparse autoencoder", "activation",
+        "circuit", "logit-lens", "probing", "attribution", "causal", "ablation",
+        "steering", "transparenz", "erklaer",
+    ],
+    "Anwendungen": [
+        "industr", "fertigung", "medizin", "biologie", "chemie", "physik",
+        "bildung", "tutor", "lern", "content", "marketing", "journalismus",
+        "recht", "finanz", "gesundheit", "wissenschaft", "astronomie", "ocean",
+        "klima",
+    ],
+    "Oekonomie & Gesellschaft": [
+        "oekonomie", "kosten", "open-source", "closed", "arbeitsmarkt", "umwelt",
+        "gesellschaft", "wirtschaft", "markt", "lizenz", "urheber",
+    ],
+}
+
+
+def normalize(text: str) -> str:
+    """Lowercase + umlaut normalization (ä→ae etc.)."""
+    s = text.lower()
+    for a, b in (("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("ß", "ss")):
+        s = s.replace(a, b)
+    return s
+
+
+def _node_text(node: Any) -> str:
+    """Node body without frontmatter.
+
+    Audit #25: the blind parts[2] lost everything after an internal
+    `---` horizontal rule in the body. Frontmatter end is the FIRST
+    `---` line alone — everything after that is the complete body.
+    """
+    t = getattr(node, "text", "") or ""
+    m = re.match(r"^---\n.*?\n---\n\n?(.*)$", t, re.DOTALL)
+    if m:
+        return m.group(1).strip()
+    return t.strip()
+
+
+@dataclass
+class AreaStat:
+    name: str
+    count: int = 0
+    examples: list[dict] = field(default_factory=list)
+
+
+@dataclass
+class CoverageResult:
+    total: int
+    areas: list[AreaStat]  # sorted by count descending
+    unclassified: int
+    taxonomy: dict[str, list[str]]
+
+
+def analyze_coverage(brain: Brain, taxonomy: dict[str, list[str]] | None = None) -> CoverageResult:
+    """Classifies all nodes against the taxonomy and counts coverage per area.
+
+    A node can belong to multiple areas (keyword overlap).
+    Read-only — does not modify the brain.
+    """
+    tax = taxonomy or DEFAULT_TAXONOMY
+    count: Counter = Counter()
+    examples: dict[str, list] = defaultdict(list)
+    nodes = brain.read_nodes()
+    # Tombstones are append-only history, not live knowledge: they must not
+    # count toward coverage (they would inflate UNCLASSIFIED and every area).
+    nodes = [n for n in nodes if n.status != "tombstone"]
+    unclassified = 0
+    # Audit #60: substring matching let "test" match "latest" — keywords now
+    # match with word boundaries (umlauts are ASCII-shaped by normalize()).
+    kw_res = {area: [re.compile(rf"\b{re.escape(k)}\b") for k in kws]
+              for area, kws in tax.items()}
+    for node in nodes:
+        b = normalize(_node_text(node))
+        matched = [area for area, patterns in kw_res.items()
+                   if any(p.search(b) for p in patterns)]
+        if not matched:
+            unclassified += 1
+        for m in matched:
+            count[m] += 1
+            if len(examples[m]) < 5:
+                examples[m].append({"id": node.id, "text": _node_text(node)[:72]})
+    areas = [AreaStat(name=a, count=count[a], examples=examples[a]) for a in tax]
+    areas.sort(key=lambda s: s.count, reverse=True)
+    return CoverageResult(total=len(nodes), areas=areas, unclassified=unclassified, taxonomy=tax)
+
+
+def find_gaps(coverage: CoverageResult, threshold: int) -> list[AreaStat]:
+    """Areas below the threshold, ascending by coverage (thinnest first)."""
+    return [a for a in sorted(coverage.areas, key=lambda s: s.count) if a.count < threshold]
+
+
+def render(coverage: CoverageResult, threshold: int) -> str:
+    """Textual coverage/gap report."""
+    lines = [
+        f"Coverage ({coverage.total} nodes, {len(coverage.areas)} areas):",
+        f"{'Nodes':>5}  {'Area':<40}  Coverage",
+        "-----  " + "-" * 40 + "  ---------",
+    ]
+    maxc = max((a.count for a in coverage.areas), default=1)
+    # Audit #24: a brain without hits (all counts 0) must not crash with
+    # ZeroDivisionError — maxc at least 1.
+    maxc = max(maxc, 1)
+    for a in coverage.areas:
+        bar = "#" * int(a.count / maxc * 30)
+        gap = "  ← GAP" if a.count < threshold else ""
+        lines.append(f"{a.count:>5}  {a.name:<40}  {bar}{gap}")
+    lines.append(f"{coverage.unclassified:>5}  {'UNCLASSIFIED':<40}")
+    gaps = find_gaps(coverage, threshold)
+    if gaps:
+        lines.append("")
+        lines.append(f"GAPS (< {threshold} nodes) — next research targets (thinnest first):")
+        for a in gaps:
+            lines.append(f"  {a.count:>3}  {a.name}")
+    return "\n".join(lines)
+
+
+def load_taxonomy(path: str) -> dict[str, list[str]]:
+    """Loads a taxonomy from a JSON file ({"area": ["kw", ...]})."""
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, dict) or not all(isinstance(v, list) for v in data.values()):
+        raise ValueError("Taxonomy must be a JSON object: {\"area\": [\"kw\", ...]}")
+    return data

graph_engine/graph.py

@@ -0,0 +1,133 @@
+"""Graph traversal over live edges (report #1, MCP phase 1).
+
+SDK-free and CLI-reusable: the MCP `neighbors` tool and any future CLI
+command share these functions. The undirected view is deliberate — "what
+is this node connected to" does not care which endpoint was newer at
+ingest time (edges always point newest -> older).
+"""
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from .brain import Brain, Edge, Node
+
+
+@dataclass
+class Neighbor:
+    id: str
+    snippet: str
+    kind: str
+    direction: str      # "out" (this node is the source) | "in"
+    pending: bool
+    confidence: float | None
+    hops: int
+    predicate: str | None = None
+
+
+def live_edges(brain: Brain, include_pending: bool = False) -> list[Edge]:
+    """Edges that count as real relations: currently valid and not invalidated,
+    not rejected, endpoints not tombstoned (hygiene.py precedent — tombstones
+    are edge-less BY DESIGN), and optionally not pending."""
+    node_status = {n.id: n.status for n in brain.read_nodes()}
+    out = []
+    for e in brain.read_edges():
+        if not e.is_current or e.rejected:
+            continue
+        if e.source in node_status and node_status[e.source] == "tombstone":
+            continue
+        if e.target in node_status and node_status[e.target] == "tombstone":
+            continue
+        if e.pending and not include_pending:
+            continue
+        out.append(e)
+    return out
+
+
+def neighbors(brain: Brain, node_id: str, hops: int = 1,
+              kinds: list[str] | None = None,
+              include_pending: bool = False,
+              limit: int | None = None) -> list[Neighbor]:
+    """BFS over the UNDIRECTED live-edge set, deterministic order.
+
+    Ordering: by (hops, kind, id) so tests are stable regardless of dict
+    iteration order. `limit` caps the RESULT (after sorting), not the
+    per-hop expansion — a cap during expansion would make the result
+    order-dependent in a subtle way.
+    """
+    edges = live_edges(brain, include_pending=include_pending)
+    if kinds is not None:
+        allowed = set(kinds)
+        edges = [e for e in edges if e.kind in allowed]
+    adj: dict[str, list[tuple[str, Edge, str]]] = {}
+    for e in edges:
+        adj.setdefault(e.source, []).append((e.target, e, "out"))
+        adj.setdefault(e.target, []).append((e.source, e, "in"))
+
+    id2node = {n.id: n for n in brain.read_nodes()}
+    seen: dict[str, int] = {node_id: 0}
+    results: list[Neighbor] = []
+    queue = deque([node_id])
+    while queue:
+        cur = queue.popleft()
+        depth = seen[cur]
+        if depth >= hops:
+            continue
+        for other, edge, direction in sorted(adj.get(cur, []),
+                                             key=lambda t: (t[1].kind, t[0])):
+            if other in seen:
+                continue
+            seen[other] = depth + 1
+            node = id2node.get(other)
+            results.append(Neighbor(
+                id=other,
+                snippet=(node.text if node else "?"),
+                kind=edge.kind,
+                direction=direction if depth == 0 else
+                          ("in" if direction == "out" else "out"),
+                pending=edge.pending,
+                confidence=edge.confidence,
+                hops=depth + 1,
+                predicate=edge.predicate,
+            ))
+            queue.append(other)
+    results.sort(key=lambda nb: (nb.hops, nb.kind, nb.id))
+    if limit is not None:
+        results = results[:limit]
+    return results
+
+
+def shortest_path(brain: Brain, a: str, b: str, max_depth: int = 4,
+                  include_pending: bool = False) -> list[str] | None:
+    """BFS shortest path a -> b over undirected live edges, depth-capped.
+    Returns the node id sequence INCLUDING both endpoints, or None."""
+    if a == b:
+        return [a]
+    edges = live_edges(brain, include_pending=include_pending)
+    adj: dict[str, list[str]] = {}
+    for e in edges:
+        adj.setdefault(e.source, []).append(e.target)
+        adj.setdefault(e.target, []).append(e.source)
+    prev: dict[str, str] = {a: a}
+    depth: dict[str, int] = {a: 0}
+    queue = deque([a])
+    while queue:
+        cur = queue.popleft()
+        if depth[cur] >= max_depth:
+            continue
+        for nxt in sorted(adj.get(cur, [])):
+            if nxt in prev:
+                continue
+            prev[nxt] = cur
+            depth[nxt] = depth[cur] + 1
+            if nxt == b:
+                return _walk(prev, b)
+            queue.append(nxt)
+    return None
+
+
+def _walk(prev: dict[str, str], end: str) -> list[str]:
+    path = [end]
+    while prev[path[-1]] != path[-1]:
+        path.append(prev[path[-1]])
+    return list(reversed(path))