structuremappingmemory 1.0.0__py3-none-any.whl

sma/eval/agentic/memories.py

@@ -0,0 +1,308 @@
+"""Uniform ``Memory`` interface + six retriever wrappers for the agentic suite.
+
+The agentic harness holds everything fixed except the retrieval ``Memory``: SMA
+(the universal ontology adapter) vs an enterprise-RAG/KG gauntlet (BM25, BGE
+dense, Hybrid-RRF, Hybrid+Rerank, HippoRAG). Every memory implements the same
+three-method contract so the harness never sees a retriever's internals:
+
+    index(items)              -> None      # build over IndexItem records
+    retrieve(query, k)        -> list[Retrieved]   # ranked, rank 1..k
+    novelty(query)            -> float     # in [0,1], higher = "this is new"
+
+Confidence (on each :class:`Retrieved`) drives cite-or-abstain and is squashed
+to ``[0,1]`` per method; novelty is the method's best out-of-distribution signal.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Iterable, Protocol
+
+from sma.eval.baselines.bm25 import rank_bm25_like
+from sma.eval.baselines.hipporag import HippoRAGRetriever
+from sma.index.macfac import MacFacIndex
+from sma.ontology import MountedOntology
+from sma.sage.pools import SagePool
+
+
+@dataclass
+class IndexItem:
+    """One indexable entity (the gold answer is its ``key``)."""
+
+    key: str  # entity id (gold answer)
+    term_ids: frozenset[str]  # ontology term ids (for SMA)
+    text: str  # space-joined term NAMES (for text baselines)
+    meta: dict = field(default_factory=dict)
+
+
+@dataclass
+class Query:
+    """A retrieval query in both ontology-term and text form."""
+
+    term_ids: frozenset[str]
+    text: str
+
+
+@dataclass
+class Retrieved:
+    """A single ranked hit returned by a :class:`Memory`."""
+
+    key: str
+    score: float
+    confidence: float  # drives cite-or-abstain; method-specific, in [0,1]
+    rank: int
+
+
+class Memory(Protocol):
+    """The only thing that varies across harness runs."""
+
+    name: str
+
+    def index(self, items: list[IndexItem]) -> None: ...
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]: ...
+
+    def novelty(self, query: Query) -> float:  # higher = more "this is new"
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Task 2: SMA memory (universal adapter)
+# ---------------------------------------------------------------------------
+
+
+class SmaMemory:
+    """SMA: mount an ontology, index cases via MacFac, novelty via SagePool."""
+
+    name = "sma"
+
+    def __init__(self, mounted: MountedOntology):
+        self.mounted = mounted
+
+    def index(self, items: list[IndexItem]) -> None:
+        self._key: dict[str, str] = {}
+        cases = []
+        self.pool = SagePool("agentic", assimilation_threshold=0.2)
+        for it in items:
+            c = self.mounted.build_case(it.term_ids, metadata={"key": it.key})
+            self._key[c.case_id] = it.key
+            cases.append(c)
+            self.pool.assimilate(c)
+        self.index_ = MacFacIndex(config=self.mounted.config, canon=self.mounted.canon)
+        self.index_.build(cases)
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]:
+        qc = self.mounted.build_case(query.term_ids)
+        res = self.index_.retrieve(qc, k=k, shortlist=80, fac_budget=40)
+        if not res:
+            return []
+        top = max(r.score for r in res) or 1.0
+        out = []
+        for i, r in enumerate(res, 1):
+            conf = min(max(r.score / top, 0.0), 1.0)
+            out.append(Retrieved(self._key.get(r.case_id, ""), r.score, conf, i))
+        return out
+
+    def novelty(self, query: Query) -> float:
+        return self.pool.expectation_violation(self.mounted.build_case(query.term_ids))
+
+
+# ---------------------------------------------------------------------------
+# Task 3: text baselines — BM25, Dense (BGE), HippoRAG
+# ---------------------------------------------------------------------------
+
+
+class BM25Memory:
+    """Lexical BM25-like baseline over term-name documents."""
+
+    name = "bm25"
+
+    def index(self, items: list[IndexItem]) -> None:
+        self._docs: list[tuple[str, str]] = [(it.key, it.text) for it in items]
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]:
+        ranked = rank_bm25_like(query.text, self._docs, k=k)
+        if not ranked:
+            return []
+        top = ranked[0][1]
+        conf = top / (top + 1.0) if top > 0 else 0.0
+        return [Retrieved(key, score, conf, i) for i, (key, score) in enumerate(ranked, 1)]
+
+    def novelty(self, query: Query) -> float:
+        ranked = rank_bm25_like(query.text, self._docs, k=1)
+        if not ranked:
+            return 1.0
+        top = ranked[0][1]
+        conf = top / (top + 1.0) if top > 0 else 0.0
+        return 1.0 - conf
+
+
+_BGE: dict = {}
+
+
+def _bge():
+    """Load the BGE-small embedder once (cached at module level)."""
+    if "m" not in _BGE:
+        from sentence_transformers import SentenceTransformer
+
+        _BGE["m"] = SentenceTransformer("BAAI/bge-small-en-v1.5")
+    return _BGE["m"]
+
+
+class DenseMemory:
+    """Neural dense baseline: BGE-small embeddings + cosine similarity."""
+
+    name = "dense"
+
+    def index(self, items: list[IndexItem]) -> None:
+        self._keys = [it.key for it in items]
+        texts = [it.text for it in items]
+        model = _bge()
+        # normalize_embeddings=True -> cosine == dot product
+        self._mat = model.encode(texts, normalize_embeddings=True)
+
+    def _scores(self, query: Query):
+        model = _bge()
+        q = model.encode([query.text], normalize_embeddings=True)[0]
+        return [float(sum(q[d] * row[d] for d in range(len(q)))) for row in self._mat]
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]:
+        if not self._keys:
+            return []
+        scores = self._scores(query)
+        order = sorted(range(len(scores)), key=lambda i: (-scores[i], self._keys[i]))[:k]
+        top = max(scores) if scores else 0.0
+        conf = min(max(top, 0.0), 1.0)
+        return [Retrieved(self._keys[i], scores[i], conf, rank) for rank, i in enumerate(order, 1)]
+
+    def novelty(self, query: Query) -> float:
+        if not self._keys:
+            return 1.0
+        top = max(self._scores(query))
+        return 1.0 - min(max(top, 0.0), 1.0)
+
+
+class HippoMemory:
+    """HippoRAG-2-style KG retrieval (phrase graph + Personalized PageRank)."""
+
+    name = "hipporag"
+
+    def index(self, items: list[IndexItem]) -> None:
+        self._retriever = HippoRAGRetriever()
+        self._retriever.build([(it.key, it.text) for it in items])
+        self._n = len(items)
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]:
+        ranked = self._retriever.retrieve(query.text, k=k)
+        if not ranked:
+            return []
+        total = sum(s for _, s in ranked) or 1.0
+        conf = min(max(ranked[0][1] / total, 0.0), 1.0)
+        return [Retrieved(key, score, conf, i) for i, (key, score) in enumerate(ranked, 1)]
+
+    def novelty(self, query: Query) -> float:
+        ranked = self._retriever.retrieve(query.text, k=max(1, self._n))
+        if not ranked:
+            return 1.0
+        total = sum(s for _, s in ranked) or 1.0
+        conf = min(max(ranked[0][1] / total, 0.0), 1.0)
+        return 1.0 - conf
+
+
+# ---------------------------------------------------------------------------
+# Task 4: SOTA hybrid — Hybrid-RRF + Hybrid+Rerank
+# ---------------------------------------------------------------------------
+
+
+class HybridRRFMemory:
+    """Reciprocal Rank Fusion of a BM25 and a dense member memory."""
+
+    name = "hybrid_rrf"
+
+    def __init__(self, bm25_mem: BM25Memory, dense_mem: DenseMemory, k_rrf: int = 60):
+        self.bm25_mem = bm25_mem
+        self.dense_mem = dense_mem
+        self.k_rrf = k_rrf
+
+    def index(self, items: list[IndexItem]) -> None:
+        self.bm25_mem.index(items)
+        self.dense_mem.index(items)
+        self._n = len(items)
+
+    def _fused(self, query: Query, k: int) -> list[tuple[str, float]]:
+        depth = max(k, self._n)
+        fused: dict[str, float] = {}
+        for mem in (self.bm25_mem, self.dense_mem):
+            for r in mem.retrieve(query, depth):
+                fused[r.key] = fused.get(r.key, 0.0) + 1.0 / (self.k_rrf + r.rank)
+        return sorted(fused.items(), key=lambda kv: (-kv[1], kv[0]))[:k]
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]:
+        ranked = self._fused(query, k)
+        if not ranked:
+            return []
+        top = ranked[0][1] or 1.0
+        return [
+            Retrieved(key, score, min(max(score / top, 0.0), 1.0), i)
+            for i, (key, score) in enumerate(ranked, 1)
+        ]
+
+    def novelty(self, query: Query) -> float:
+        return min(self.bm25_mem.novelty(query), self.dense_mem.novelty(query))
+
+
+_RERANKER: dict = {}
+
+
+def _reranker(name: str = "BAAI/bge-reranker-base"):
+    """Load the BGE cross-encoder reranker once (cached at module level)."""
+    if name not in _RERANKER:
+        from sentence_transformers import CrossEncoder
+
+        _RERANKER[name] = CrossEncoder(name)
+    return _RERANKER[name]
+
+
+def _sigmoid(x: float) -> float:
+    return 1.0 / (1.0 + math.exp(-x))
+
+
+class HybridRerankMemory:
+    """Cross-encoder reranking of a hybrid memory's top-n candidates."""
+
+    name = "hybrid_rerank"
+
+    def __init__(
+        self,
+        hybrid: HybridRRFMemory,
+        cross_encoder: str = "BAAI/bge-reranker-base",
+        top_n: int = 30,
+    ):
+        self.hybrid = hybrid
+        self.cross_encoder = cross_encoder
+        self.top_n = top_n
+
+    def index(self, items: list[IndexItem]) -> None:
+        self.hybrid.index(items)
+        self._text = {it.key: it.text for it in items}
+
+    def retrieve(self, query: Query, k: int) -> list[Retrieved]:
+        candidates = self.hybrid.retrieve(query, self.top_n)
+        if not candidates:
+            return []
+        model = _reranker(self.cross_encoder)
+        pairs = [(query.text, self._text.get(c.key, "")) for c in candidates]
+        logits = model.predict(pairs)
+        scored = sorted(
+            zip(candidates, (float(s) for s in logits)),
+            key=lambda cs: (-cs[1], cs[0].key),
+        )[:k]
+        top_conf = _sigmoid(scored[0][1])
+        return [
+            Retrieved(c.key, logit, top_conf, i)
+            for i, (c, logit) in enumerate(scored, 1)
+        ]
+
+    def novelty(self, query: Query) -> float:
+        return self.hybrid.novelty(query)

sma/eval/agentic/metrics.py

@@ -0,0 +1,82 @@
+"""Metrics for the agentic ontology suite.
+
+Three headline metrics, matching the design spec (prereg section 4):
+
+* ``tail_topk`` — per-method top-k accuracy on ALL queries and on the
+  registered RARE slice (the tail). Top-k accuracy is the fraction of
+  rows whose true-entity rank is <= k, reusing the convention from
+  ``sma.eval.ontology_bench``.
+* ``risk_coverage_aurc`` — cite-or-abstain selective-prediction curve:
+  sort by confidence (desc), sweep coverage 0->1, risk is the cumulative
+  error rate over the covered head. AURC is the mean risk over the sweep;
+  LOWER is better (a well-calibrated ranker keeps its mistakes for last).
+* ``novelty_f1`` — F1 of predicted-novel flags vs the truly held-out set.
+"""
+
+from __future__ import annotations
+
+ABSENT_RANK = 999
+
+
+def tail_topk(
+    rows: list[dict],
+    k: int,
+) -> dict[str, dict[str, float]]:
+    """Per-method top-k accuracy on the ALL slice and the RARE slice.
+
+    ``rows`` is a list of ``{method_name: rank, "rare": bool}`` dicts, where
+    ``rank`` is the 1-based rank of the true entity (``ABSENT_RANK`` if it
+    never surfaced). Returns ``{method: {"all": acc, "rare": acc}}`` where
+    ``acc`` is the fraction of rows with ``rank <= k``.
+    """
+    methods = sorted({m for r in rows for m in r if m != "rare"})
+    rare_rows = [r for r in rows if r.get("rare")]
+    out: dict[str, dict[str, float]] = {}
+    for m in methods:
+        out[m] = {
+            "all": _topk_acc(rows, m, k),
+            "rare": _topk_acc(rare_rows, m, k),
+        }
+    return out
+
+
+def _topk_acc(rows: list[dict], method: str, k: int) -> float:
+    """Fraction of ``rows`` whose ``method`` rank is <= k."""
+    if not rows:
+        return 0.0
+    hits = sum(1 for r in rows if r.get(method, ABSENT_RANK) <= k)
+    return hits / len(rows)
+
+
+def risk_coverage_aurc(
+    confidences: list[float],
+    correct: list[bool],
+) -> tuple[float, list[tuple[float, float]]]:
+    """Area under the risk-coverage curve (lower is better).
+
+    Items are sorted by ``confidences`` descending; coverage sweeps from the
+    most-confident prediction to all of them. At each coverage point the risk
+    is the cumulative error rate over the covered head. Returns the AURC (mean
+    risk over the sweep) and the ``(coverage, risk)`` curve points.
+    """
+    n = len(correct)
+    if n == 0:
+        return 0.0, []
+    order = sorted(range(n), key=lambda i: -confidences[i])
+    cum_err = 0
+    pts: list[tuple[float, float]] = []
+    for j, i in enumerate(order, 1):
+        cum_err += 0 if correct[i] else 1
+        pts.append((j / n, cum_err / j))  # (coverage, risk)
+    aurc = sum(r for _, r in pts) / max(len(pts), 1)
+    return aurc, pts
+
+
+def novelty_f1(pred: list[bool], truth: list[bool]) -> float:
+    """F1 of predicted-novel flags vs the truly held-out (novel) set."""
+    tp = sum(1 for p, t in zip(pred, truth) if p and t)
+    fp = sum(1 for p, t in zip(pred, truth) if p and not t)
+    fn = sum(1 for p, t in zip(pred, truth) if not p and t)
+    prec = tp / (tp + fp) if tp + fp else 0.0
+    rec = tp / (tp + fn) if tp + fn else 0.0
+    return 2 * prec * rec / (prec + rec) if prec + rec else 0.0

sma/eval/agentic_qa/__init__.py

@@ -0,0 +1,27 @@
+"""Phase 5 LLM-QA harness: the one-shot agent + trustworthy-specialist metrics."""
+
+from __future__ import annotations
+
+from sma.eval.agentic_qa.agent import MockLLM, QAAgent
+from sma.eval.agentic_qa.metrics import (
+    abstention,
+    accuracy,
+    citation_faithfulness,
+    grounding_auroc,
+    novelty_f1,
+    novelty_recall,
+)
+from sma.eval.agentic_qa.pools import QAItem, build_pools
+
+__all__ = [
+    "QAAgent",
+    "MockLLM",
+    "QAItem",
+    "build_pools",
+    "accuracy",
+    "citation_faithfulness",
+    "abstention",
+    "grounding_auroc",
+    "novelty_recall",
+    "novelty_f1",
+]