structuremappingmemory 1.0.0__py3-none-any.whl

sma/eval/agentic_qa/agent.py

@@ -0,0 +1,383 @@
+"""The one-shot QA agent for the Phase 5 LLM-QA "trustworthy specialist" phase.
+
+A single :class:`QAAgent` holds the LLM and prompt FIXED and swaps only the
+retrieval ``Memory`` (none / dense-RAG / SMA), exactly as registered in
+``configs/preregistration_v2_llmqa.md`` section 2. For each
+:class:`~sma.eval.agentic_qa.pools.QAItem` it runs one agent turn and returns a
+result dict carrying every field the trustworthy-QA metrics read
+(``sma.eval.agentic_qa.metrics``): ``gold_id``, ``gold_name``, ``answerable``,
+``novel``, ``abstained``, ``pred_id``, ``answer``, ``novelty_flag``,
+``confidence``, ``grounding_score``.
+
+Two grounding regimes:
+
+* **grounded** (a memory is given) — retrieve top-k candidates, render them as a
+  numbered list, and ask the LLM for a strict one-line JSON ``{"choice": <n>}``
+  where ``n`` is a candidate number or ``0`` to abstain. ``pred_id`` is the
+  chosen candidate's key (the disease id), so correctness/citation can be checked
+  structurally against the gold. When a calibrated ``score_threshold`` is given,
+  a case whose top RAW grounding score falls below it is abstained AND flagged
+  novel *before* the LLM call (the structural score, not the saturated confidence
+  or the expectation-violation flag, is what separates known from unknown); with
+  no threshold the novelty flag falls back to ``memory.novelty(query)``.
+* **closed-book** (``memory is None``) — the LLM answers from the case alone with
+  a strict one-line JSON ``{"diagnosis": "<name or ABSTAIN>"}``; ``pred_id`` is
+  ``None`` (no citation), ``confidence`` is a flat ``0.5``, and novelty is N/A.
+
+JSON parsing is defensive (strips ```` ``` ```` code fences, scans for the first
+``{...}`` object) and falls back to ABSTAIN on any parse/validation failure, so a
+malformed model reply degrades to the safe action rather than crashing the run.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Protocol
+
+from sma.eval.agentic import Query
+from sma.eval.agentic_qa.pools import QAItem
+
+# How many characteristic feature names to show per candidate (keeps the prompt
+# bounded; the candidate is identified by its number, not by an exhaustive list).
+_FEATURES_PER_CANDIDATE = 6
+
+SYSTEM_PROMPT = (
+    "You are a careful diagnostic assistant. You are given a clinical case and a "
+    "numbered list of candidate diseases retrieved from a grounded knowledge base, "
+    "each with a few of its characteristic features. Choose the single candidate "
+    "whose characteristic features best match the case. Answer ONLY when a "
+    "candidate genuinely grounds the case; if none of the candidates fit, abstain. "
+    "Reply with STRICT one-line JSON and nothing else: "
+    '{"choice": <candidate number, or 0 for none / abstain>}.'
+)
+
+CLOSED_BOOK_SYSTEM_PROMPT = (
+    "You are a careful diagnostic assistant. You are given a clinical case and no "
+    "external knowledge. Name the single most likely disease, or abstain if you are "
+    "not confident. Reply with STRICT one-line JSON and nothing else: "
+    '{"diagnosis": "<disease name, or ABSTAIN>"}.'
+)
+
+ABSTAIN = "ABSTAIN"
+
+
+class LLM(Protocol):
+    """The fixed LLM backend (``DeepSeekOrchestrator`` or a mock in tests)."""
+
+    def complete(
+        self, messages: list[dict], max_tokens: int = 600, temperature: float = 0.0
+    ) -> str: ...
+
+
+class MockLLM:
+    """A deterministic stand-in for the real LLM (NEVER calls DeepSeek).
+
+    Used by the tests and the ``--mock`` driver so the whole harness can run with
+    zero API spend. By default it picks candidate ``1`` in the grounded regime and
+    echoes a fixed diagnosis closed-book; pass ``choice`` / ``diagnosis`` to script
+    other behaviours (e.g. ``choice=0`` to exercise the abstain path). When
+    ``raw`` is set it is returned verbatim, to test defensive JSON parsing.
+    """
+
+    def __init__(
+        self,
+        choice: int = 1,
+        diagnosis: str = "Mock disease",
+        raw: str | None = None,
+    ):
+        self.choice = choice
+        self.diagnosis = diagnosis
+        self.raw = raw
+        self.calls: list[list[dict]] = []
+
+    def complete(
+        self, messages: list[dict], max_tokens: int = 600, temperature: float = 0.0
+    ) -> str:
+        self.calls.append(messages)
+        if self.raw is not None:
+            return self.raw
+        # Closed-book prompts ask for a "diagnosis" key; grounded ask for "choice".
+        system = messages[0]["content"] if messages else ""
+        if "diagnosis" in system:
+            return json.dumps({"diagnosis": self.diagnosis})
+        return json.dumps({"choice": self.choice})
+
+
+def _strip_fences(text: str) -> str:
+    """Drop Markdown code fences so JSON wrapped in ```` ```json ... ``` ```` parses."""
+    t = text.strip()
+    if t.startswith("```"):
+        # Remove the opening fence (with optional language tag) and closing fence.
+        t = re.sub(r"^```[a-zA-Z0-9]*\s*", "", t)
+        t = re.sub(r"\s*```$", "", t.strip())
+    return t.strip()
+
+
+def _parse_json_object(text: str) -> dict | None:
+    """Best-effort parse of a single JSON object from a (possibly noisy) reply.
+
+    Tries the whole stripped string first, then falls back to the first balanced
+    ``{...}`` substring. Returns ``None`` when nothing parses to a dict.
+    """
+    stripped = _strip_fences(text)
+    for candidate in (stripped, _first_brace_object(stripped)):
+        if not candidate:
+            continue
+        try:
+            obj = json.loads(candidate)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if isinstance(obj, dict):
+            return obj
+    return None
+
+
+def _first_brace_object(text: str) -> str | None:
+    """Return the first balanced ``{...}`` substring, or ``None``."""
+    start = text.find("{")
+    if start < 0:
+        return None
+    depth = 0
+    for i in range(start, len(text)):
+        c = text[i]
+        if c == "{":
+            depth += 1
+        elif c == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return None
+
+
+class QAAgent:
+    """One-shot retrieve-then-answer agent with a swappable retrieval memory.
+
+    ``memory`` is one of the frozen ``Memory`` retrievers (``SmaMemory`` /
+    ``DenseMemory`` / ...) or ``None`` for the closed-book condition. ``key_to_name``
+    / ``key_to_terms`` map an :class:`IndexItem` key (disease id) to its display
+    name and its ontology term ids, used to render the numbered candidate list;
+    pass the same maps that back the indexed knowledge. ``k`` is the retrieval
+    depth and ``novelty_threshold`` is the cut for the ``expectation_violation``
+    novelty flag (only meaningful for SMA).
+    """
+
+    def __init__(
+        self,
+        llm: LLM,
+        memory: Any | None,
+        *,
+        key_to_name: dict[str, str] | None = None,
+        key_to_terms: dict[str, frozenset[str]] | None = None,
+        k: int = 5,
+        novelty_threshold: float = 0.5,
+        score_threshold: float | None = None,
+    ):
+        self.llm = llm
+        self.memory = memory
+        self.key_to_name = key_to_name or {}
+        self.key_to_terms = key_to_terms or {}
+        self.k = k
+        self.novelty_threshold = novelty_threshold
+        # Calibrated cite-or-abstain: the RAW structural grounding score (not the
+        # saturated normalized confidence, nor the expectation-violation flag — both
+        # of which fail to separate known/unknown, AUROC~0.48) is the abstention
+        # signal. Below this threshold the memory has no grounding -> abstain + flag
+        # novel, WITHOUT spending an LLM call. None = no gate (LLM-only abstention).
+        self.score_threshold = score_threshold
+
+    # -- rendering ----------------------------------------------------------
+    def _feature_text(self, key: str) -> str:
+        """A few characteristic feature NAMES for a candidate disease."""
+        terms = sorted(self.key_to_terms.get(key, frozenset()))
+        names = [self._term_name(t) for t in terms[:_FEATURES_PER_CANDIDATE]]
+        return ", ".join(n for n in names if n)
+
+    def _term_name(self, term_id: str) -> str:
+        """Resolve a term id to a human name via the SMA ontology when available."""
+        mounted = getattr(self.memory, "mounted", None)
+        if mounted is not None:
+            term = mounted.graph.terms.get(term_id)
+            if term is not None and term.name:
+                return term.name
+        return term_id
+
+    def _render_candidates(self, retrieved: list) -> tuple[str, list[str]]:
+        """Build the numbered candidate block and the parallel key list.
+
+        Returns ``(text, keys)`` where ``keys[i]`` is the disease id of candidate
+        ``i + 1`` (so a parsed ``{"choice": n}`` maps to ``keys[n - 1]``).
+        """
+        lines: list[str] = []
+        keys: list[str] = []
+        for i, r in enumerate(retrieved, 1):
+            keys.append(r.key)
+            name = self.key_to_name.get(r.key, r.key)
+            features = self._feature_text(r.key)
+            feat = f" -- characteristic features: {features}" if features else ""
+            lines.append(f"[{i}] {name}{feat}")
+        return "\n".join(lines), keys
+
+    # -- answer -------------------------------------------------------------
+    def answer(self, item: QAItem) -> dict:
+        """Run one agent turn over ``item`` and return the metrics result dict."""
+        if self.memory is None:
+            return self._answer_closed_book(item)
+        return self._answer_grounded(item)
+
+    def _result(
+        self,
+        item: QAItem,
+        *,
+        abstained: bool,
+        pred_id: str | None,
+        answer: str,
+        novelty_flag: bool,
+        confidence: float,
+        grounding_score: float | None,
+    ) -> dict:
+        """Assemble the per-item result dict the trustworthy-QA metrics read."""
+        return {
+            "gold_id": item.gold_id,
+            "gold_name": item.gold_name,
+            "answerable": item.answerable,
+            "novel": item.novel,
+            "abstained": abstained,
+            "pred_id": pred_id,
+            "answer": answer,
+            "novelty_flag": novelty_flag,
+            "confidence": confidence,
+            # The RAW top structural grounding score (None closed-book). This is
+            # the signal that actually separates known from unknown; the metrics
+            # use it for threshold-free discrimination AUROC.
+            "grounding_score": grounding_score,
+        }
+
+    def _answer_grounded(self, item: QAItem) -> dict:
+        query = Query(item.case_terms, item.case_text)
+        retrieved = self.memory.retrieve(query, self.k)
+        confidence = retrieved[0].confidence if retrieved else 0.0
+        grounding_score = retrieved[0].score if retrieved else 0.0
+
+        # Calibrated cite-or-abstain. If the top RAW grounding score is below the
+        # validation-calibrated threshold, the memory does not structurally ground
+        # this case -> ABSTAIN and FLAG NOVEL, WITHOUT spending an LLM call. The
+        # raw structural match score is the discriminating signal (answerable vs
+        # out-of-knowledge AUROC ~0.84); the squashed confidence (top hit always
+        # ~1.0) and the expectation-violation flag are not (AUROC ~0.48). A None
+        # threshold disables the gate -> pure LLM-mediated abstention (legacy).
+        if self.score_threshold is not None and grounding_score < self.score_threshold:
+            return self._result(
+                item,
+                abstained=True,
+                pred_id=None,
+                answer=ABSTAIN,
+                novelty_flag=True,
+                confidence=confidence,
+                grounding_score=grounding_score,
+            )
+
+        candidates_text, keys = self._render_candidates(retrieved)
+
+        # With a calibrated gate, the structural signal IS the novelty signal:
+        # above threshold here -> not flagged. Without a gate, fall back to the
+        # memory's own expectation-violation novelty vs novelty_threshold.
+        if self.score_threshold is not None:
+            novelty_flag = False
+        else:
+            novelty_flag = bool(self.memory.novelty(query) > self.novelty_threshold)
+
+        user = (
+            f"Clinical case:\n{item.case_text}\n\n"
+            f"Candidate diseases:\n{candidates_text or '(none retrieved)'}\n\n"
+            "Rule: choose the candidate whose characteristic features best match "
+            "the case; answer only if a candidate genuinely grounds the case, "
+            "otherwise choose 0 to abstain.\n"
+            'Reply with STRICT one-line JSON: {"choice": <candidate number or 0>}.'
+        )
+        reply = self.llm.complete(
+            [
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": user},
+            ],
+            max_tokens=600,
+            temperature=0.0,
+        )
+        choice = self._parse_choice(reply, n_candidates=len(keys))
+
+        if choice == 0:
+            pred_id: str | None = None
+            answer = ABSTAIN
+            abstained = True
+        else:
+            pred_id = keys[choice - 1]
+            answer = self.key_to_name.get(pred_id, pred_id)
+            abstained = False
+
+        return self._result(
+            item,
+            abstained=abstained,
+            pred_id=pred_id,
+            answer=answer,
+            novelty_flag=novelty_flag,
+            confidence=confidence,
+            grounding_score=grounding_score,
+        )
+
+    def _answer_closed_book(self, item: QAItem) -> dict:
+        user = (
+            f"Clinical case:\n{item.case_text}\n\n"
+            "Name the single most likely disease, or abstain if not confident.\n"
+            'Reply with STRICT one-line JSON: {"diagnosis": "<disease name or ABSTAIN>"}.'
+        )
+        reply = self.llm.complete(
+            [
+                {"role": "system", "content": CLOSED_BOOK_SYSTEM_PROMPT},
+                {"role": "user", "content": user},
+            ],
+            max_tokens=600,
+            temperature=0.0,
+        )
+        diagnosis = self._parse_diagnosis(reply)
+        abstained = diagnosis.strip().upper() == ABSTAIN
+        answer = ABSTAIN if abstained else diagnosis
+
+        return self._result(
+            item,
+            abstained=abstained,
+            pred_id=None,
+            answer=answer,
+            novelty_flag=False,
+            confidence=0.5,
+            grounding_score=None,
+        )
+
+    # -- parsing ------------------------------------------------------------
+    @staticmethod
+    def _parse_choice(reply: str, *, n_candidates: int) -> int:
+        """Parse ``{"choice": n}`` -> int in ``0..n_candidates``; abstain on failure.
+
+        Any parse error, missing/ill-typed ``choice``, or out-of-range index
+        collapses to ``0`` (abstain), the safe action.
+        """
+        obj = _parse_json_object(reply)
+        if obj is None or "choice" not in obj:
+            return 0
+        try:
+            choice = int(obj["choice"])
+        except (TypeError, ValueError):
+            return 0
+        if choice < 0 or choice > n_candidates:
+            return 0
+        return choice
+
+    @staticmethod
+    def _parse_diagnosis(reply: str) -> str:
+        """Parse ``{"diagnosis": "..."}`` -> str; abstain on failure."""
+        obj = _parse_json_object(reply)
+        if obj is None:
+            return ABSTAIN
+        value = obj.get("diagnosis")
+        if not isinstance(value, str) or not value.strip():
+            return ABSTAIN
+        return value.strip()

sma/eval/agentic_qa/metrics.py

@@ -0,0 +1,239 @@
+"""Trustworthy-QA metrics for the Phase 5 LLM-QA harness (prereg v2 section 4).
+
+Given per-item agent results, compute the four pre-registered axes that
+distinguish a *verifiable specialist* from a confident-but-opaque RAG agent:
+
+* :func:`accuracy` — answer correct on the **answerable** pool (the accuracy
+  floor; the capability gains must not cost accuracy).
+* :func:`citation_faithfulness` — ALCE-style support score over **answered
+  answerable** items: did the cited candidate actually turn out to be the gold?
+  N/A (``None``) for the closed-book condition, which has no citation.
+* :func:`abstention` — selective prediction over the union of **answerable**
+  (should answer) and **held-out / out-of-knowledge** (should abstain):
+  abstain-recall, false-abstain, selective-accuracy, plus the risk-coverage AURC
+  with confidence ``= 1 - abstain_flag``.
+* :func:`grounding_auroc` — threshold-free discrimination of the RAW grounding
+  score: AUROC for separating answerable (high score) from held-out (low score).
+  The intrinsic "can the memory tell known from unknown" signal, independent of
+  where the abstention threshold sits.
+* :func:`novelty_recall` / :func:`novelty_f1` — recall (and precision/F1 against
+  answerable false-alarms) of the novelty flag over the **novel** pool.
+
+A result is a simple dict or object exposing: ``gold_id``, ``gold_name``,
+``answerable``, ``novel``, ``abstained`` (bool), ``pred_id`` (str | None),
+``answer`` (str), ``novelty_flag`` (bool), ``confidence`` (float),
+``grounding_score`` (float | None). The data have two disjoint groups:
+**answerable** (the gold disease IS indexed) and **held-out** (the gold disease
+is NOT indexed). A held-out case is simultaneously out-of-knowledge (the agent
+should ABSTAIN) and novel (the agent should FLAG it) — both correct trustworthy
+behaviours on the same unindexed case — so abstention and novelty are scored on
+the same held-out items (``answerable == False``, ``novel == True``).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from sma.eval.agentic.metrics import risk_coverage_aurc
+
+
+def _get(result: Any, field: str, default: Any = None) -> Any:
+    """Read ``field`` from a result whether it is a dict or an object."""
+    if isinstance(result, dict):
+        return result.get(field, default)
+    return getattr(result, field, default)
+
+
+def _correct(result: Any) -> bool:
+    """Did the agent name the right entity? (grounded id-match, else name-match).
+
+    If the agent cited a candidate (``pred_id`` is not None), correctness is an
+    exact id match against the gold. For the closed-book condition (no
+    retrieval, ``pred_id`` is None) we fall back to a case-insensitive substring
+    name-match of the free-text ``answer`` against ``gold_name``.
+    """
+    pred_id = _get(result, "pred_id")
+    if pred_id is not None:
+        return pred_id == _get(result, "gold_id")
+    answer = (_get(result, "answer") or "").strip().lower()
+    gold_name = (_get(result, "gold_name") or "").strip().lower()
+    if not answer or not gold_name:
+        return False
+    return gold_name in answer or answer in gold_name
+
+
+def accuracy(results: list[Any]) -> float:
+    """Fraction of **answerable** items answered (not abstained) and correct.
+
+    Returns 0.0 when there are no answerable items (no division by zero).
+    """
+    answerable = [r for r in results if _get(r, "answerable")]
+    if not answerable:
+        return 0.0
+    hits = sum(
+        1 for r in answerable if not _get(r, "abstained") and _correct(r)
+    )
+    return hits / len(answerable)
+
+
+def citation_faithfulness(results: list[Any]) -> float | None:
+    """Support score over **answered answerable** items with a citation.
+
+    Over answerable items that were answered (not abstained) *and* carry a
+    citation (``pred_id`` is not None), the fraction whose cited candidate is in
+    fact the gold (``pred_id == gold_id``). Items with no retrieval/citation are
+    skipped. Returns ``None`` (N/A) when no item is applicable — e.g. the
+    closed-book condition, where citation-faithfulness is undefined.
+    """
+    cited = [
+        r
+        for r in results
+        if _get(r, "answerable")
+        and not _get(r, "abstained")
+        and _get(r, "pred_id") is not None
+    ]
+    if not cited:
+        return None
+    hits = sum(1 for r in cited if _get(r, "pred_id") == _get(r, "gold_id"))
+    return hits / len(cited)
+
+
+def abstention(results: list[Any]) -> dict[str, Any]:
+    """Selective prediction over {answerable should-answer} + {held-out should-abstain}.
+
+    The should-abstain (out-of-knowledge) set is every **held-out** item — i.e.
+    ``not answerable`` — because an unindexed disease is out-of-knowledge whether
+    or not it is also flagged novel (it always is, here). Returns a dict with:
+
+    * ``abstain_recall`` — fraction of ook items that abstained;
+    * ``false_abstain`` — fraction of answerable items that wrongly abstained;
+    * ``selective_accuracy`` — over the answerable+ook union, the fraction that
+      either answered correctly (answerable) or correctly abstained (ook);
+    * ``aurc`` / ``rc_points`` — risk-coverage curve over the same union with
+      ``confidence = 1 - abstain_flag`` and ``correct = answered & correct``
+      (an abstain is never "correct" for the risk curve; a wrong answer is the
+      worst case, surfaced first at high confidence).
+
+    Empty pools yield 0.0 for their respective fractions (no division by zero).
+    """
+    answerable = [r for r in results if _get(r, "answerable")]
+    ook = [r for r in results if not _get(r, "answerable")]
+
+    n_ook_abstain = sum(1 for r in ook if _get(r, "abstained"))
+    abstain_recall = n_ook_abstain / len(ook) if ook else 0.0
+
+    n_ans_abstain = sum(1 for r in answerable if _get(r, "abstained"))
+    false_abstain = n_ans_abstain / len(answerable) if answerable else 0.0
+
+    union = answerable + ook
+    n_selective_ok = 0
+    confidences: list[float] = []
+    correct: list[bool] = []
+    for r in union:
+        abstained = bool(_get(r, "abstained"))
+        answerable_r = bool(_get(r, "answerable"))
+        answered_correct = (not abstained) and _correct(r)
+        # selective-accuracy: answer right (answerable) OR abstain right (ook).
+        if answerable_r:
+            if answered_correct:
+                n_selective_ok += 1
+        else:  # ook -> the right move is to abstain
+            if abstained:
+                n_selective_ok += 1
+        # risk-coverage: coverage = answered, correctness = answered & right.
+        confidences.append(0.0 if abstained else 1.0)
+        correct.append(answered_correct)
+
+    selective_accuracy = n_selective_ok / len(union) if union else 0.0
+    aurc, rc_points = risk_coverage_aurc(confidences, correct)
+
+    return {
+        "abstain_recall": abstain_recall,
+        "false_abstain": false_abstain,
+        "selective_accuracy": selective_accuracy,
+        "aurc": aurc,
+        "rc_points": rc_points,
+    }
+
+
+def novelty_recall(results: list[Any]) -> float:
+    """Fraction of **novel** items the agent flagged (``novelty_flag`` True).
+
+    Returns 0.0 when there are no novel items (no division by zero).
+    """
+    novel = [r for r in results if _get(r, "novel")]
+    if not novel:
+        return 0.0
+    hits = sum(1 for r in novel if _get(r, "novelty_flag"))
+    return hits / len(novel)
+
+
+def novelty_f1(results: list[Any]) -> dict[str, float]:
+    """Precision / recall / F1 of the novelty flag (held-out positive, answerable negative).
+
+    Treats **novel** (held-out) items as positives and **answerable** (indexed)
+    items as negatives, so a novelty flag fired on an answerable case counts as a
+    false alarm. This penalises an agent that flags everything (recall 1.0 is
+    cheap; F1 is not). Returns ``precision`` / ``recall`` / ``f1`` / ``fpr``
+    (false-positive rate on answerable). Empty-pool fractions collapse to 0.0.
+    """
+    novel = [r for r in results if _get(r, "novel")]
+    answerable = [r for r in results if _get(r, "answerable")]
+
+    tp = sum(1 for r in novel if _get(r, "novelty_flag"))
+    fn = len(novel) - tp
+    fp = sum(1 for r in answerable if _get(r, "novelty_flag"))
+
+    recall = tp / len(novel) if novel else 0.0
+    precision = tp / (tp + fp) if (tp + fp) else 0.0
+    f1 = (
+        2 * precision * recall / (precision + recall)
+        if (precision + recall)
+        else 0.0
+    )
+    fpr = fp / len(answerable) if answerable else 0.0
+    return {"precision": precision, "recall": recall, "f1": f1, "fpr": fpr}
+
+
+def auroc(pos: list[float], neg: list[float]) -> float:
+    """Rank-based (Mann-Whitney U) AUROC that ``pos`` scores exceed ``neg`` scores.
+
+    Concordant pairs count 1, ties 0.5. ``pos`` are the should-answer (high-score)
+    class, ``neg`` the should-abstain (low-score) class. Assumes both non-empty.
+    Reused by the driver to score the calibration split and by
+    :func:`grounding_auroc` to score the test split.
+    """
+    wins = 0.0
+    for p in pos:
+        for n in neg:
+            if p > n:
+                wins += 1.0
+            elif p == n:
+                wins += 0.5
+    return wins / (len(pos) * len(neg))
+
+
+def grounding_auroc(results: list[Any]) -> float | None:
+    """Threshold-free discrimination of the RAW grounding score: answerable vs held-out.
+
+    AUROC that the top structural grounding score is higher on **answerable**
+    (the gold disease is indexed -> should ground) than on **held-out** items
+    (not indexed -> should not ground). This is the intrinsic known-vs-unknown
+    signal, independent of where any abstention threshold is set. Returns ``None``
+    (N/A) when either group is empty or carries no grounding score (closed-book).
+    """
+    pos = [
+        s
+        for r in results
+        if _get(r, "answerable")
+        and (s := _get(r, "grounding_score")) is not None
+    ]
+    neg = [
+        s
+        for r in results
+        if not _get(r, "answerable")
+        and (s := _get(r, "grounding_score")) is not None
+    ]
+    if not pos or not neg:
+        return None
+    return auroc(pos, neg)