evidence-graph 0.1.1__py3-none-any.whl

evidence_graph/__init__.py

@@ -0,0 +1,198 @@
+"""evidence-graph: knowledge graphs whose edges carry their evidence, or abstain.
+
+The library takes retrieved passages with provenance (`RetrievalHit`) and produces structure that a
+scientist can check: typed edges between documents, recorded disagreements, a tamper-evident
+derivation record, and a FAIR-packaged export.
+
+It never opens a socket, never reads a credential, and has no runtime dependencies. Anything that
+needs a model is injected by the caller.
+
+Layers, roughly in the order you would use them:
+
+    evidence          RetrievalHit — a passage plus the provenance needed to cite it
+    biolink           Biolink Model categories, predicates, and CURIE typing
+    grounding         resolve a surface string to a biomedical CURIE, or refuse
+    ontology          type a concept term against a small built-in vocabulary
+    relations         propose typed document→document edges, each carrying its evidence
+    conflicts         detect numeric and documented disagreement between sources
+    adjudicate        assert / abstain / conflict per candidate edge, over a scorer Protocol
+    maintain          re-run without overwriting: revision history per edge
+    evaluation        edge-level precision as a function of abstention (risk-coverage)
+    graph             assemble nodes and edges into a graph payload
+    provenance        W3C PROV-O derivation records with SHA-256 tamper-evidence
+    kgx               KGX interchange: Biolink-compatible TSV and JSON Lines
+    fair              DataCite report, Dublin Core XML, RO-Crate 1.1 JSON-LD
+    ro_crate_validation   structural validation of an RO-Crate against the 1.1 spec
+
+Status: 0.1.1. The API will change. See CONTRIBUTING.md for what belongs here.
+"""
+
+from __future__ import annotations
+
+from evidence_graph.adjudicate import (
+    DEFAULT_ASSERT_AT,
+    DEFAULT_SCORER,
+    Adjudication,
+    AdjudicationResult,
+    EdgeScore,
+    EdgeScorer,
+    GroundingScorer,
+    Verdict,
+    adjudicate,
+    edge_key,
+    open_questions,
+)
+from evidence_graph.biolink import (
+    BIOLINK_VERSION,
+    CATEGORIES,
+    DEFAULT_CATEGORY,
+    category_chain,
+    category_for_curie,
+    category_for_prefix,
+)
+from evidence_graph.conflicts import (
+    detect_conflict,
+    detect_documented_conflict,
+    detect_numeric_conflict,
+    detect_numeric_conflict_any,
+)
+from evidence_graph.evaluation import (
+    CurvePoint,
+    EvaluationReport,
+    baseline_precision,
+    risk_coverage,
+    risk_coverage_svg,
+)
+from evidence_graph.evidence import RetrievalHit
+from evidence_graph.fair import (
+    DEFAULT_PRODUCER,
+    Producer,
+    dublin_core_xml,
+    fair_record,
+    fair_report,
+    render_dublin_core,
+    ro_crate,
+)
+from evidence_graph.graph import build_project_graph
+from evidence_graph.grounding import (
+    Concept,
+    Grounding,
+    GroundingReport,
+    Vocabulary,
+    ground,
+    ground_all,
+    load_vocabulary,
+    normalize_term,
+)
+from evidence_graph.kgx import (
+    KgxGraph,
+    from_adjudications,
+    to_jsonl,
+    to_tsv,
+    validate_kgx,
+)
+from evidence_graph.maintain import (
+    EdgeHistory,
+    EdgeStatus,
+    History,
+    Revision,
+    changelog,
+    contested_edges,
+    evidence_fingerprint,
+    live_edges,
+    reconcile,
+)
+from evidence_graph.ontology import ONTOLOGY_TYPES, classify_concept
+from evidence_graph.provenance import (
+    PROV_SCHEMA_VERSION,
+    ProvenanceEntry,
+    answer_lineage,
+    compute_checksum,
+    finalize,
+    lineage_report,
+    prov_jsonld,
+    prov_jsonld_nodes,
+    trace_lineage,
+    verify_checksum,
+    verify_lineage,
+)
+from evidence_graph.relations import extract_typed_relations, normalize_entity
+from evidence_graph.ro_crate_validation import validate_ro_crate
+
+__version__ = "0.1.1"
+
+__all__ = [
+    "Adjudication",
+    "AdjudicationResult",
+    "BIOLINK_VERSION",
+    "CATEGORIES",
+    "Concept",
+    "CurvePoint",
+    "DEFAULT_ASSERT_AT",
+    "DEFAULT_CATEGORY",
+    "DEFAULT_PRODUCER",
+    "DEFAULT_SCORER",
+    "EdgeHistory",
+    "EdgeScore",
+    "EdgeScorer",
+    "EdgeStatus",
+    "EvaluationReport",
+    "Grounding",
+    "GroundingReport",
+    "GroundingScorer",
+    "History",
+    "KgxGraph",
+    "ONTOLOGY_TYPES",
+    "PROV_SCHEMA_VERSION",
+    "Producer",
+    "ProvenanceEntry",
+    "RetrievalHit",
+    "Revision",
+    "Verdict",
+    "Vocabulary",
+    "adjudicate",
+    "answer_lineage",
+    "baseline_precision",
+    "build_project_graph",
+    "category_chain",
+    "category_for_curie",
+    "category_for_prefix",
+    "changelog",
+    "classify_concept",
+    "compute_checksum",
+    "contested_edges",
+    "detect_conflict",
+    "detect_documented_conflict",
+    "detect_numeric_conflict",
+    "detect_numeric_conflict_any",
+    "dublin_core_xml",
+    "edge_key",
+    "evidence_fingerprint",
+    "extract_typed_relations",
+    "fair_record",
+    "fair_report",
+    "finalize",
+    "from_adjudications",
+    "ground",
+    "ground_all",
+    "lineage_report",
+    "live_edges",
+    "load_vocabulary",
+    "normalize_entity",
+    "normalize_term",
+    "open_questions",
+    "prov_jsonld",
+    "prov_jsonld_nodes",
+    "reconcile",
+    "render_dublin_core",
+    "risk_coverage",
+    "risk_coverage_svg",
+    "ro_crate",
+    "to_jsonl",
+    "to_tsv",
+    "trace_lineage",
+    "validate_kgx",
+    "validate_ro_crate",
+    "verify_checksum",
+    "verify_lineage",
+]

evidence_graph/adjudicate.py

@@ -0,0 +1,399 @@
+"""Assert, abstain, or record a conflict — the verdict step between proposing an edge and keeping it.
+
+A proposer (`relations.extract_typed_relations`, or a model) is optimistic by construction: it emits
+every edge its patterns fire on. That is the right behaviour for a proposer and the wrong behaviour
+for a graph, because a plausible edge and a supported edge are indistinguishable once they are in
+the store. This module is the gate in between.
+
+Every candidate gets one of three verdicts:
+
+  ASSERT    the evidence carries the claim; the edge goes into the graph.
+  ABSTAIN   the evidence does not reach; the edge is kept as an OPEN QUESTION with its evidence
+            attached, so a curator can confirm or reject it. It is never silently dropped — an
+            abstention is a finding, and discarding it throws away the only record that the system
+            looked and was not convinced.
+  CONFLICT  the candidate set disagrees with itself about this pair of documents (one edge says
+            they agree on a quantity, another says they disagree). Both are kept, side by side,
+            rather than resolved by picking the higher score.
+
+**Scoring is pluggable and the default is keyless.** `EdgeScorer` is a Protocol: given an edge and
+its evidence, return a score and a rationale. The default `GroundingScorer` runs no model. It asks a
+narrower question than "is this edge true" — it asks **"is this edge actually grounded in the text
+it cites?"**: does the evidence exist, does each quote really contain the trigger the proposer
+claims fired, do both endpoints carry evidence, does the quote show the cue the edge type implies.
+
+That distinction is the honest part. Verbatim grounding is necessary and not sufficient for
+correctness: a perfectly grounded edge can still be a wrong reading of the sentence. An LLM scorer
+supplied by the caller adds semantic judgement on top. The library defines the seam and implements
+the half that needs no key, so the pipeline degrades to a weaker but still correct graph offline.
+
+`assert_at` is the abstention dial. Raising it asserts fewer edges at higher precision; the
+trade-off it makes is measurable with `evidence_graph.evaluation.risk_coverage`.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any, Protocol
+
+# Default score floor for assertion, on the 0-10 scale every scorer reports. 6.0 sits above
+# "evidence exists and is well formed" (which the grounding scorer alone can reach) and below
+# "every check passed", so a structurally sound but weakly cued edge abstains rather than asserts.
+DEFAULT_ASSERT_AT = 6.0
+
+# Edge types that make incompatible claims about the same pair of documents and quantity. If a
+# proposer emits both, the truth is not "whichever scored higher" — it is that the corpus disagrees.
+INCOMPATIBLE_TYPES = frozenset({"supports", "contradicts"})
+
+# Edge types whose claim is a COMPARISON between two documents, and which therefore have to cite
+# evidence from both. `decided` and `uses-protocol` are one-sided by construction — the deciding
+# document is the only one that records the decision — so demanding evidence from both ends
+# penalises them for being correctly formed. Found on a real corpus, where it accounted for 4433
+# abstentions.
+TWO_SIDED_TYPES = frozenset({"supports", "contradicts"})
+
+# Surface cues an edge type implies. Used by the grounding scorer to check that the quote shows
+# what the edge claims, not merely that a quote exists.
+_TYPE_CUES: dict[str, re.Pattern[str]] = {
+    "decided": re.compile(
+        r"\b(?:decide[sd]?|decision|chose|choose|selected?|opt(?:ed|s)?\s+for|conclude[sd]?|"
+        r"recommend(?:s|ed)?|approve[sd]?|finali[sz]ed?|went\s+with|settled\s+on)\b",
+        re.IGNORECASE,
+    ),
+    "uses-protocol": re.compile(r"\b(?:sop|protocol|s\.?o\.?p\.?)\b", re.IGNORECASE),
+}
+
+
+class Verdict(StrEnum):
+    """What the adjudicator concluded about one candidate edge."""
+
+    ASSERT = "assert"
+    ABSTAIN = "abstain"
+    CONFLICT = "conflict"
+
+
+@dataclass(frozen=True)
+class EdgeScore:
+    """A scorer's judgement of one candidate edge.
+
+    score: 0-10. Scorers must use this scale so thresholds and risk-coverage curves are comparable
+        between a deterministic scorer and a model.
+    rationale: why, in one short human-readable line. This is shown to the curator reviewing an
+        abstention, so "no evidence carries the trigger 'ph=5.5'" beats "score 2.0".
+    checks: per-check detail, for a scorer that runs several. Free-form and advisory.
+    """
+
+    score: float
+    rationale: str
+    checks: dict[str, bool] = field(default_factory=dict)
+
+
+class EdgeScorer(Protocol):
+    """How the caller supplies judgement.
+
+    An implementation receives the candidate edge (the shape `relations.extract_typed_relations`
+    emits) and returns an `EdgeScore`. It must be deterministic for a given input if the caller
+    wants reproducible graphs; a model-backed scorer usually is not, which is why the library
+    default is not one.
+    """
+
+    def __call__(self, edge: dict[str, Any]) -> EdgeScore: ...
+
+
+@dataclass(frozen=True)
+class Adjudication:
+    """One candidate edge plus the verdict on it, and everything needed to explain the verdict."""
+
+    edge: dict[str, Any]
+    verdict: Verdict
+    score: float
+    rationale: str
+    checks: dict[str, bool] = field(default_factory=dict)
+    # Populated for CONFLICT: the other adjudications this one disagrees with.
+    conflicts_with: tuple[str, ...] = ()
+
+    @property
+    def key(self) -> str:
+        """Stable identity for this edge: source, type, target, quantity."""
+        return edge_key(self.edge)
+
+    @property
+    def asserted(self) -> bool:
+        return self.verdict is Verdict.ASSERT
+
+
+@dataclass(frozen=True)
+class AdjudicationResult:
+    """The outcome of adjudicating a candidate set, partitioned by verdict.
+
+    `abstained` is deliberately as prominent as `asserted`. It is the review queue, and a pipeline
+    that ignores it is throwing away the system's own record of what it could not establish.
+    """
+
+    adjudications: tuple[Adjudication, ...]
+
+    @property
+    def asserted(self) -> tuple[Adjudication, ...]:
+        return tuple(a for a in self.adjudications if a.verdict is Verdict.ASSERT)
+
+    @property
+    def abstained(self) -> tuple[Adjudication, ...]:
+        return tuple(a for a in self.adjudications if a.verdict is Verdict.ABSTAIN)
+
+    @property
+    def conflicted(self) -> tuple[Adjudication, ...]:
+        return tuple(a for a in self.adjudications if a.verdict is Verdict.CONFLICT)
+
+    @property
+    def coverage(self) -> float:
+        """Share of candidates that were asserted. The x-axis of a risk-coverage curve."""
+        if not self.adjudications:
+            return 0.0
+        return len(self.asserted) / len(self.adjudications)
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "candidates": len(self.adjudications),
+            "asserted": len(self.asserted),
+            "abstained": len(self.abstained),
+            "conflicted": len(self.conflicted),
+            "coverage": round(self.coverage, 4),
+        }
+
+
+def edge_key(edge: dict[str, Any]) -> str:
+    """Stable identity for an edge: `source|type|target|quantity`."""
+    return "|".join(
+        str(edge.get(k, "")) for k in ("source", "type", "target", "quantity")
+    )
+
+
+def _pair_key(edge: dict[str, Any]) -> tuple[str, str, str]:
+    """Identity of the *claim* an edge makes, ignoring its direction and its type.
+
+    `supports` and `contradicts` between the same two documents about the same quantity are claims
+    about the same thing, so they collide here on purpose — that collision is how a conflict is
+    found. Direction is dropped because "A agrees with B" and "B agrees with A" are one claim.
+    """
+    a, b = str(edge.get("source", "")), str(edge.get("target", ""))
+    lo, hi = (a, b) if a <= b else (b, a)
+    return (lo, hi, str(edge.get("quantity", "")))
+
+
+# --------------------------------------------------------------------------------------------
+# The default scorer: grounding, not truth
+# --------------------------------------------------------------------------------------------
+
+
+# A proposer's `trigger` field is overloaded: sometimes it is a checkable claim about the quote,
+# sometimes a descriptive note about why the edge fired. Only the first kind can be verified, and
+# treating the second kind as verifiable fails every edge of that type — which is exactly what
+# happened on the first real corpus, where 100% of abstentions blamed an unverifiable trigger.
+#
+#   ph=5.5             -> checkable: the VALUE must appear in the quote.
+#   supports re: mAb-X -> checkable: the entity must appear in the quote.
+#   mentions mAb-X     -> checkable: likewise.
+#   decision on ph     -> a note. Not checkable, and not evidence of anything wrong.
+#   protocol reference -> a note.
+#
+# Note that only the value is required, never the label: labels are normalised (`concentration_molar`)
+# and by design do not appear in prose.
+_TRIGGER_MEASUREMENT = re.compile(r"^[\w .+/-]+=(?P<value>.+)$")
+_TRIGGER_ENTITY = re.compile(r"(?:\bre:|^mentions)\s+(?P<entity>.+)$", re.IGNORECASE)
+
+
+def _checkable_terms(trigger: str) -> list[str] | None:
+    """What must appear verbatim in the quote for this trigger, or None if it is a note."""
+    trigger = (trigger or "").strip()
+    if not trigger:
+        return None
+    match = _TRIGGER_MEASUREMENT.match(trigger)
+    if match:
+        return [match.group("value").strip()]
+    match = _TRIGGER_ENTITY.search(trigger)
+    if match:
+        return [match.group("entity").strip()]
+    return None
+
+
+class GroundingScorer:
+    """Deterministic scorer: is this edge grounded in the text it cites?
+
+    Runs no model and needs no key. It checks what can be checked by reading the evidence against
+    the claim, and it is deliberately clear about what that does and does not establish — see the
+    module docstring. Each check contributes to a 0-10 score:
+
+      has_evidence        the edge cites at least one passage at all                    (hard gate)
+      evidence_complete   every cited passage carries a quote and a locator
+      trigger_verbatim    the value the proposer says fired is actually in the quote (only
+                          checked when the trigger makes a checkable claim — see `_checkable_terms`)
+      both_endpoints      a COMPARING edge (supports/contradicts) cites evidence from both
+                          documents; not applied to one-sided types, see TWO_SIDED_TYPES
+      type_cue_present    the quote shows the cue the edge type implies (decided, uses-protocol)
+
+    `trigger_verbatim` is the one that catches real defects: a proposer that reports `ph=5.5` from a
+    passage that does not contain 5.5 has mis-parsed, and the resulting edge is unciteable.
+    """
+
+    #: Weight per check. `has_evidence` is a gate rather than a weight — without it the score is 0.
+    WEIGHTS = {
+        "evidence_complete": 2.0,
+        "trigger_verbatim": 4.0,
+        "both_endpoints": 2.5,
+        "type_cue_present": 1.5,
+    }
+
+    def __call__(self, edge: dict[str, Any]) -> EdgeScore:
+        evidence = edge.get("evidence") or []
+        if not evidence:
+            return EdgeScore(0.0, "no evidence cited", {"has_evidence": False})
+
+        checks: dict[str, bool] = {"has_evidence": True}
+        reasons: list[str] = []
+
+        checks["evidence_complete"] = all(
+            (ev.get("quote") or "").strip() and (ev.get("chunk_id") or ev.get("source_uri"))
+            for ev in evidence
+        )
+        if not checks["evidence_complete"]:
+            reasons.append("a cited passage is missing its quote or locator")
+
+        def grounded(ev: dict[str, Any]) -> bool:
+            terms = _checkable_terms(ev.get("trigger", ""))
+            quote = (ev.get("quote") or "").lower()
+            return all(term.lower() in quote for term in terms) if terms else True
+
+        checkable = [ev for ev in evidence if _checkable_terms(ev.get("trigger", "")) is not None]
+        if checkable:
+            checks["trigger_verbatim"] = all(grounded(ev) for ev in checkable)
+            if not checks["trigger_verbatim"]:
+                bad = next(ev for ev in checkable if not grounded(ev))
+                reasons.append(f"quote does not contain the trigger {bad['trigger']!r}")
+        # When no trigger makes a checkable claim there is nothing to verify. The check is left out
+        # of `checks` entirely, so it neither earns credit nor costs any — an edge is not penalised
+        # for a proposer that reported a note instead of a value.
+
+        if str(edge.get("type", "")) in TWO_SIDED_TYPES:
+            endpoints = {str(edge.get("source", "")), str(edge.get("target", ""))}
+            cited = {str(ev.get("artifact_id", "")) for ev in evidence}
+            checks["both_endpoints"] = endpoints.issubset(cited)
+            if not checks["both_endpoints"]:
+                missing = sorted(endpoints - cited)
+                reasons.append(f"no evidence cited from {', '.join(missing)}")
+
+        cue = _TYPE_CUES.get(str(edge.get("type", "")))
+        if cue is not None:
+            checks["type_cue_present"] = any(cue.search(ev.get("quote") or "") for ev in evidence)
+            if not checks["type_cue_present"]:
+                reasons.append(f"no {edge['type']} cue in any cited quote")
+
+        earned = sum(self.WEIGHTS[name] for name, ok in checks.items() if ok and name in self.WEIGHTS)
+        available = sum(self.WEIGHTS[name] for name in checks if name in self.WEIGHTS)
+        score = round(10.0 * earned / available, 2) if available else 0.0
+
+        rationale = "; ".join(reasons) if reasons else "evidence carries the claim verbatim"
+        return EdgeScore(score, rationale, checks)
+
+
+#: The library default. Keyless, deterministic, and honest about scoring grounding rather than truth.
+DEFAULT_SCORER: EdgeScorer = GroundingScorer()
+
+
+# --------------------------------------------------------------------------------------------
+# Adjudication
+# --------------------------------------------------------------------------------------------
+
+
+def adjudicate(
+    edges: list[dict[str, Any]],
+    *,
+    scorer: EdgeScorer | None = None,
+    assert_at: float = DEFAULT_ASSERT_AT,
+) -> AdjudicationResult:
+    """Score every candidate edge and return a verdict for each.
+
+    Order of resolution matters. Conflicts are found FIRST, across the whole candidate set, because
+    a conflict is a property of the set rather than of any single edge: two edges that make
+    incompatible claims about the same document pair are both kept and marked, instead of one of
+    them winning on score. Only the remaining edges face the `assert_at` threshold.
+
+    Deterministic given a deterministic scorer: candidates keep their input order, and nothing here
+    depends on iteration order of a set.
+    """
+    scorer = scorer or DEFAULT_SCORER
+    scored = [(edge, scorer(edge)) for edge in edges]
+
+    # Which claims have incompatible candidates? Group by the claim, not the edge.
+    by_claim: dict[tuple[str, str, str], set[str]] = {}
+    for edge, _ in scored:
+        etype = str(edge.get("type", ""))
+        if etype in INCOMPATIBLE_TYPES:
+            by_claim.setdefault(_pair_key(edge), set()).add(etype)
+    contested = {claim for claim, types in by_claim.items() if len(types & INCOMPATIBLE_TYPES) > 1}
+
+    out: list[Adjudication] = []
+    for edge, result in scored:
+        claim = _pair_key(edge)
+        if claim in contested and str(edge.get("type", "")) in INCOMPATIBLE_TYPES:
+            others = tuple(
+                sorted(
+                    edge_key(other)
+                    for other, _ in scored
+                    if _pair_key(other) == claim
+                    and str(other.get("type", "")) in INCOMPATIBLE_TYPES
+                    and edge_key(other) != edge_key(edge)
+                )
+            )
+            out.append(
+                Adjudication(
+                    edge=edge,
+                    verdict=Verdict.CONFLICT,
+                    score=result.score,
+                    rationale=(
+                        "the candidate set disagrees about this pair: "
+                        + " and ".join(sorted(by_claim[claim]))
+                    ),
+                    checks=result.checks,
+                    conflicts_with=others,
+                )
+            )
+            continue
+
+        verdict = Verdict.ASSERT if result.score >= assert_at else Verdict.ABSTAIN
+        rationale = result.rationale
+        if verdict is Verdict.ABSTAIN and not rationale.startswith("no evidence"):
+            rationale = f"{rationale} (score {result.score} below {assert_at})"
+        out.append(
+            Adjudication(
+                edge=edge,
+                verdict=verdict,
+                score=result.score,
+                rationale=rationale,
+                checks=result.checks,
+            )
+        )
+
+    return AdjudicationResult(tuple(out))
+
+
+def open_questions(result: AdjudicationResult) -> list[dict[str, Any]]:
+    """Render the abstentions as a curator's review queue.
+
+    Each entry carries the claim, why the adjudicator was not convinced, and the evidence it looked
+    at, so a human can settle it without going back to the corpus themselves.
+    """
+    return [
+        {
+            "key": a.key,
+            "source": a.edge.get("source"),
+            "target": a.edge.get("target"),
+            "type": a.edge.get("type"),
+            "quantity": a.edge.get("quantity"),
+            "score": a.score,
+            "why_not_asserted": a.rationale,
+            "evidence": a.edge.get("evidence") or [],
+        }
+        for a in result.abstained
+    ]