sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/rag.py

@@ -0,0 +1,329 @@
+"""The public, illustrative ``rag.rerank.v1`` frozen-pool pack.
+
+The packaged JSON is byte-for-byte the product-neutral fixture from the local
+``rag-evals/fixtures/dataset-v2.json`` source pinned in T-001. The validation
+rules below port its ``src/dataset-v2.ts`` dataset checks; that source is not a
+runtime dependency. Reference labels are separate from :class:`FrozenPoolCase`
+and must never be passed to a candidate process.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from functools import lru_cache
+from importlib import resources
+from typing import Any, Mapping
+
+from .contracts import (
+    CASE_SCHEMA_VERSION,
+    PACK_ID,
+    CaseReferences,
+    ContractError,
+    CorpusVersions,
+    EvidenceGroup,
+    FrozenCandidate,
+    FrozenPoolCase,
+    ReferenceCandidate,
+    _finite_number,
+    _id,
+    _text,
+)
+from .packs import PackDescriptor, get_pack
+
+FIXTURE_SHA256 = "74884a0176738649d85cc8b681323103961939aaa0f3d5f1dd1565ed2e53d86b"
+_FIXTURE_NAME = "rag-rerank-v1.json"
+_SEVERITIES = frozenset({"low", "medium", "high", "critical"})
+_CATEGORIES = frozenset(
+    {
+        "straightforward_factual", "semantic_paraphrase", "exact_product_feature_name",
+        "competitor_named_entity", "typo_misspelling", "acronym_abbreviation",
+        "short_underspecified", "long_conversational", "conversational_follow_up",
+        "ambiguous", "multi_part", "multi_document", "unsupported_not_in_kb",
+        "off_topic", "false_premise", "contradictory_sources", "stale_current_conflict",
+        "long_passage_truncation", "adjacent_chunk_dependency", "duplicate_documentation",
+        "presales_language", "postsales_language", "customer_specific_terminology",
+    }
+)
+
+
+@dataclass(frozen=True)
+class RagRerankPack:
+    descriptor: PackDescriptor
+    cases: tuple[FrozenPoolCase, ...]
+    references: tuple[CaseReferences, ...]
+    metric_release: str
+    dataset_digest: str
+    relevance_threshold: int
+
+    def reference_for(self, case_id: str) -> CaseReferences:
+        for reference in self.references:
+            if reference.case_id == case_id:
+                return reference
+        raise ContractError(f"No reference for case {case_id!r}")
+
+
+def _mapping(value: object, path: str) -> Mapping[str, Any]:
+    if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value):
+        raise ContractError(f"{path} must be a JSON object")
+    return value
+
+
+def _list(value: object, path: str, *, nonempty: bool = False) -> list[Any]:
+    if not isinstance(value, list) or (nonempty and not value):
+        raise ContractError(f"{path} must be a {'nonempty ' if nonempty else ''}JSON array")
+    return value
+
+
+def _unique_ids(value: object, path: str, *, nonempty: bool = False) -> tuple[str, ...]:
+    values = tuple(_id(item, f"{path}[{index}]") for index, item in enumerate(_list(value, path, nonempty=nonempty)))
+    if len(set(values)) != len(values):
+        raise ContractError(f"{path} must contain unique identifiers")
+    return values
+
+
+def _optional_text(value: Mapping[str, Any], key: str, path: str) -> None:
+    if key in value:
+        _text(value[key], f"{path}.{key}")
+
+
+def _validate_candidate_annotations(raw: Mapping[str, Any], path: str) -> None:
+    if "known_hard_negative" in raw and type(raw["known_hard_negative"]) is not bool:
+        raise ContractError(f"{path}.known_hard_negative must be boolean")
+    _optional_text(raw, "annotator_rationale", path)
+    if "evidence_spans" in raw:
+        for index, item in enumerate(_list(raw["evidence_spans"], f"{path}.evidence_spans")):
+            span = _mapping(item, f"{path}.evidence_spans[{index}]")
+            start, end = span.get("start"), span.get("end")
+            if type(start) is not int or type(end) is not int or start < 0 or end <= start or end > len(raw["input"]):
+                raise ContractError(f"{path}.evidence_spans[{index}] has invalid bounds")
+            if "text" in span and span["text"] != raw["input"][start:end]:
+                raise ContractError(f"{path}.evidence_spans[{index}].text must match the exact passage span")
+    if "annotations" in raw:
+        annotations = _list(raw["annotations"], f"{path}.annotations", nonempty=True)
+        annotators: set[str] = set()
+        for index, item in enumerate(annotations):
+            annotation = _mapping(item, f"{path}.annotations[{index}]")
+            annotator = _id(annotation.get("annotator_id"), f"{path}.annotations[{index}].annotator_id")
+            if annotator in annotators:
+                raise ContractError(f"{path}.annotations has duplicate annotator {annotator!r}")
+            annotators.add(annotator)
+            grade = annotation.get("relevance_grade")
+            if type(grade) is not int or not 0 <= grade <= 3:
+                raise ContractError(f"{path}.annotations[{index}].relevance_grade must be 0..3")
+            _optional_text(annotation, "rationale", f"{path}.annotations[{index}]")
+            if "created_at" in annotation:
+                timestamp = annotation["created_at"]
+                try:
+                    datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+                except (AttributeError, ValueError) as exc:
+                    raise ContractError(f"{path}.annotations[{index}].created_at must be a timestamp") from exc
+    if "adjudication" in raw:
+        adjudication = _mapping(raw["adjudication"], f"{path}.adjudication")
+        _id(adjudication.get("adjudicator_id"), f"{path}.adjudication.adjudicator_id")
+        if adjudication.get("relevance_grade") != raw.get("relevance_grade"):
+            raise ContractError(f"{path}.adjudication.relevance_grade must match final grade")
+        _text(adjudication.get("rationale"), f"{path}.adjudication.rationale")
+
+
+def _validate_task(raw_value: object, index: int, dataset_id: str, dataset_version: str, label_authority: str) -> tuple[FrozenPoolCase, CaseReferences]:
+    path = f"tasks[{index}]"
+    raw = _mapping(raw_value, path)
+    case_id = _id(raw.get("task_id"), f"{path}.task_id")
+    query_id = _id(raw.get("query_id"), f"{path}.query_id")
+    cluster_id = _id(raw.get("cluster_id"), f"{path}.cluster_id")
+    split_group_id = _id(raw.get("split_group_id"), f"{path}.split_group_id")
+    _id(raw.get("domain"), f"{path}.domain")
+    query = _text(raw.get("query"), f"{path}.query")
+    answerability = raw.get("answerability")
+    if answerability not in ("answerable", "unsupported"):
+        raise ContractError(f"{path}.answerability must be answerable or unsupported")
+    if not isinstance(raw.get("business_severity"), str) or raw["business_severity"] not in _SEVERITIES:
+        raise ContractError(f"{path}.business_severity is invalid")
+    categories = _unique_ids(raw.get("query_categories"), f"{path}.query_categories", nonempty=True)
+    if any(category not in _CATEGORIES for category in categories):
+        raise ContractError(f"{path}.query_categories contains an unsupported category")
+    if "query_variants" in raw:
+        variants = _list(raw["query_variants"], f"{path}.query_variants", nonempty=True)
+        for variant_index, variant in enumerate(variants):
+            _text(variant, f"{path}.query_variants[{variant_index}]")
+        if len(set(variants)) != len(variants):
+            raise ContractError(f"{path}.query_variants must be unique")
+    _optional_text(raw, "source_case_id", path)
+    if raw.get("suite_type") == "customer_acceptance":
+        _id(raw.get("customer"), f"{path}.customer")
+    elif "customer" in raw:
+        _id(raw["customer"], f"{path}.customer")
+    versions_raw = _mapping(raw.get("versions"), f"{path}.versions")
+    versions = CorpusVersions(**{field: _id(versions_raw.get(field), f"{path}.versions.{field}") for field in ("corpus", "index", "chunking", "embedding")})
+    candidate_values = _list(raw.get("candidates"), f"{path}.candidates", nonempty=True)
+    group_values = _list(raw.get("evidence_groups"), f"{path}.evidence_groups")
+    candidates: list[FrozenCandidate] = []
+    reference_candidates: list[ReferenceCandidate] = []
+    raw_candidates: list[Mapping[str, Any]] = []
+    for candidate_index, value in enumerate(candidate_values):
+        candidate_path = f"{path}.candidates[{candidate_index}]"
+        item = _mapping(value, candidate_path)
+        raw_candidates.append(item)
+        if type(item.get("dense_rank")) is not int or item["dense_rank"] != candidate_index + 1:
+            raise ContractError(f"{candidate_path}.dense_rank must equal pool position")
+        grade = item.get("relevance_grade")
+        if type(grade) is not int or not 0 <= grade <= 3:
+            raise ContractError(f"{candidate_path}.relevance_grade must be 0..3")
+        frozen = FrozenCandidate(
+            candidate_id=_id(item.get("candidate_id"), f"{candidate_path}.candidate_id"),
+            document_id=_id(item.get("document_id"), f"{candidate_path}.document_id"),
+            source_id=_id(item.get("source_id"), f"{candidate_path}.source_id"),
+            chunk_id=_id(item.get("chunk_id"), f"{candidate_path}.chunk_id"),
+            url=_text(item.get("url"), f"{candidate_path}.url"),
+            input=_text(item.get("input"), f"{candidate_path}.input"),
+            dense_rank=item["dense_rank"],
+            dense_score=_finite_number(item.get("dense_score"), f"{candidate_path}.dense_score"),
+        )
+        candidates.append(frozen)
+        reference_candidates.append(ReferenceCandidate(
+            frozen.candidate_id,
+            grade,
+            _unique_ids(item.get("evidence_group_ids"), f"{candidate_path}.evidence_group_ids"),
+        ))
+        _validate_candidate_annotations(item, candidate_path)
+    candidate_ids = {candidate.candidate_id for candidate in candidates}
+    if len(candidate_ids) != len(candidates):
+        raise ContractError(f"{path}.candidates must have unique IDs")
+    for candidate_index, item in enumerate(raw_candidates):
+        if "duplicate_of_candidate_id" in item:
+            duplicate_id = _id(item["duplicate_of_candidate_id"], f"{path}.candidates[{candidate_index}].duplicate_of_candidate_id")
+            if duplicate_id not in candidate_ids or duplicate_id == candidates[candidate_index].candidate_id:
+                raise ContractError(f"{path}.candidates[{candidate_index}].duplicate_of_candidate_id is invalid")
+    groups: list[EvidenceGroup] = []
+    for group_index, value in enumerate(group_values):
+        group_path = f"{path}.evidence_groups[{group_index}]"
+        item = _mapping(value, group_path)
+        groups.append(EvidenceGroup(
+            evidence_group_id=_id(item.get("evidence_group_id"), f"{group_path}.evidence_group_id"),
+            description=_text(item.get("description"), f"{group_path}.description"),
+            required=item.get("required"),
+            acceptable_candidate_ids=_unique_ids(item.get("acceptable_candidate_ids"), f"{group_path}.acceptable_candidate_ids"),
+        ))
+    group_ids = {group.evidence_group_id for group in groups}
+    if len(group_ids) != len(groups):
+        raise ContractError(f"{path}.evidence_groups must have unique IDs")
+    refs_by_id = {reference.candidate_id: reference for reference in reference_candidates}
+    for reference in reference_candidates:
+        for group_id in reference.evidence_group_ids:
+            if group_id not in group_ids:
+                raise ContractError(f"{path}.candidates[{reference.candidate_id}] links unknown evidence group {group_id}")
+            group = next(group for group in groups if group.evidence_group_id == group_id)
+            if reference.candidate_id not in group.acceptable_candidate_ids:
+                raise ContractError(f"{path} evidence-group link is not reciprocal for {reference.candidate_id}")
+    for group in groups:
+        for candidate_id in group.acceptable_candidate_ids:
+            reference = refs_by_id.get(candidate_id)
+            if reference is None or group.evidence_group_id not in reference.evidence_group_ids:
+                raise ContractError(f"{path} evidence-group link is not reciprocal for {candidate_id}")
+            if reference.relevance_grade < 2:
+                raise ContractError(f"{path} acceptable evidence {candidate_id} must have grade >= 2")
+    required_count = sum(group.required for group in groups)
+    if answerability == "answerable" and required_count == 0:
+        raise ContractError(f"{path} answerable case needs a required evidence group")
+    if answerability == "unsupported":
+        if required_count != 0 or raw.get("expected_unsupported_behavior") != "abstain":
+            raise ContractError(f"{path} unsupported case requires no required groups and abstain behavior")
+    elif "expected_unsupported_behavior" in raw:
+        raise ContractError(f"{path} answerable case cannot specify unsupported behavior")
+    case = FrozenPoolCase(
+        dataset_id=dataset_id,
+        dataset_version=dataset_version,
+        case_id=case_id,
+        query_id=query_id,
+        query=query,
+        split=raw.get("split"),
+        split_group_id=split_group_id,
+        cluster_id=cluster_id,
+        suite_type=raw.get("suite_type"),
+        versions=versions,
+        candidates=tuple(candidates),
+    )
+    references = CaseReferences(case_id, label_authority, tuple(reference_candidates), tuple(groups))
+    return case, references
+
+
+def validate_dataset_v2(value: object) -> tuple[tuple[FrozenPoolCase, ...], tuple[CaseReferences, ...], int]:
+    """Validate dataset-v2 identity, pools, labels, and reciprocal group links.
+
+    This validates the *schema*. Release identity is checked separately by the
+    pinned fixture hash in :func:`load_rag_rerank_pack`. As v2 has no label
+    authority field, arbitrary input is marked provisional by this validator.
+    """
+
+    dataset = _mapping(value, "dataset")
+    if dataset.get("schema_version") != CASE_SCHEMA_VERSION:
+        raise ContractError(f"schema_version must be {CASE_SCHEMA_VERSION}")
+    dataset_id = _id(dataset.get("dataset_id"), "dataset_id")
+    dataset_version = _id(dataset.get("version"), "version")
+    threshold = dataset.get("default_relevance_threshold")
+    if type(threshold) is not int or threshold not in (1, 2, 3):
+        raise ContractError("default_relevance_threshold must be 1, 2, or 3")
+    tasks = _list(dataset.get("tasks"), "tasks", nonempty=True)
+    pairs = tuple(_validate_task(task, index, dataset_id, dataset_version, "provisional") for index, task in enumerate(tasks))
+    cases = tuple(case for case, _ in pairs)
+    references = tuple(reference for _, reference in pairs)
+    for field in ("case_id", "query_id"):
+        values = [getattr(case, field) for case in cases]
+        if len(set(values)) != len(values):
+            raise ContractError(f"tasks must have unique {field} values")
+    for field in ("cluster_id", "split_group_id"):
+        split_by_group: dict[str, str] = {}
+        for case in cases:
+            group = getattr(case, field)
+            if group in split_by_group and split_by_group[group] != case.split:
+                raise ContractError(f"{field} {group!r} crosses dataset splits")
+            split_by_group[group] = case.split
+    return cases, references, threshold
+
+
+@lru_cache(maxsize=1)
+def load_rag_rerank_pack() -> RagRerankPack:
+    """Load the installed six-case synthetic pack independently of current CWD."""
+
+    payload = resources.files("sphynx").joinpath("data", _FIXTURE_NAME).read_bytes()
+    sha = hashlib.sha256(payload).hexdigest()
+    if sha != FIXTURE_SHA256:
+        raise ContractError("Installed RAG fixture bytes differ from pinned illustrative release")
+    try:
+        value = json.loads(payload)
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise ContractError("Installed RAG fixture is not UTF-8 JSON") from exc
+    cases, _, threshold = validate_dataset_v2(value)
+    # Synthetic authority applies only after the exact installed fixture byte
+    # hash above has been verified, never to an arbitrary v2 dataset.
+    references = tuple(
+        _validate_task(task, index, cases[index].dataset_id, cases[index].dataset_version, "synthetic_authored")[1]
+        for index, task in enumerate(value["tasks"])
+    )
+    descriptor = get_pack(PACK_ID)
+    if not descriptor.runnable or descriptor.case_count != len(cases) or descriptor.metric_release is None:
+        raise ContractError("RAG descriptor disagrees with the installed fixture or metric release")
+    if len(cases) != 6 or sum(len(case.candidates) for case in cases) != 25:
+        raise ContractError("Illustrative RAG release must contain exactly six cases and 25 passages")
+    return RagRerankPack(descriptor, cases, references, descriptor.metric_release, f"sha256:{sha}", threshold)
+
+
+def load_cases(pack_id: str) -> tuple[FrozenPoolCase, ...]:
+    """Return only label-free cases in fixed pack order for candidate execution."""
+
+    if pack_id != PACK_ID:
+        raise ValueError(f"No case loader for pack {pack_id!r}; installed: {PACK_ID}")
+    return load_rag_rerank_pack().cases
+
+
+def validate_pack_case_identity(case: FrozenPoolCase) -> None:
+    """Reject a changed pool/corpus/cohort before comparing to pack references."""
+
+    if not isinstance(case, FrozenPoolCase):
+        raise ContractError("case must be FrozenPoolCase")
+    original = next((item for item in load_cases(PACK_ID) if item.case_id == case.case_id), None)
+    if original is None or original.pool_digest != case.pool_digest:
+        raise ContractError(f"Case {case.case_id!r} does not match the frozen pack pool and corpus identity")

sphynx/rag_diagnostics.py

@@ -0,0 +1,309 @@
+"""Versioned, offline RAG diagnostics over caller-held paired traces.
+
+These APIs regrade saved outputs. They never execute candidates, contact a model,
+download a benchmark, grant hosted admission, or infer semantic review labels.
+Reports contain owner-held text and references: keep them in private storage.
+"""
+from __future__ import annotations
+
+import copy
+from importlib.resources import files
+import json
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+from uuid import uuid4
+
+from ._rag_diagnostics_common import VERSION, canonical, digest
+from ._rag_diagnostics_contracts import (
+    ContractError, candidate_view, case_input_digest, load_dataset, validate_dataset, _json_domain,
+)
+from ._rag_diagnostics_engine import evaluate
+from ._rag_diagnostics_catalog import INFO, describe
+from ._rag_diagnostics_calibration import judge_audit, ppi_mean
+from ._rag_diagnostics_importers import import_beir
+
+__all__ = [
+    "load_rag_diagnostics_dataset", "validate_rag_diagnostics_dataset",
+    "rag_candidate_view", "grade_rag_diagnostics", "load_rag_diagnostics_demo",
+    "list_rag_diagnostics_metrics", "rag_diagnostics_capabilities",
+    "save_rag_diagnostics_report", "load_rag_diagnostics_report",
+    "render_rag_diagnostics_report", "import_rag_diagnostics_beir",
+    "audit_rag_judge", "estimate_rag_ppi_mean",
+    "create_rag_diagnostics_review_template",
+]
+
+CONDITIONAL = {
+    "context.retrieval_sufficiency_recall": (
+        "Retrieval sufficiency recall", r"S_{\mathrm{retrieved}}/N_{\mathrm{answerable}}",
+        "Among independently source-answerable cases, count sufficient retrieved contexts. Unknown eligibility or missing judgments suppress the primary value.",
+        ["independent source answerability", "trace-bound retrieved-context sufficiency"],
+    ),
+    "generation.success_given_sufficient_context": (
+        "Success given sufficient context", r"N_{\mathrm{correct,\ answered,\ sufficient}}/N_{\mathrm{sufficient}}",
+        "Among sufficient delivered contexts, count correct non-abstaining answers. Correctness is reviewed independently of support.",
+        ["trace-bound prompt sufficiency", "independent answer correctness", "successful response and abstention status"],
+    ),
+    "generation.unsupported_answer_given_insufficient_context": (
+        "Unsupported answer given insufficient context", r"N_{\mathrm{unsupported,\ insufficient}}/N_{\mathrm{insufficient}}",
+        "Among insufficient delivered contexts, count responses with at least one unsupported claim. Requires exhaustive review; an empty appropriate abstention is not an unsupported answer.",
+        ["trace-bound prompt sufficiency", "exhaustive response claim review"],
+    ),
+    "outcome.abstention_recall": (
+        "Abstention recall", r"N_{\mathrm{abstained,\ expected}}/N_{\mathrm{expected}}",
+        "Among cases explicitly labeled as requiring abstention, count actual abstentions. Source answerability is not substituted for the expected action.",
+        ["independent expected-abstention label", "response status and abstention"],
+    ),
+}
+
+
+def load_rag_diagnostics_dataset(path: str | Path) -> dict[str, Any]:
+    """Read a bounded UTF-8 dataset and validate IDs, lineage and review digests."""
+    return load_dataset(path)
+
+
+def validate_rag_diagnostics_dataset(dataset: dict[str, Any]) -> dict[str, Any]:
+    """Validate the versioned paired-trace contract; return the supplied object."""
+    return validate_dataset(dataset)
+
+
+def rag_candidate_view(case: dict[str, Any]) -> dict[str, Any]:
+    """Copy only case ID, question and documents; omit protected scoring labels."""
+    return candidate_view(case)
+
+
+def load_rag_diagnostics_demo() -> dict[str, Any]:
+    """Load 32 explicitly synthetic saved cases spanning 16 diagnostic kinds."""
+    value = json.loads(files("sphynx").joinpath("data/rag-diagnostics-synthetic.v1.json").read_text(encoding="utf-8"))
+    return validate_dataset(value)
+
+
+def create_rag_diagnostics_review_template(
+    dataset: dict[str, Any], *, variant: str = "candidate"
+) -> dict[str, Any]:
+    """Create pending, exact-input review forms; no semantic labels are inferred.
+
+    Fill the form through independent review before copying a case review into
+    its protected references. claims_exhaustive=False and null judgments are
+    deliberate. For recorded traces, identify the actual human reviewer.
+    """
+    validate_dataset(dataset)
+    if variant not in ("baseline", "candidate"):
+        raise ContractError("Select baseline or candidate for review")
+    return {
+        "schema_version": "sphynx.rag-diagnostics.review-template.v1",
+        "status": "pending_review", "variant": variant,
+        "reviews": {
+            case["id"]: {
+                "trace_digest": digest(case["traces"][variant]),
+                "case_input_digest": case_input_digest(case),
+                "provenance": {"kind": "synthetic" if case["traces"][variant]["execution_mode"] == "synthetic_replay" else "human_adjudicated",
+                               "reviewer": "REPLACE_WITH_ACTUAL_REVIEWER"},
+                "answer_correct": None, "answer_relevant": None,
+                "abstention_expected": None,
+                "context_sufficiency": {stage: None for stage in ("retrieved", "selected", "prompt")},
+                "claims_exhaustive": False, "claims": [],
+            } for case in dataset["cases"]
+        },
+    }
+
+
+def grade_rag_diagnostics(
+    dataset: dict[str, Any], *, cutoff: int = 3, bootstrap_samples: int = 500
+) -> dict[str, Any]:
+    """Grade both saved variants with explicit unavailable values and paired deltas.
+
+    The input must use sphynx.rag-diagnostics.dataset.v1. Semantic review must be
+    bound to the exact trace and case inputs. Missing eligible pairs suppress the
+    primary comparison. Recorded telemetry is never replay grading duration.
+    """
+    result = evaluate(copy.deepcopy(dataset), cutoff, bootstrap_samples)
+    result["evaluation_id"] = "ragdiag-" + uuid4().hex
+    result["sdk_version"] = "0.5.0"
+    result["trust_tier"] = "local_unverified"
+    result["hosted_admitted"] = False
+    return result
+
+
+def _requirements(identity: str) -> list[str]:
+    if identity.startswith("retrieval."):
+        return ["known qrels over the full reference population", "ordered recorded stage IDs"]
+    if identity.startswith("evidence."):
+        return ["required evidence atoms or relations", "protected support-document mapping", "recorded stage IDs"]
+    if identity.startswith("context."):
+        return ["exact recorded context", "independent trace-bound sufficiency review"]
+    if identity.startswith(("generation.", "citation.", "outcome.")):
+        return ["successful saved response or explicit failure", "trace-bound independent semantic review", "metric-specific claim, fact or citation judgments"]
+    return ["explicit saved execution status", "synthetic or component-reported telemetry where required"]
+
+
+def list_rag_diagnostics_metrics(*, cutoff: int = 3) -> list[dict[str, Any]]:
+    """List 29 case definitions and four conditional definitions with requirements."""
+    if type(cutoff) is not int or not 1 <= cutoff <= 1000:
+        raise ValueError("cutoff must be an integer from 1 to 1000")
+    result = []
+    for base in INFO:
+        identities = (
+            [base + "@" + str(cutoff)] if base.startswith("retrieval.")
+            else [base + "." + stage for stage in ("retrieved", "selected", "prompt")]
+            if base == "context.sufficient" else [base]
+        )
+        for identity in identities:
+            item = describe(identity)
+            result.append({
+                "metric_id": identity, "metric_version": VERSION, "scope": "case",
+                **item, "requires": _requirements(identity),
+                "missing_policy": "Unavailable inputs remain null with a reason; zero-population cases are explicit.",
+            })
+    for identity, (title, formula, meaning, requires) in CONDITIONAL.items():
+        result.append({
+            "metric_id": identity, "metric_version": VERSION, "scope": "conditional",
+            "title": title, "formula": formula, "meaning": meaning, "requires": requires,
+            "missing_policy": "Unknown eligibility or missing eligible scores suppress the primary value; observed subsets are named separately.",
+        })
+    return result
+
+
+def rag_diagnostics_capabilities(
+    dataset: dict[str, Any], *, cutoff: int = 3
+) -> dict[str, Any]:
+    """Resolve actual score availability from supplied inputs without execution."""
+    report = grade_rag_diagnostics(dataset, cutoff=cutoff, bootstrap_samples=0)
+    variants = {}
+    for variant in ("baseline", "candidate"):
+        variants[variant] = {
+            "case_metrics": {
+                key: {field: value[field] for field in (
+                    "metric_id", "stage", "eligible_cases", "scored_cases",
+                    "missing_cases", "not_applicable_cases", "complete", "reason_counts",
+                )}
+                for key, value in report["aggregates"][variant].items()
+            },
+            "conditional_metrics": report["conditional"][variant],
+        }
+    return {
+        "schema_version": "sphynx.rag-diagnostics.capabilities.v1",
+        "metric_version": VERSION, "dataset_digest": report["dataset_digest"],
+        "source": "Availability for supplied saved inputs; no hosted admission or fresh execution.",
+        "variants": variants,
+    }
+
+
+def import_rag_diagnostics_beir(
+    corpus_path: str | Path, queries_path: str | Path, qrels_path: str | Path,
+    baseline_run_path: str | Path, candidate_run_path: str | Path, *,
+    release: str = "external-beir-replay.v1", cluster_mapping_path: str | Path | None = None,
+    expected_sha256: dict[str, str] | None = None,
+) -> dict[str, Any]:
+    """Import caller-held corpus/query/qrel files and two saved TREC runs offline.
+
+    Optional SHA256 pins must cover every supplied file. This creates no answers
+    or semantic reviews, downloads nothing, and grants no data or license access.
+    """
+    return import_beir(
+        corpus_path, queries_path, qrels_path, baseline_run_path, candidate_run_path,
+        release=release, cluster_mapping_path=cluster_mapping_path,
+        expected_sha256=expected_sha256,
+    )
+
+
+def audit_rag_judge(
+    records: Sequence[Mapping[str, Any]], *, provenance_kind: str = "unspecified"
+) -> dict[str, Any]:
+    """Compute descriptive agreement over supplied human/judge labels; no model."""
+    return judge_audit(records, provenance_kind=provenance_kind)
+
+
+def estimate_rag_ppi_mean(
+    predictions: Sequence[Mapping[str, Any]], audit: Sequence[Mapping[str, Any]], *,
+    declarations: Mapping[str, Any] | None = None, confidence: float = 0.95,
+) -> dict[str, Any]:
+    """Compute gated independent-sample PPI arithmetic, never validated coverage."""
+    return ppi_mean(predictions, audit, declarations=declarations, confidence=confidence)
+
+
+def _validate_report(report: dict[str, Any]) -> None:
+    _json_domain(report)
+    if not isinstance(report, dict) or report.get("schema_version") != "sphynx.rag-diagnostics.report.v1" or report.get("metric_version") != VERSION:
+        raise ContractError("Unsupported RAG diagnostic report schema or metric version")
+
+
+def save_rag_diagnostics_report(report: dict[str, Any], path: str | Path) -> dict[str, Any]:
+    """Save a checksummed private JSON report; refuse to overwrite an existing file."""
+    _validate_report(report)
+    envelope = {
+        "schema_version": "sphynx.rag-diagnostics.saved-report.v1",
+        "content_sha256": digest(report), "report": report,
+    }
+    raw = canonical(envelope) + b"\n"
+    if len(raw) > 20_000_000:
+        raise ContractError("Saved diagnostic report exceeds 20MB")
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with target.open("xb") as handle:
+            handle.write(raw)
+    except FileExistsError:
+        raise ContractError("Report already exists; choose a new report filename") from None
+    return {"path": str(target.resolve()), "content_sha256": envelope["content_sha256"],
+            "evaluation_id": report.get("evaluation_id"), "bytes": len(raw)}
+
+
+def load_rag_diagnostics_report(path: str | Path) -> dict[str, Any]:
+    """Reopen a checksummed report without regrading or executing a candidate.
+
+    A checksum detects accidental byte/content changes; it is not a signature
+    or proof of the truth of owner-supplied review or telemetry.
+    """
+    target = Path(path)
+    if target.is_symlink() or not target.is_file() or target.stat().st_size > 20_000_000:
+        raise ContractError("Expected a regular diagnostic report no larger than 20MB")
+    raw = target.read_bytes()
+    if len(raw) > 20_000_000:
+        raise ContractError("Saved diagnostic report exceeds 20MB")
+    def pairs(items):
+        result = {}
+        for key, value in items:
+            if key in result:
+                raise ContractError("Duplicate JSON key in diagnostic report")
+            result[key] = value
+        return result
+    try:
+        envelope = json.loads(raw, object_pairs_hook=pairs,
+                              parse_constant=lambda _: (_ for _ in ()).throw(ContractError("Nonfinite JSON number")))
+    except (ValueError, UnicodeDecodeError, RecursionError) as exc:
+        raise ContractError("Invalid diagnostic report JSON") from exc
+    _json_domain(envelope)
+    if type(envelope) is not dict or set(envelope) != {"schema_version", "content_sha256", "report"} or envelope["schema_version"] != "sphynx.rag-diagnostics.saved-report.v1":
+        raise ContractError("Unsupported saved diagnostic report envelope")
+    report = envelope["report"]
+    _validate_report(report)
+    if envelope["content_sha256"] != digest(report):
+        raise ContractError("Diagnostic report checksum mismatch")
+    return report
+
+
+def render_rag_diagnostics_report(report: dict[str, Any]) -> str:
+    """Render aggregate scores and missingness as Markdown without raw case text."""
+    _validate_report(report)
+    lines = [
+        "# Sphynx saved-trace RAG diagnostics", "",
+        "LOCAL_UNVERIFIED. Regrading saved outputs does not run a candidate.",
+        "Semantic reviews are supplied labels; synthetic examples do not measure model quality.", "",
+        f"Dataset: {report['dataset_release']}; cases: {report['case_count']}; cutoff: {report['cutoff']}.", "",
+        "| Metric / stage | Baseline | Candidate | Missing pairs |",
+        "|---|---:|---:|---:|",
+    ]
+    def show(value):
+        return "Unavailable" if value is None else f"{value:.6g}"
+    for row in report["comparisons"]:
+        label = row["key"].replace("|", " / ").replace("\n", " ")
+        lines.append(f"| {label} | {show(row['baseline']['value'])} | {show(row['candidate']['value'])} | {row['missing_pairs']} |")
+    lines += ["", "Conditional metrics retain their own eligible populations:", "",
+              "| Metric | Baseline | Candidate |", "|---|---:|---:|"]
+    for left, right in zip(report["conditional"]["baseline"], report["conditional"]["candidate"]):
+        lines.append(f"| {left['metric_id']} | {show(left['value'])} | {show(right['value'])} |")
+    lines += ["", "Saved telemetry is unverified synthetic/component data. No fresh model timing was measured.",
+              "Failure localization is observational. PPI arithmetic does not establish validated coverage.", ""]
+    return "\n".join(lines)
+
+