sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/retrieval.py

@@ -0,0 +1,750 @@
+"""Owner-held full retrieval architecture evaluation.
+
+Candidates own retrieval, fusion, reranking and context selection. Questions,
+source bytes and provisional references are fixed; candidate pools may differ.
+Local adapters are not sandboxed. Captures contain no reference labels and are
+not currently compatible with the installed-pack cloud synchronization API.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import platform
+import random
+import statistics
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Iterable, Mapping
+
+from .contracts import ContractError, canonical_json
+from .evidence import EvidenceError
+
+RETRIEVAL_PACK_ID = "rag.retrieve.v1"
+RETRIEVAL_INTERFACE_VERSION = "sphynx.rag.retrieve.v1"
+RETRIEVAL_DATASET_SCHEMA = "sphynx.rag.retrieve.dataset.v1"
+RETRIEVAL_REQUEST_SCHEMA = "sphynx.rag.retrieve.request.v1"
+RETRIEVAL_RESPONSE_SCHEMA = "sphynx.rag.retrieve.response.v1"
+RETRIEVAL_METRIC_RELEASE = "rag.retrieve.metric.v1"
+_RUN_SCHEMA = "sphynx.rag.retrieve.run.v1"
+_EVAL_SCHEMA = "sphynx.rag.retrieve.evaluation.v1"
+_STAGES = ("candidate", "fused", "reranked", "context")
+_CHUNK_FIELDS = ("chunk_id", "document_id", "customer", "text", "text_sha256", "source_class", "source_uri")
+_QUALITY = ("complete_evidence_success", "candidate_evidence_recall_at_50", "final_evidence_group_recall",
+            "ndcg_at_5", "hit_at_5", "mrr_at_5", "fused_evidence_group_recall", "reranked_evidence_group_recall")
+_QUALIFICATION = "Provisional machine-generated source-anchor recovery; unjudged passages assumed zero for development only."
+
+
+def _digest(value: Any) -> str:
+    return "sha256:" + hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
+
+
+def _bytes_digest(raw: bytes) -> str:
+    return "sha256:" + hashlib.sha256(raw).hexdigest()
+
+
+def _text(value: Any, field: str) -> str:
+    if not isinstance(value, str) or not value.strip() or "\x00" in value:
+        raise ContractError(f"{field} must be nonempty text without NUL")
+    return value
+
+
+def _unique_ids(value: Any, field: str, *, empty: bool = True) -> list[str]:
+    if not isinstance(value, list) or (not empty and not value):
+        raise ContractError(f"{field} must be a {'nonempty ' if not empty else ''}list")
+    result = [_text(item, field) for item in value]
+    if len(result) != len(set(result)):
+        raise ContractError(f"{field} contains duplicate identifiers")
+    return result
+
+
+def _strict_json(raw: bytes) -> Any:
+    def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, value in items:
+            if key in result:
+                raise ContractError("JSON contains duplicate keys")
+            result[key] = value
+        return result
+
+    def invalid_constant(value: str) -> Any:
+        raise ContractError("JSON contains a nonfinite number")
+
+    def finite_float(value: str) -> float:
+        parsed = float(value)
+        if not math.isfinite(parsed):
+            raise ContractError("JSON floating-point exponent overflows to a nonfinite number")
+        return parsed
+
+    try:
+        return json.loads(raw, object_pairs_hook=pairs, parse_constant=invalid_constant, parse_float=finite_float)
+    except (UnicodeError, json.JSONDecodeError) as exc:
+        raise ContractError("expected UTF-8 JSON") from exc
+
+
+@dataclass(frozen=True)
+class RetrievalPack:
+    dataset_digest: str
+    corpus_digest: str
+    cohort_digest: str
+    labels_digest: str
+    metadata: Mapping[str, Any]
+    corpus: tuple[Mapping[str, Any], ...]
+    cases: tuple[Mapping[str, Any], ...]
+
+    @property
+    def pack_id(self) -> str:
+        return RETRIEVAL_PACK_ID
+
+
+@dataclass(frozen=True)
+class RetrievalRun:
+    run_id: str
+    path: Path
+    manifest_sha256: str
+    pack_id: str = RETRIEVAL_PACK_ID
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"run_id": self.run_id, "path": str(self.path), "manifest_sha256": self.manifest_sha256,
+                "pack_id": self.pack_id}
+
+    def to_json(self) -> str:
+        return canonical_json(self.to_dict())
+
+
+@dataclass(frozen=True)
+class RetrievalEvaluation:
+    evaluation_id: str
+    path: Path
+    artifact_sha256: str
+    data: Mapping[str, Any]
+
+    @property
+    def metrics(self) -> Mapping[str, Mapping[str, Any]]:
+        return self.data["metrics"]
+
+    @property
+    def cases(self) -> tuple[Mapping[str, Any], ...]:
+        return tuple(self.data["cases"])
+
+    @property
+    def customer_slices(self) -> Mapping[str, Any]:
+        return self.data["customer_slices"]
+
+    @property
+    def run_id(self) -> str:
+        return self.data["run_id"]
+
+    @property
+    def status(self) -> str:
+        return self.data["status"]
+
+    @property
+    def pack_id(self) -> str:
+        return RETRIEVAL_PACK_ID
+
+    def to_dict(self) -> dict[str, Any]:
+        return _strict_json(canonical_json(dict(self.data)).encode("utf-8"))
+
+    def to_json(self) -> str:
+        return canonical_json(self.to_dict())
+
+
+def load_retrieval_pack(dataset_path: str | Path, *, expected_sha256: str | None = None) -> RetrievalPack:
+    """Load an external corpus and answerable cohort without installing data.
+
+    This initial release supports only the explicitly qualified development
+    anchor proxy. It cannot relabel machine-generated anchors as human qrels.
+    """
+    raw = Path(dataset_path).expanduser().resolve(strict=True).read_bytes()
+    digest = _bytes_digest(raw)
+    if expected_sha256 is not None and "sha256:" + expected_sha256.removeprefix("sha256:") != digest:
+        raise ContractError("retrieval dataset differs from expected SHA-256")
+    value = _strict_json(raw)
+    if not isinstance(value, dict) or value.get("schema_version") != RETRIEVAL_DATASET_SCHEMA:
+        raise ContractError("unsupported retrieval dataset schema")
+    meta = value.get("metadata")
+    expected_meta = {"origin": "reconstructed_development", "label_status": "provisional",
+                     "label_basis": "machine_generated_source_anchor_recovery",
+                     "unjudged_policy": "assume_zero_development", "development_only": True}
+    if not isinstance(meta, dict) or any(meta.get(key) != expected for key, expected in expected_meta.items()):
+        raise ContractError("retrieval metadata must explicitly qualify provisional development labels")
+    if type(meta.get("development_only")) is not bool:
+        raise ContractError("development_only must be a JSON boolean")
+    corpus = value.get("corpus")
+    if not isinstance(corpus, list) or not corpus:
+        raise ContractError("retrieval corpus must be a nonempty list")
+    chunk_map: dict[str, dict[str, Any]] = {}
+    for item in corpus:
+        if not isinstance(item, dict):
+            raise ContractError("corpus chunks must be objects")
+        chunk = {field: _text(item.get(field), "chunk." + field) for field in _CHUNK_FIELDS}
+        text_sha = hashlib.sha256(chunk["text"].encode("utf-8")).hexdigest()
+        if chunk["text_sha256"].removeprefix("sha256:") != text_sha:
+            raise ContractError("corpus text digest does not match pinned bytes")
+        chunk["text_sha256"] = text_sha
+        if chunk["chunk_id"] in chunk_map:
+            raise ContractError("corpus chunk IDs must be globally unique")
+        chunk_map[chunk["chunk_id"]] = chunk
+    cases = value.get("cases")
+    if not isinstance(cases, list) or not cases:
+        raise ContractError("retrieval cases must be a nonempty list")
+    normalized: list[dict[str, Any]] = []
+    query_ids: set[str] = set()
+    for case in cases:
+        if not isinstance(case, dict):
+            raise ContractError("retrieval cases must be objects")
+        row = {field: _text(case.get(field), "case." + field)
+               for field in ("query_id", "query", "customer", "cluster_id")}
+        if row["query_id"] in query_ids:
+            raise ContractError("query IDs must be unique")
+        query_ids.add(row["query_id"])
+        context = case.get("conversation_context", [])
+        if not isinstance(context, (str, list)):
+            raise ContractError("conversation_context must be a string or JSON list")
+        # Preserve the frozen conversation rather than flattening differently
+        # for each architecture. No owner reference keys are serialized here.
+        row["conversation_context"] = _strict_json(canonical_json(context).encode("utf-8"))
+        refs = case.get("references")
+        if not isinstance(refs, dict):
+            raise ContractError("each case requires owner references")
+        groups = refs.get("evidence_groups")
+        qrels = refs.get("graded_relevance")
+        if not isinstance(groups, list) or not groups or not isinstance(qrels, dict):
+            raise ContractError("answerable cases require groups and graded relevance")
+        group_rows: list[dict[str, Any]] = []
+        group_ids: set[str] = set()
+        for group in groups:
+            if not isinstance(group, dict) or type(group.get("required")) is not bool:
+                raise ContractError("evidence groups require a boolean required field")
+            group_id = _text(group.get("evidence_group_id"), "evidence_group_id")
+            if group_id in group_ids:
+                raise ContractError("evidence group IDs must be unique within a case")
+            group_ids.add(group_id)
+            anchors = _unique_ids(group.get("acceptable_chunk_ids"), "acceptable_chunk_ids", empty=False)
+            for anchor in anchors:
+                if anchor not in chunk_map or chunk_map[anchor]["customer"] != row["customer"]:
+                    raise ContractError("references must resolve within the case customer corpus")
+                if type(qrels.get(anchor)) is not int or not 1 <= qrels[anchor] <= 3:
+                    raise ContractError("every acceptable anchor needs a positive graded relevance judgment")
+            group_rows.append({"evidence_group_id": group_id, "required": group["required"],
+                               "acceptable_chunk_ids": sorted(anchors)})
+        if not any(group["required"] for group in group_rows):
+            raise ContractError("this pack requires corpus-answerable cases with required evidence")
+        for anchor, relevance in qrels.items():
+            if (anchor not in chunk_map or chunk_map[anchor]["customer"] != row["customer"]
+                    or type(relevance) is not int or not 0 <= relevance <= 3):
+                raise ContractError("graded relevance must be 0..3 for same-customer corpus chunks")
+        row["references"] = {"evidence_groups": sorted(group_rows, key=lambda group: group["evidence_group_id"]),
+                             "graded_relevance": dict(sorted(qrels.items()))}
+        normalized.append(row)
+    corpus_rows = tuple(chunk_map[key] for key in sorted(chunk_map))
+    case_rows = tuple(normalized)
+    cohort = [{key: row[key] for key in ("query_id", "query", "customer", "cluster_id", "conversation_context")}
+              for row in case_rows]
+    labels = [{"query_id": row["query_id"], "references": row["references"]} for row in case_rows]
+    return RetrievalPack(digest, _digest(corpus_rows), _digest(cohort), _digest(labels), meta, corpus_rows, case_rows)
+
+
+def _policy(max_context_bytes: int) -> dict[str, Any]:
+    if type(max_context_bytes) is not int or max_context_bytes <= 0:
+        raise ContractError("max_context_bytes must be a positive integer")
+    return {"policy_version": "v1", "candidate_depth": 50, "ranking_depth": 5,
+            "max_context_bytes": max_context_bytes, "context_separator": "\n\n",
+            "token_budget_policy": "utf8_byte_budget_proxy.v1", "trials": 1,
+            "execution_isolation": "local_unsandboxed"}
+
+
+def build_retrieval_request(pack: RetrievalPack, case: str | Mapping[str, Any], *, max_context_bytes: int = 16000) -> dict[str, Any]:
+    """Construct the only candidate-visible request, using an explicit allowlist."""
+    query_id = case if isinstance(case, str) else case.get("query_id")
+    row = next((item for item in pack.cases if item["query_id"] == query_id), None)
+    if row is None:
+        raise ContractError("query does not belong to retrieval pack")
+    value = {"schema_version": RETRIEVAL_REQUEST_SCHEMA, "pack_id": RETRIEVAL_PACK_ID,
+             "interface_version": RETRIEVAL_INTERFACE_VERSION, "query_id": row["query_id"],
+             "query": row["query"], "customer": row["customer"],
+             "conversation_context": _strict_json(canonical_json(row["conversation_context"]).encode("utf-8")),
+             "corpus_digest": pack.corpus_digest, "execution_policy": _policy(max_context_bytes),
+             "corpus": [{key: chunk[key] for key in _CHUNK_FIELDS}
+                        for chunk in pack.corpus if chunk["customer"] == row["customer"]]}
+    value["request_sha256"] = _digest(value)
+    return value
+
+
+def _candidate(candidate: Mapping[str, Any]) -> dict[str, Any]:
+    if not isinstance(candidate, Mapping):
+        raise ContractError("candidate metadata must be a mapping")
+    value = {"name": _text(candidate.get("name"), "candidate.name"),
+             "version": _text(candidate.get("version"), "candidate.version"),
+             "interface_version": candidate.get("interface_version", RETRIEVAL_INTERFACE_VERSION)}
+    if value["interface_version"] != RETRIEVAL_INTERFACE_VERSION:
+        raise ContractError("candidate interface must match retrieval pack")
+    architecture = candidate.get("architecture")
+    required = ("embedding", "index", "fusion", "reranker", "chunking", "context_selection", "code_revision")
+    if not isinstance(architecture, Mapping) or any(key not in architecture or not isinstance(architecture[key], (str, Mapping))
+                                                   or not architecture[key] for key in required):
+        raise ContractError("architecture must explicitly pin embedding, index, fusion, reranker, chunking, context_selection and code_revision")
+    value["architecture"] = _strict_json(canonical_json(dict(architecture)).encode("utf-8"))
+    value["configuration_sha256"] = _digest(value)
+    value["configuration_authority"] = "caller_declared_not_code_attestation"
+    return value
+
+
+def _response(raw: Any, request: Mapping[str, Any]) -> dict[str, Any]:
+    if not isinstance(raw, dict) or raw.get("schema_version") != RETRIEVAL_RESPONSE_SCHEMA:
+        raise ContractError("unsupported retrieval response schema")
+    for field in ("query_id", "request_sha256", "corpus_digest"):
+        if raw.get(field) != request[field]:
+            raise ContractError("response does not echo its request identity")
+    stages = raw.get("stages")
+    if not isinstance(stages, dict) or set(stages) != set(_STAGES):
+        raise ContractError("response requires candidate, fused, reranked and context stages")
+    allowed = {chunk["chunk_id"]: chunk for chunk in request["corpus"]}
+    previous: set[str] | None = None
+    normalized: dict[str, list[str]] = {}
+    for stage in _STAGES:
+        ids = _unique_ids(stages[stage], "stages." + stage)
+        if any(chunk_id not in allowed for chunk_id in ids):
+            raise ContractError("response contains unknown or wrong-customer chunks")
+        if stage == "candidate" and len(ids) > 50:
+            raise ContractError("candidate stage exceeds depth 50")
+        if previous is not None and not set(ids) <= previous:
+            raise ContractError("each stage must be a subset of its preceding stage")
+        normalized[stage] = ids
+        previous = set(ids)
+    text = "\n\n".join(allowed[chunk_id]["text"] for chunk_id in normalized["context"])
+    if len(text.encode("utf-8")) > request["execution_policy"]["max_context_bytes"]:
+        raise ContractError("selected context exceeds the UTF-8 byte budget")
+    if "context_text" in raw and raw["context_text"] != text:
+        raise ContractError("candidate context text differs from pinned source text")
+    return {"schema_version": RETRIEVAL_RESPONSE_SCHEMA,
+            **{field: raw[field] for field in ("query_id", "request_sha256", "corpus_digest")},
+            "stages": normalized, "context_text": text, "context_bytes": len(text.encode("utf-8")),
+            "stage_ranks": {stage: [{"chunk_id": chunk_id, "rank": rank} for rank, chunk_id in enumerate(ids, 1)]
+                            for stage, ids in normalized.items()}}
+
+
+def _write(path: Path, value: Any) -> str:
+    raw = (canonical_json(value) + "\n").encode("utf-8")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("xb") as stream:
+        stream.write(raw)
+    return _bytes_digest(raw)
+
+
+def _checked(root: Path, entry: Mapping[str, Any]) -> Any:
+    if not isinstance(entry, Mapping):
+        raise EvidenceError("artifact index entry must be an object")
+    relative = entry.get("path")
+    if not isinstance(relative, str) or Path(relative).is_absolute():
+        raise EvidenceError("artifact path is invalid")
+    target = (root / relative).resolve()
+    if not target.is_relative_to(root.resolve()):
+        raise EvidenceError("artifact path escapes bundle")
+    try:
+        raw = target.read_bytes()
+    except OSError as exc:
+        raise EvidenceError("artifact is missing") from exc
+    if _bytes_digest(raw) != entry.get("sha256"):
+        raise EvidenceError("retrieval artifact hash mismatch")
+    return _strict_json(raw)
+
+
+def _persist_capture(candidate: Mapping[str, Any], pack: RetrievalPack, responses: Iterable[Mapping[str, Any]],
+                     output_dir: str | Path, max_context_bytes: int, provenance: str,
+                     elapsed: Mapping[str, float] | None, runner_metadata: Mapping[str, Any] | None) -> RetrievalRun:
+    identity = _candidate(candidate)
+    policy = _policy(max_context_bytes)
+    known = {case["query_id"] for case in pack.cases}
+    by_query: dict[str, Mapping[str, Any]] = {}
+    for response in responses:
+        if not isinstance(response, Mapping) or response.get("query_id") not in known:
+            raise ContractError("capture responses must identify a known query")
+        query_id = response["query_id"]
+        if query_id in by_query:
+            raise ContractError("capture contains duplicate query responses")
+        # Batch generators may reuse a mutable mapping between yields. Freeze
+        # each observation immediately instead of retaining caller aliases.
+        try:
+            by_query[query_id] = _strict_json(canonical_json(dict(response)).encode("utf-8"))
+        except ContractError:
+            by_query[query_id] = {"query_id": query_id, "response": {"unserializable_response": "caller_supplied"}}
+    run_id = "run-" + uuid.uuid4().hex
+    root = Path(output_dir).expanduser().resolve() / run_id
+    root.mkdir(parents=True, exist_ok=False)
+    rows: list[dict[str, Any]] = []
+    for case in pack.cases:
+        request = build_retrieval_request(pack, case, max_context_bytes=max_context_bytes)
+        prefix = "captures/cap-" + uuid.uuid4().hex + "/"
+        artifacts: dict[str, Any] = {}
+
+        def save(role: str, value: Any) -> None:
+            relative = prefix + role + ".json"
+            artifacts[role] = {"path": relative, "sha256": _write(root / relative, value)}
+
+        save("request", request)
+        supplied = by_query.get(case["query_id"])
+        error_code: str | None = None
+        status = "missing_response"
+        if supplied is not None:
+            wrapper = "response" in supplied and "schema_version" not in supplied
+            raw = supplied.get("response") if wrapper else dict(supplied)
+            if wrapper and raw is None:
+                status, error_code = "candidate_error", str(supplied.get("error_class", "CandidateError"))
+            else:
+                try:
+                    save("raw_response", raw)
+                    validated = _response(raw, request)
+                    save("validated_response", validated)
+                    status = "success"
+                except ContractError:
+                    if "raw_response" not in artifacts:
+                        save("raw_response", {"unserializable_response": type(raw).__name__})
+                    status, error_code = "invalid_response", "ContractError"
+        observed_ms = elapsed.get(case["query_id"]) if elapsed is not None else None
+        if observed_ms is not None and (type(observed_ms) not in (float, int) or not math.isfinite(observed_ms) or observed_ms < 0):
+            raise ContractError("observed adapter latency must be finite and nonnegative")
+        rows.append({"query_id": case["query_id"], "customer": case["customer"], "cluster_id": case["cluster_id"],
+                     "status": status, "error_code": error_code, "artifacts": artifacts,
+                     "adapter_wall_ms": observed_ms, "timing_authority": "adapter_observed" if observed_ms is not None else "unavailable"})
+    manifest = {"schema_version": _RUN_SCHEMA, "run_id": run_id, "pack_id": RETRIEVAL_PACK_ID,
+                "status": "completed" if all(row["status"] == "success" for row in rows) else "completed_with_failures",
+                "dataset_digest": pack.dataset_digest, "corpus_digest": pack.corpus_digest,
+                "cohort_digest": pack.cohort_digest, "labels_digest": pack.labels_digest,
+                "candidate": identity, "execution_policy": policy, "metric_release": RETRIEVAL_METRIC_RELEASE,
+                "trust_tier": "local_unverified", "provenance": provenance,
+                "runner": {"release": "sphynx.retrieval.local.v1", "python": f"{sys.version_info.major}.{sys.version_info.minor}",
+                           "platform": platform.system(), "machine": platform.machine(),
+                           "clock_domain": "python.perf_counter_ns.process_local"},
+                "runner_metadata": dict(runner_metadata or {}), "runner_metadata_authority": "caller_declared",
+                "candidate_reported_observations_authority": "candidate_reported_not_runner_verified",
+                "cases": rows, "case_count": len(rows)}
+    digest = _write(root / "manifest.json", manifest)
+    with (root / "manifest.sha256").open("x", encoding="ascii") as stream:
+        stream.write(digest + "\n")
+    return RetrievalRun(run_id, root, digest)
+
+
+def capture_retrieval(candidate: Mapping[str, Any], dataset_path: str | Path, responses: Iterable[Mapping[str, Any]],
+                      output_dir: str | Path, *, max_context_bytes: int = 16000,
+                      expected_sha256: str | None = None, runner_metadata: Mapping[str, Any] | None = None) -> RetrievalRun:
+    """Save batch responses as caller-supplied evidence, without timing claims.
+
+    Missing/invalid responses remain failed captures. An empty, valid retrieval
+    is a genuine zero-quality observation. Candidate timing/cost diagnostics
+    are retained in raw responses but are not runner-verified measurements.
+    """
+    pack = load_retrieval_pack(dataset_path, expected_sha256=expected_sha256)
+    return _persist_capture(candidate, pack, responses, output_dir, max_context_bytes, "caller_supplied", None, runner_metadata)
+
+
+def run_retrieval(candidate: Mapping[str, Any], adapter: Callable[[dict[str, Any]], Mapping[str, Any]],
+                  dataset_path: str | Path, output_dir: str | Path, *, max_context_bytes: int = 16000,
+                  expected_sha256: str | None = None, runner_metadata: Mapping[str, Any] | None = None) -> RetrievalRun:
+    """Run all cases in-process and measure complete adapter wall time.
+
+    This local callable has no timeout or OS isolation. Use only trusted local
+    development candidates. Initialization outside the callable is excluded
+    and must be separately recorded if evaluating indexing/startup cost.
+    """
+    pack = load_retrieval_pack(dataset_path, expected_sha256=expected_sha256)
+    _candidate(candidate)
+    _policy(max_context_bytes)
+    responses: list[Mapping[str, Any]] = []
+    elapsed: dict[str, float] = {}
+    for case in pack.cases:
+        request = build_retrieval_request(pack, case, max_context_bytes=max_context_bytes)
+        start = time.perf_counter_ns()
+        try:
+            raw = adapter(request)
+            # Copy immediately: adapters may reuse the same dict on every case.
+            try:
+                raw = _strict_json(canonical_json(dict(raw) if isinstance(raw, Mapping) else raw).encode("utf-8"))
+            except ContractError:
+                raw = {"unserializable_response": type(raw).__name__}
+            response = {"query_id": case["query_id"], "response": raw}
+        except Exception as exc:
+            response = {"query_id": case["query_id"], "response": None, "error_class": type(exc).__name__}
+        elapsed[case["query_id"]] = (time.perf_counter_ns() - start) / 1_000_000
+        responses.append(response)
+    return _persist_capture(candidate, pack, responses, output_dir, max_context_bytes, "adapter_observed", elapsed, runner_metadata)
+
+
+def verify_retrieval_run(run: RetrievalRun | str | Path) -> dict[str, Any]:
+    """Verify immutable artifact bytes; hashes are not a remote trust boundary."""
+    root = (run.path if isinstance(run, RetrievalRun) else Path(run)).expanduser().resolve()
+    try:
+        stored = (root / "manifest.sha256").read_text(encoding="ascii").strip()
+    except OSError as exc:
+        raise EvidenceError("retrieval manifest checksum is missing") from exc
+    if isinstance(run, RetrievalRun) and stored != run.manifest_sha256:
+        raise EvidenceError("retrieval run reference checksum differs")
+    manifest = _checked(root, {"path": "manifest.json", "sha256": stored})
+    if not isinstance(manifest, dict) or manifest.get("schema_version") != _RUN_SCHEMA:
+        raise EvidenceError("unsupported retrieval run manifest")
+    if manifest.get("run_id") != root.name or manifest.get("pack_id") != RETRIEVAL_PACK_ID:
+        raise EvidenceError("retrieval run identity differs")
+    rows = manifest.get("cases")
+    if not isinstance(rows, list) or manifest.get("case_count") != len(rows):
+        raise EvidenceError("retrieval run case inventory differs")
+    for row in rows:
+        if not isinstance(row, dict) or not isinstance(row.get("artifacts"), dict):
+            raise EvidenceError("retrieval capture inventory is invalid")
+        for entry in row["artifacts"].values():
+            _checked(root, entry)
+    return manifest
+
+
+def _recall(groups: list[Mapping[str, Any]], ids: list[str]) -> float:
+    present = set(ids)
+    required = [group for group in groups if group["required"]]
+    return sum(bool(present.intersection(group["acceptable_chunk_ids"])) for group in required) / len(required)
+
+
+def _case_metrics(case: Mapping[str, Any], validated: Mapping[str, Any] | None) -> dict[str, float | None]:
+    if validated is None:
+        return dict.fromkeys(_QUALITY)
+    refs = case["references"]
+    stages = validated["stages"]
+    recalls = {stage: _recall(refs["evidence_groups"], stages[stage]) for stage in _STAGES}
+    ranking = stages["reranked"][:5]
+    grades = refs["graded_relevance"]
+    dcg = sum((2 ** grades.get(chunk_id, 0) - 1) / math.log2(rank + 1) for rank, chunk_id in enumerate(ranking, 1))
+    ideal = sum((2 ** grade - 1) / math.log2(rank + 1)
+                for rank, grade in enumerate(sorted(grades.values(), reverse=True)[:5], 1))
+    first = next((rank for rank, chunk_id in enumerate(ranking, 1) if grades.get(chunk_id, 0) > 0), None)
+    return {"complete_evidence_success": float(recalls["context"] == 1.0),
+            "candidate_evidence_recall_at_50": recalls["candidate"], "final_evidence_group_recall": recalls["context"],
+            "ndcg_at_5": dcg / ideal, "hit_at_5": float(first is not None), "mrr_at_5": 1 / first if first else 0.0,
+            "fused_evidence_group_recall": recalls["fused"], "reranked_evidence_group_recall": recalls["reranked"]}
+
+
+def _percentile(values: list[float], fraction: float) -> float:
+    ordered = sorted(values)
+    index = (len(ordered) - 1) * fraction
+    lo, hi = math.floor(index), math.ceil(index)
+    return ordered[lo] + (ordered[hi] - ordered[lo]) * (index - lo)
+
+
+def _aggregate(rows: list[Mapping[str, Any]]) -> dict[str, dict[str, Any]]:
+    total = len(rows)
+    result: dict[str, dict[str, Any]] = {}
+    for metric in _QUALITY:
+        values = [row["values"][metric] for row in rows if row["values"][metric] is not None]
+        complete = len(values) == total
+        result[metric] = {"value": statistics.fmean(values) if complete and values else None,
+                          "complete": complete, "eligible_cases": total, "scored_cases": len(values),
+                          "missing_cases": total - len(values), "unit": "ratio", "direction": "higher",
+                          "undefined_reason": None if complete else "missing_or_invalid_capture",
+                          "qualification": _QUALIFICATION}
+    valid = sum(row["status"] == "success" for row in rows)
+    for name, numerator in (("valid_response_coverage", valid), ("failure_rate", total - valid)):
+        result[name] = {"value": numerator / total if total else None, "complete": True,
+                        "eligible_cases": total, "scored_cases": total, "missing_cases": 0,
+                        "unit": "ratio", "direction": "higher" if name == "valid_response_coverage" else "lower",
+                        "undefined_reason": None}
+    timing = [row["adapter_wall_ms"] for row in rows if row["adapter_wall_ms"] is not None]
+    for metric, percentile in (("retrieval_latency_p50_ms", .5), ("retrieval_latency_p95_ms", .95)):
+        complete = len(timing) == total
+        result[metric] = {"value": _percentile(timing, percentile) if complete and timing else None,
+                          "complete": complete, "eligible_cases": total, "scored_cases": len(timing),
+                          "missing_cases": total - len(timing), "unit": "ms", "direction": "lower",
+                          "undefined_reason": None if complete else "adapter_timing_not_observed",
+                          "qualification": "Local adapter wall time; excludes initialization outside callable, no stage attribution."}
+    for metric, reason in (("grounded_answer_success", "planned_answer_grader_not_implemented"),
+                           ("cost_per_evidence_complete_query", "no_runner_verified_cost_observations")):
+        result[metric] = {"value": None, "complete": False, "eligible_cases": total, "scored_cases": 0,
+                          "missing_cases": total, "unit": "ratio" if metric.startswith("grounded") else "USD",
+                          "direction": "higher" if metric.startswith("grounded") else "lower", "undefined_reason": reason}
+    for record in result.values():
+        record.update({"eligible_count": record["eligible_cases"], "observed_count": record["scored_cases"],
+                       "missing_count": record["missing_cases"], "undefined_count": 0})
+        if record.get("undefined_reason") in {"planned_answer_grader_not_implemented", "no_runner_verified_cost_observations", "adapter_timing_not_observed"}:
+            record["status"] = "unavailable"
+        else:
+            record["status"] = "complete" if record["complete"] else "incomplete"
+    return result
+
+
+def grade_retrieval(run: RetrievalRun | str | Path, dataset_path: str | Path, *, expected_sha256: str | None = None) -> RetrievalEvaluation:
+    """Regrade only verified captures and persist a fresh immutable evaluation."""
+    pack = load_retrieval_pack(dataset_path, expected_sha256=expected_sha256)
+    root = (run.path if isinstance(run, RetrievalRun) else Path(run)).expanduser().resolve()
+    manifest = verify_retrieval_run(run)
+    for field in ("dataset_digest", "corpus_digest", "cohort_digest", "labels_digest"):
+        if manifest.get(field) != getattr(pack, field):
+            raise ContractError("retrieval run requires the exact matching dataset, corpus, cohort and labels")
+    if manifest.get("metric_release") != RETRIEVAL_METRIC_RELEASE or manifest.get("trust_tier") != "local_unverified":
+        raise EvidenceError("unsupported retrieval metric release or trust identity")
+    policy = manifest.get("execution_policy")
+    if not isinstance(policy, dict) or policy != _policy(policy.get("max_context_bytes")):
+        raise EvidenceError("retrieval execution policy differs")
+    if manifest.get("candidate") != _candidate(manifest.get("candidate", {})):
+        raise EvidenceError("candidate configuration digest differs")
+    if manifest.get("provenance") not in {"caller_supplied", "adapter_observed"}:
+        raise EvidenceError("retrieval capture provenance is invalid")
+    captures = manifest["cases"]
+    if len(captures) != len(pack.cases):
+        raise EvidenceError("retrieval run does not cover the complete cohort")
+    rows: list[dict[str, Any]] = []
+    for case, capture in zip(pack.cases, captures):
+        if any(capture.get(field) != case[field] for field in ("query_id", "customer", "cluster_id")):
+            raise EvidenceError("retrieval capture identity or order differs from cohort")
+        request = build_retrieval_request(pack, case, max_context_bytes=policy["max_context_bytes"])
+        artifacts = capture["artifacts"]
+        if "request" not in artifacts or _checked(root, artifacts["request"]) != request:
+            raise EvidenceError("saved request differs from label-free pinned input")
+        status = capture.get("status")
+        validated = None
+        if status == "success":
+            if "raw_response" not in artifacts or "validated_response" not in artifacts:
+                raise EvidenceError("successful capture lacks raw and validated response")
+            validated = _response(_checked(root, artifacts["raw_response"]), request)
+            if validated != _checked(root, artifacts["validated_response"]):
+                raise EvidenceError("saved stages/context differ from raw response")
+        elif status == "invalid_response":
+            if "validated_response" in artifacts or "raw_response" not in artifacts:
+                raise EvidenceError("invalid capture has an inconsistent artifact inventory")
+            try:
+                _response(_checked(root, artifacts["raw_response"]), request)
+            except ContractError:
+                pass
+            else:
+                raise EvidenceError("invalid capture contains a valid response")
+        elif status in {"missing_response", "candidate_error"}:
+            if "validated_response" in artifacts or "raw_response" in artifacts:
+                raise EvidenceError("failed capture contains a response")
+        else:
+            raise EvidenceError("unsupported capture status")
+        elapsed = capture.get("adapter_wall_ms")
+        if elapsed is not None and (manifest["provenance"] != "adapter_observed" or type(elapsed) not in (int, float)
+                                    or not math.isfinite(elapsed) or elapsed < 0):
+            raise EvidenceError("capture claims unsupported timing observations")
+        rows.append({"query_id": case["query_id"], "customer": case["customer"], "cluster_id": case["cluster_id"],
+                     "status": status, "values": _case_metrics(case, validated), "adapter_wall_ms": elapsed,
+                     "stage_counts": {stage: len(validated["stages"][stage]) for stage in _STAGES} if validated else None,
+                     "context_bytes": validated["context_bytes"] if validated else None})
+    evaluation_id = "eval-" + uuid.uuid4().hex
+    data = {"schema_version": _EVAL_SCHEMA, "evaluation_id": evaluation_id, "run_id": manifest["run_id"],
+            "pack_id": RETRIEVAL_PACK_ID, "status": "completed" if all(row["status"] == "success" for row in rows) else "incomplete",
+            **{field: getattr(pack, field) for field in ("dataset_digest", "corpus_digest", "cohort_digest", "labels_digest")},
+            "origin": pack.metadata["origin"], "label_status": pack.metadata["label_status"],
+            "label_basis": pack.metadata["label_basis"], "unjudged_policy": pack.metadata["unjudged_policy"],
+            "development_only": True, "qualification": _QUALIFICATION, "trust_tier": "local_unverified",
+            "metric_semantics": _QUALIFICATION + " Complete-evidence means complete known anchor groups, not reviewed answer sufficiency. Context uses a UTF-8 byte budget proxy, not model tokens.",
+            "candidate": manifest["candidate"], "execution_policy": policy, "runner": manifest["runner"],
+            "source_run_manifest_sha256": (root / "manifest.sha256").read_text(encoding="ascii").strip(),
+            "provenance": manifest["provenance"], "runner_metadata": manifest["runner_metadata"],
+            "metric_release": RETRIEVAL_METRIC_RELEASE, "case_count": len(rows), "cases": rows,
+            "success_count": sum(row["status"] == "success" for row in rows),
+            "failure_count": sum(row["status"] != "success" for row in rows),
+            "metrics": _aggregate(rows), "customer_slices": {customer: {"case_count": sum(row["customer"] == customer for row in rows),
+                      "metrics": _aggregate([row for row in rows if row["customer"] == customer])}
+                      for customer in sorted({row["customer"] for row in rows})}}
+    evaluation_path = root / "evaluations" / evaluation_id
+    digest = _write(evaluation_path / "evaluation.json", data)
+    with (evaluation_path / "evaluation.sha256").open("x", encoding="ascii") as stream:
+        stream.write(digest + "\n")
+    return RetrievalEvaluation(evaluation_id, evaluation_path, digest, data)
+
+
+def _read_evaluation(evaluation: RetrievalEvaluation | str | Path) -> dict[str, Any]:
+    path = evaluation.path if isinstance(evaluation, RetrievalEvaluation) else Path(evaluation).expanduser().resolve()
+    try:
+        stored = (path / "evaluation.sha256").read_text(encoding="ascii").strip()
+    except OSError as exc:
+        raise EvidenceError("evaluation checksum is missing") from exc
+    if isinstance(evaluation, RetrievalEvaluation) and stored != evaluation.artifact_sha256:
+        raise EvidenceError("evaluation reference checksum differs")
+    data = _checked(path, {"path": "evaluation.json", "sha256": stored})
+    if not isinstance(data, dict) or data.get("schema_version") != _EVAL_SCHEMA or data.get("evaluation_id") != path.name:
+        raise EvidenceError("evaluation schema or identity differs")
+    if isinstance(evaluation, RetrievalEvaluation) and data != evaluation.to_dict():
+        raise EvidenceError("in-memory evaluation differs from its verified artifact")
+    # An evaluation remains tied to its source captures, not only its cached
+    # aggregate bytes. A changed source artifact also invalidates comparison.
+    run_root = path.parent.parent
+    run_manifest = verify_retrieval_run(RetrievalRun(data["run_id"], run_root, data["source_run_manifest_sha256"]))
+    for field in ("corpus_digest", "cohort_digest", "labels_digest", "candidate", "execution_policy", "runner", "provenance"):
+        if data.get(field) != run_manifest.get(field):
+            raise EvidenceError("evaluation does not match its source run")
+    return data
+
+
+def _bootstrap(pairs: list[dict[str, Any]], *, samples: int = 2000) -> dict[str, Any]:
+    groups: dict[tuple[str, str], list[float]] = {}
+    for pair in pairs:
+        groups.setdefault((pair["customer"], pair["cluster_id"]), []).append(pair["delta_b_minus_a"])
+    customers = sorted({key[0] for key in groups})
+    keys_by_customer = {customer: sorted(key for key in groups if key[0] == customer) for customer in customers}
+    if not groups:
+        return {"lower": None, "upper": None, "confidence": .95, "samples": 0, "cluster_count": 0}
+    rng = random.Random(0)
+    values = []
+    for _ in range(samples):
+        # Resample whole families within each customer, retaining the original
+        # customer mixture even when family sizes differ.
+        weighted = 0.0
+        for customer, keys in keys_by_customer.items():
+            sampled = [delta for _ in keys for delta in groups[keys[rng.randrange(len(keys))]]]
+            weight = sum(len(groups[key]) for key in keys) / len(pairs)
+            weighted += statistics.fmean(sampled) * weight
+        values.append(weighted)
+    return {"lower": _percentile(values, .025), "upper": _percentile(values, .975),
+            "confidence": .95, "samples": samples, "cluster_count": len(groups), "seed": 0,
+            "unit": "paired_cluster_bootstrap_stratified_by_customer", "qualification": "Development cohort uncertainty; not population generalization."}
+
+
+def compare_retrieval(a: RetrievalEvaluation | str | Path, b: RetrievalEvaluation | str | Path) -> dict[str, Any]:
+    """Pair quality across fixed corpus/cohort/labels, allowing different pools.
+
+    Timing deltas additionally require matching measurement scopes and declared
+    environment metadata. Existing frozen-pool reranker guards are unchanged.
+    """
+    left, right = _read_evaluation(a), _read_evaluation(b)
+    fields = ("pack_id", "corpus_digest", "cohort_digest", "labels_digest", "metric_release", "execution_policy",
+              "label_basis", "label_status", "unjudged_policy", "trust_tier", "runner")
+    for field in fields:
+        if left.get(field) != right.get(field):
+            raise ContractError("retrieval comparisons require matching " + field)
+    if [row["query_id"] for row in left["cases"]] != [row["query_id"] for row in right["cases"]]:
+        raise ContractError("retrieval comparisons require the complete ordered cohort")
+    timing_metadata = left["runner_metadata"]
+    timing_ok = (left["provenance"] == right["provenance"] == "adapter_observed"
+                 and timing_metadata == right["runner_metadata"]
+                 and isinstance(timing_metadata, dict)
+                 and bool(timing_metadata.get("environment")) and bool(timing_metadata.get("timing_scope")))
+    metrics: dict[str, Any] = {}
+    for metric in _QUALITY:
+        pairs = [{"query_id": x["query_id"], "customer": x["customer"], "cluster_id": x["cluster_id"],
+                  "a": x["values"][metric], "b": y["values"][metric],
+                  "delta_b_minus_a": y["values"][metric] - x["values"][metric]}
+                 for x, y in zip(left["cases"], right["cases"])
+                 if x["values"][metric] is not None and y["values"][metric] is not None]
+        av, bv = left["metrics"][metric]["value"], right["metrics"][metric]["value"]
+        complete = len(pairs) == left["case_count"]
+        metrics[metric] = {"a": av, "b": bv, "delta_b_minus_a": bv - av if av is not None and bv is not None else None,
+                           "paired_case_count": len(pairs), "eligible_case_count": left["case_count"], "complete": complete,
+                           "paired_cases": pairs, "paired_cluster_bootstrap_95_ci": _bootstrap(pairs) if complete else {
+                               "lower": None, "upper": None, "undefined_reason": "incomplete_paired_cohort"}}
+    for metric in ("retrieval_latency_p50_ms", "retrieval_latency_p95_ms"):
+        av, bv = left["metrics"][metric]["value"], right["metrics"][metric]["value"]
+        metrics[metric] = {"a": av, "b": bv, "delta_b_minus_a": bv - av if timing_ok and av is not None and bv is not None else None,
+                           "comparison_eligible": timing_ok,
+                           "undefined_reason": None if timing_ok and av is not None and bv is not None else "timing_scope_or_observation_mismatch"}
+    return {"schema_version": "sphynx.rag.retrieve.comparison.v1", "pack_id": RETRIEVAL_PACK_ID,
+            "a_evaluation_id": left["evaluation_id"], "b_evaluation_id": right["evaluation_id"],
+            "corpus_digest": left["corpus_digest"], "cohort_digest": left["cohort_digest"], "labels_digest": left["labels_digest"],
+            "case_count": left["case_count"], "qualification": _QUALIFICATION, "development_only": True,
+            "metrics": metrics, "customer_slices": {customer: {metric: {
+                "a": left["customer_slices"][customer]["metrics"][metric]["value"],
+                "b": right["customer_slices"][customer]["metrics"][metric]["value"],
+                "delta_b_minus_a": right["customer_slices"][customer]["metrics"][metric]["value"] - left["customer_slices"][customer]["metrics"][metric]["value"]
+                    if left["customer_slices"][customer]["metrics"][metric]["value"] is not None
+                    and right["customer_slices"][customer]["metrics"][metric]["value"] is not None else None}
+                for metric in _QUALITY} for customer in left["customer_slices"]}}