sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/results.py

@@ -0,0 +1,852 @@
+"""Immutable local grading, scorecards, and fair paired comparisons.
+
+Only the installed illustrative RAG reranking pack is gradable in this release.
+Candidate-visible captures contain no reference labels; this module loads the
+separate public references after validating the saved runner evidence.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import uuid
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Mapping
+
+from .comparison_core import compare_validated_records
+from .assessment_contracts import CaseView, EvidenceView, EvaluatorContext, _verified_evaluator_context
+from .contracts import (
+    PACK_ID, CaseReferences, EvidenceGroup, ReferenceCandidate,
+    build_candidate_request, canonical_json,
+)
+from .evidence import (
+    RUN_SCHEMA_VERSION,
+    EvidenceError,
+    RunRef,
+    read_capture_artifact,
+    read_capture_manifest,
+    sha256_bytes,
+    verify_run,
+    write_json,
+    write_jsonl,
+)
+from .fast_search import FastSearchSmokeResult
+from .metrics import (
+    GRADER_RELEASE,
+    METRIC_RELEASE,
+    CaseMetricResult,
+    MetricAggregate,
+    aggregate_case_metrics,
+    grade_case,
+    metric_definitions,
+)
+from .rag import load_rag_rerank_pack
+from .records import EVALUATION_SCHEMA_VERSION
+from .runner import ENV_POLICY_ID, MAX_STREAM_BYTES, _case_snapshot, _parse_response
+
+METRICS_SCHEMA_VERSION = "sphynx.metrics.v1"
+JUDGMENT_SCHEMA_VERSION = "sphynx.judgment.v1"
+_EVALUATION_FILES = frozenset({"judgments.jsonl", "metrics.json"})
+_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
+_RUN_ID = re.compile(r"run-[0-9a-f]{32}\Z")
+_EVAL_ID = re.compile(r"eval-[0-9a-f]{32}\Z")
+_GRADABLE = frozenset({"completed", "incomplete"})
+_METRIC_IDS = tuple(definition.metric_id for definition in metric_definitions())
+
+
+class ResultError(ValueError):
+    """An evaluation or comparison cannot be read or trusted."""
+
+
+class _Ungradable(Exception):
+    def __init__(self, status: str, reason: str):
+        self.status = status
+        self.reason = reason
+        super().__init__(reason)
+
+
+@dataclass(frozen=True)
+class EvaluationRef:
+    """Reference to a published evaluation, or a wholly unreadable root."""
+
+    evaluation_id: str | None
+    path: Path | None
+    run_id: str | None
+    pack_id: str | None
+    status: str
+    reason: str | None = None
+
+
+@dataclass(frozen=True)
+class Scorecard:
+    data: dict[str, Any]
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(self.to_json())
+
+    def to_json(self) -> str:
+        return canonical_json(self.data)
+
+    def to_markdown(self) -> str:
+        data = self.data
+        counts = data["counts"]
+        releases = data["releases"]
+        lines = [
+            "# Sphynx illustrative RAG reranking scorecard",
+            "",
+            "**LOCAL_UNVERIFIED — synthetic/illustrative cases. This is not a validated benchmark or full-agent answer-quality result.**",
+            "",
+            f"- Evaluation: `{data['evaluation_id']}`; run: `{data['run_id']}`",
+            f"- Candidate: `{data['candidate']['name']}` version `{data['candidate']['version']}`",
+            f"- Pack: `{data['pack_id']}` release `{releases['pack_release']}`; track: `{data['track']}`",
+            f"- Cases: `{releases['dataset_id']}` version `{releases['dataset_version']}`; schema `{releases['case_schema_version']}`",
+            f"- Grader: `{releases['grader_release']}`; metrics: `{releases['metric_release']}`",
+            f"- Status: **{data['status']}**" + (f" — {data['reason']}" if data['reason'] else ""),
+            f"- Attempts: {counts['attempted_count'] if counts['attempted_count'] is not None else 'unverified'}; "
+            f"successful: {counts['success_count'] if counts['success_count'] is not None else 'unverified'}; "
+            f"failed: {counts['failure_count'] if counts['failure_count'] is not None else 'unverified'}",
+            f"- Primary quality coverage: {counts['observed_count']}/{counts['eligible_count']} eligible; "
+            f"missing {counts['missing_count']}; undefined {counts['undefined_count']}",
+            "",
+            "| Metric | Role | Value | Eligible | Observed | Missing | Undefined | Complete |",
+            "| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |",
+        ]
+        for definition in data["metric_definitions"]:
+            metric = data["metrics"][definition["metric_id"]]
+            display_values = [(definition["metric_id"], metric["value"], "")]
+            if definition["metric_id"] == "runner_latency_ms":
+                display_values = [
+                    ("Runner latency p50", metric["value"], " ms"),
+                    ("Runner latency p95", metric["p95"], " ms"),
+                ]
+            for label, number, unit in display_values:
+                value = "null" if number is None else f"{number:.6f}{unit}"
+                lines.append(
+                    f"| {label} | {'primary' if definition['primary'] else 'diagnostic'} | "
+                    f"{value} | {metric['eligible_count']} | {metric['observed_count']} | "
+                    f"{metric['missing_count']} | {metric['undefined_count']} | {str(metric['complete']).lower()} |"
+                )
+        lines.extend(["", "A null quality value means the eligible cohort is incomplete; missing cases are never scored as zero."])
+        return "\n".join(lines) + "\n"
+
+
+@dataclass(frozen=True)
+class Comparison:
+    data: dict[str, Any]
+
+    @property
+    def compatible(self) -> bool:
+        return self.data["compatible"]
+
+    @property
+    def reason(self) -> str | None:
+        return self.data["reason"]
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(self.to_json())
+
+    def to_json(self) -> str:
+        return canonical_json(self.data)
+
+    def to_markdown(self) -> str:
+        data = self.data
+        lines = [
+            "# Sphynx illustrative paired RAG reranking comparison",
+            "",
+            "**LOCAL_UNVERIFIED — synthetic/illustrative cases. No full-agent or overall RAG claim.**",
+            "",
+            f"- Evaluation A: `{data['evaluation_a']}`",
+            f"- Evaluation B: `{data['evaluation_b']}`",
+            f"- Compatible: **{str(data['compatible']).lower()}**",
+        ]
+        if not data["compatible"]:
+            lines.append(f"- Reason: {data['reason']}")
+        else:
+            lines.extend([
+                "",
+                "| Metric | Complete paired mean B − A | Observed pairs | Missing pairs | Undefined |",
+                "| --- | ---: | ---: | ---: | ---: |",
+            ])
+            for metric_id, item in data["metrics"].items():
+                value = "null" if item["mean_delta"] is None else f"{item['mean_delta']:.6f}"
+                lines.append(
+                    f"| {metric_id} | {value} | {item['paired_count']} | "
+                    f"{item['missing_pair_count']} | {item['undefined_count']} |"
+                )
+            lines.extend([
+                "",
+                "Primary nDCG@5 per-case pairs:",
+                "",
+                "| Case | A | B | B − A | Reason |",
+                "| --- | ---: | ---: | ---: | --- |",
+            ])
+            for item in data["metrics"]["ndcg_at_5"]["per_case"]:
+                display = lambda value: "null" if value is None else f"{value:.6f}"
+                lines.append(
+                    f"| {item['case_id']} | {display(item['value_a'])} | {display(item['value_b'])} | "
+                    f"{display(item['delta_b_minus_a'])} | {item['reason'] or ''} |"
+                )
+            lines.extend(["", "JSON includes paired per-case values and missing reasons for every metric."])
+        return "\n".join(lines) + "\n"
+
+
+def _json_object(path: Path) -> dict[str, Any]:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise ResultError(f"cannot read JSON {path}: {exc}") from exc
+    if not isinstance(value, dict):
+        raise ResultError(f"JSON {path} must be an object")
+    return value
+
+
+def _expected_cases(pack: Any) -> list[dict[str, Any]]:
+    return [
+        {
+            "dataset_id": case.dataset_id,
+            "dataset_version": case.dataset_version,
+            "case_id": case.case_id,
+            "query_id": case.query_id,
+            "pool_digest": case.pool_digest,
+            "split": case.split,
+            "split_group_id": case.split_group_id,
+            "cluster_id": case.cluster_id,
+            "suite_type": case.suite_type,
+            "versions": case.versions.to_dict(),
+        }
+        for case in pack.cases
+    ]
+
+
+def _expected_policy(pack: Any) -> dict[str, Any]:
+    policy = pack.descriptor.execution_policy
+    return {
+        "policy_version": policy.policy_version,
+        "trials": policy.trials,
+        "per_case_timeout_seconds": policy.per_case_timeout_seconds,
+        "resource_policy_id": policy.resource_policy_id,
+        "environment_policy_id": ENV_POLICY_ID,
+        "stdout_stderr_retention_limit_bytes_each": MAX_STREAM_BYTES,
+    }
+
+
+def _identity(manifest: Mapping[str, Any], pack: Any) -> dict[str, Any]:
+    return {
+        "pack_id": PACK_ID,
+        "pack_release": pack.descriptor.pack_release,
+        "track": pack.descriptor.track,
+        "cases": _expected_cases(pack),
+        "execution_policy": manifest.get("execution_policy") if isinstance(manifest.get("execution_policy"), dict) else _expected_policy(pack),
+        "runner": manifest.get("runner") if isinstance(manifest.get("runner"), dict) else {},
+        "trust_tier": "local_unverified",
+    }
+
+
+def _check_root(ref: RunRef, manifest: dict[str, Any], pack: Any) -> None:
+    if manifest.get("schema_version") != RUN_SCHEMA_VERSION:
+        raise _Ungradable("unsupported_schema", "unsupported run schema_version")
+    if manifest.get("pack_id") != PACK_ID or manifest.get("track") != pack.descriptor.track:
+        raise _Ungradable("incompatible", "run pack or track differs from installed RAG rerank pack")
+    if manifest.get("pack_release") != pack.descriptor.pack_release:
+        raise _Ungradable("incompatible", "run pack release differs from installed pack")
+    if manifest.get("cases") != _expected_cases(pack):
+        raise _Ungradable("incompatible", "run case order, pool, corpus, or cohort differs from installed pack")
+    policy = manifest.get("execution_policy")
+    if isinstance(policy, dict) and policy.get("trials") != 1:
+        raise _Ungradable("unsupported_schema", "only one pack-owned trial per case is supported")
+    if policy != _expected_policy(pack):
+        raise _Ungradable("incompatible", "run execution policy or resource/environment identity differs from installed pack")
+    if manifest.get("trust_tier") != "local_unverified" or manifest.get("provenance") != "runner_observed":
+        raise _Ungradable("incompatible", "run trust tier or provenance is unsupported")
+    candidate = manifest.get("candidate")
+    if not isinstance(candidate, dict) or set(candidate) != {"name", "version", "interface_version", "argv_sha256"}:
+        raise _Ungradable("incompatible", "run candidate identity is missing or malformed")
+    if any(not isinstance(candidate[key], str) or not candidate[key] for key in ("name", "version")):
+        raise _Ungradable("incompatible", "run candidate name or version is malformed")
+    if candidate.get("interface_version") != pack.descriptor.interface_version or not _DIGEST.fullmatch(str(candidate.get("argv_sha256"))):
+        raise _Ungradable("incompatible", "run candidate interface or argv identity is unsupported")
+    runner = manifest.get("runner")
+    if not isinstance(runner, dict) or set(runner) != {"release", "python_version", "platform", "clock_domain"}:
+        raise _Ungradable("incompatible", "run runner environment identity is missing")
+    if runner["release"] != "v1" or runner["clock_domain"] != "python.monotonic_ns.process_local":
+        raise _Ungradable("unsupported_schema", "run runner release or timing clock is unsupported")
+    if not runner["python_version"] or not runner["platform"]:
+        raise _Ungradable("incompatible", "run runner environment identity is empty")
+    try:
+        verify_run(ref)
+    except EvidenceError as exc:
+        raise _Ungradable("corrupt", f"run evidence failed integrity verification: {exc}") from exc
+
+
+def _grade_rerank_context(context: EvaluatorContext, case: Any, pack: Any) -> CaseMetricResult:
+    """Adapt a verified, grader-owned context to the unchanged native evaluator.
+
+    Domain schemas stay here. The common context neither validates RAG payloads
+    nor grants reference access. Native grade_case checks the exact case and
+    reference identities against the installed pack again.
+    """
+    labels = context.references
+    references = CaseReferences(
+        case_id=labels["case_id"], label_authority=labels["label_authority"],
+        candidates=tuple(ReferenceCandidate(**dict(item)) for item in labels["candidates"]),
+        evidence_groups=tuple(EvidenceGroup(**dict(item)) for item in labels["evidence_groups"]),
+    )
+    return grade_case(
+        case, references, context.outputs["ranked_candidate_ids"],
+        context.outputs["runner_latency_ms"],
+        terminal_status=context.outputs["terminal_status"],
+        relevance_threshold=context.metadata["relevance_threshold"],
+    )
+
+
+def _capture_results(ref: RunRef, manifest: dict[str, Any], pack: Any) -> tuple[list[CaseMetricResult], list[dict[str, Any]], int]:
+    capture_ids = manifest["capture_ids"]
+    if len(capture_ids) != len(pack.cases):
+        raise EvidenceError("capture workload does not match one trial for every installed case")
+    results: list[CaseMetricResult] = []
+    judgments: list[dict[str, Any]] = []
+    successes = 0
+    for case, capture_id in zip(pack.cases, capture_ids):
+        capture = read_capture_manifest(ref, capture_id)
+        if capture.get("source") != "runner_observed":
+            raise EvidenceError(f"capture provenance is not runner observed: {capture_id}")
+        capture_identity = capture.get("identity")
+        if not isinstance(capture_identity, dict) or any((
+            capture_identity.get("run_id") != ref.run_id,
+            capture_identity.get("capture_id") != capture_id,
+            capture_identity.get("case_id") != case.case_id,
+            capture_identity.get("pool_digest") != case.pool_digest,
+            capture_identity.get("trial_index") != 1,
+            capture_identity.get("attempt_index") != 1,
+            capture_identity.get("trust_tier") != "local_unverified",
+        )):
+            raise EvidenceError(f"capture order, case, pool, or trial identity mismatch: {capture_id}")
+        request = build_candidate_request(case, PACK_ID)
+        expected_request = (request.to_json() + "\n").encode("utf-8")
+        expected_snapshot = (canonical_json(_case_snapshot(case)) + "\n").encode("utf-8")
+        if read_capture_artifact(ref, capture_id, "candidate_request") != expected_request:
+            raise EvidenceError(f"candidate request differs from installed case: {capture_id}")
+        if read_capture_artifact(ref, capture_id, "case_snapshot") != expected_snapshot:
+            raise EvidenceError(f"case snapshot differs from installed case: {capture_id}")
+        coverage = _json_object(ref.path / "captures" / capture_id / "coverage.json")
+        states = {item.get("capability_id"): item.get("state") for item in coverage.get("capabilities", []) if isinstance(item, dict)}
+        if any(states.get(item) != "available" for item in ("candidate_input", "case_snapshot", "runner_timing")):
+            raise EvidenceError(f"required case input or runner timing unavailable: {capture_id}")
+        timing = capture.get("timing")
+        latency_ns = timing.get("candidate_latency_ns") if isinstance(timing, dict) else None
+        if latency_ns is not None and (type(latency_ns) is not int or latency_ns < 0 or latency_ns > timing["elapsed_ns"]):
+            raise EvidenceError(f"candidate latency is invalid: {capture_id}")
+        latency_ms = latency_ns / 1_000_000 if latency_ns is not None else None
+        status = capture["status"]
+        ranked: tuple[str, ...] | None = None
+        if status == "success":
+            if states.get("candidate_stdout_raw") != "available" or states.get("candidate_response_validated") != "available":
+                raise EvidenceError(f"successful capture lacks validated raw stdout: {capture_id}")
+            stdout = read_capture_artifact(ref, capture_id, "candidate_stdout")
+            ranked, error_code, error_reason = _parse_response(stdout, request)
+            if error_code is not None or ranked is None:
+                raise EvidenceError(f"saved success response cannot be reconstructed: {capture_id}: {error_code}: {error_reason}")
+            if capture.get("ranked_candidate_ids") != list(ranked) or capture.get("failure_code") is not None:
+                raise EvidenceError(f"saved ranking or success status differs from raw stdout: {capture_id}")
+            if capture.get("exit_code") != 0 or latency_ms is None:
+                raise EvidenceError(f"saved success exit/timing is invalid: {capture_id}")
+            successes += 1
+        elif capture.get("ranked_candidate_ids") is not None or not isinstance(capture.get("failure_code"), str):
+            raise EvidenceError(f"failed capture has a ranking or lacks a failure code: {capture_id}")
+        elif status == "invalid_response":
+            # This outcome follows a completed, zero-exit process with a
+            # complete stdout capture. Re-derive its failure from those bytes;
+            # otherwise a rehashed manifest can hide a valid response.
+            if states.get("candidate_stdout_raw") != "available" or states.get("candidate_response_validated") != "not_captured":
+                raise EvidenceError(f"invalid response lacks complete raw stdout or has contradictory coverage: {capture_id}")
+            if capture.get("exit_code") != 0 or latency_ms is None:
+                raise EvidenceError(f"invalid response exit/timing is inconsistent: {capture_id}")
+            stdout = read_capture_artifact(ref, capture_id, "candidate_stdout")
+            parsed_ranking, error_code, _ = _parse_response(stdout, request)
+            if parsed_ranking is not None or error_code != capture["failure_code"]:
+                raise EvidenceError(f"invalid response status differs from raw stdout: {capture_id}")
+        try:
+            artifacts = [json.loads(line) for line in
+                         (ref.path / "captures" / capture_id / "artifacts.jsonl").read_text(encoding="utf-8").splitlines()]
+            context = _verified_evaluator_context(
+                case=CaseView(case.case_id, request.to_dict()["schema_version"], request.to_dict()),
+                outputs={"ranked_candidate_ids": ranked, "runner_latency_ms": latency_ms,
+                         "terminal_status": "ok" if status == "success" else status},
+                references=asdict(pack.reference_for(case.case_id)),
+                evidence=tuple(EvidenceView(
+                    evidence_id=capture_id + ":" + artifact["role"],
+                    sha256=artifact["sha256"], media_type=artifact["media_type"],
+                    role=artifact["role"], source=artifact["source"], visibility="local_owner",
+                    schema_version=artifact["schema_version"], case_id=case.case_id,
+                    availability=("missing" if not artifact["observed"] else
+                                  "available" if artifact["complete"] else "truncated"),
+                    metadata={"capture_id": capture_id, "retained_bytes": artifact["retained_bytes"],
+                              "observed_bytes": artifact["observed_bytes"]},
+                ) for artifact in artifacts),
+                run_id=ref.run_id, trust_tier=capture_identity["trust_tier"],
+                verified_by="sphynx.results._capture_results",
+                metadata={"relevance_threshold": pack.relevance_threshold,
+                          "capture_id": capture_id, "trial": 1, "attempt": 1},
+            )
+            result = _grade_rerank_context(context, case, pack)
+        except Exception as exc:
+            raise _Ungradable("grader_error", f"case grader failed for {case.case_id}: {type(exc).__name__}: {exc}") from exc
+        results.append(result)
+        judgments.append({
+            "schema_version": JUDGMENT_SCHEMA_VERSION,
+            "run_id": ref.run_id,
+            "capture_id": capture_id,
+            "case_id": case.case_id,
+            "pool_digest": case.pool_digest,
+            "trial_index": 1,
+            "status": "scored" if status == "success" else "candidate_failure",
+            "reason": None if status == "success" else capture.get("failure_reason") or capture["failure_code"],
+            "candidate_status": status,
+            "failure_code": capture.get("failure_code"),
+            "answerable": result.answerable,
+            "values": dict(result.values),
+            "undefined_reasons": dict(result.undefined_reasons),
+        })
+    return results, judgments, successes
+
+
+def _verify_gradable_run(ref: RunRef) -> None:
+    """Read-only semantic gate shared by grading and snapshot sync."""
+
+    manifest = _json_object(ref.path / "manifest.json")
+    pack = load_rag_rerank_pack()
+    _check_root(ref, manifest, pack)
+    _capture_results(ref, manifest, pack)
+
+
+def _non_gradable(pack: Any, status: str, reason: str, run_id: str) -> tuple[list[CaseMetricResult], list[dict[str, Any]]]:
+    results: list[CaseMetricResult] = []
+    judgments: list[dict[str, Any]] = []
+    for case in pack.cases:
+        answerable = any(group.required for group in pack.reference_for(case.case_id).evidence_groups)
+        values = tuple((metric_id, None) for metric_id in _METRIC_IDS)
+        reasons = tuple(
+            (metric_id, "runner_span_not_captured" if metric_id == "runner_latency_ms"
+             else f"candidate_{status}" if answerable else "unsupported_query")
+            for metric_id in _METRIC_IDS
+        )
+        result = CaseMetricResult(case.case_id, case.pool_digest, status, answerable, values, reasons)
+        results.append(result)
+        judgments.append({
+            "schema_version": JUDGMENT_SCHEMA_VERSION,
+            "run_id": run_id,
+            "capture_id": None,
+            "case_id": case.case_id,
+            "pool_digest": case.pool_digest,
+            "trial_index": 1,
+            "status": status,
+            "reason": reason,
+            "candidate_status": None,
+            "failure_code": None,
+            "answerable": result.answerable,
+            "values": dict(result.values),
+            "undefined_reasons": dict(result.undefined_reasons),
+        })
+    return results, judgments
+
+
+def _publish(ref: RunRef, manifest: dict[str, Any], pack: Any, status: str, reason: str | None,
+             results: list[CaseMetricResult], judgments: list[dict[str, Any]], successes: int | None) -> EvaluationRef:
+    evaluation_id = "eval-" + uuid.uuid4().hex
+    for judgment in judgments:
+        judgment["evaluation_id"] = evaluation_id
+    try:
+        aggregates = aggregate_case_metrics(results)
+    except Exception:
+        if status != "grader_error":
+            raise
+        # A broken grading function must still leave an explicit null result.
+        # It is verified again before sync/report when the grader is available.
+        aggregates = {}
+        answerable_count = sum(any(group.required for group in pack.reference_for(case.case_id).evidence_groups)
+                               for case in pack.cases)
+        for definition in metric_definitions():
+            eligible = len(pack.cases) if definition.metric_id == "runner_latency_ms" else answerable_count
+            aggregates[definition.metric_id] = MetricAggregate(
+                definition.metric_id, definition.release, definition.unit, definition.direction,
+                None, None, eligible, 0, eligible, len(pack.cases) - eligible, False,
+            )
+    primary = aggregates["ndcg_at_5"]
+    counts = {
+        "attempted_count": len(pack.cases) if successes is not None else None,
+        "success_count": successes,
+        "failure_count": len(pack.cases) - successes if successes is not None else None,
+        "eligible_count": primary.eligible_count,
+        "observed_count": primary.observed_count,
+        "missing_count": primary.missing_count,
+        "undefined_count": primary.undefined_count,
+    }
+    candidate = manifest.get("candidate") if isinstance(manifest.get("candidate"), dict) else {}
+    candidate = {
+        key: candidate.get(key) if isinstance(candidate.get(key), str) else "unknown"
+        for key in ("name", "version", "interface_version", "argv_sha256")
+    }
+    first = pack.cases[0]
+    releases = {
+        "pack_release": pack.descriptor.pack_release,
+        "case_schema_version": pack.descriptor.case_schema_version,
+        "dataset_id": first.dataset_id,
+        "dataset_version": first.dataset_version,
+        "dataset_sha256": pack.dataset_digest,
+        "grader_release": GRADER_RELEASE,
+        "metric_release": METRIC_RELEASE,
+    }
+    metric_data = {
+        "schema_version": METRICS_SCHEMA_VERSION,
+        "run_id": ref.run_id,
+        "evaluation_id": evaluation_id,
+        "pack_id": PACK_ID,
+        "track": pack.descriptor.track,
+        "status": status,
+        "reason": reason,
+        "trust_tier": "local_unverified",
+        "illustrative": True,
+        "candidate": candidate,
+        "releases": releases,
+        "identity": _identity(manifest, pack),
+        "counts": counts,
+        "metrics": {key: value.to_dict() for key, value in aggregates.items()},
+        "metric_definitions": [definition.to_dict() for definition in metric_definitions()],
+    }
+    stage = ref.path / (".staging-" + evaluation_id)
+    published = ref.path / "evaluations" / evaluation_id
+    stage.mkdir(parents=False, exist_ok=False)
+    try:
+        judgments_hash = write_jsonl(stage / "judgments.jsonl", judgments)
+        metrics_hash = write_json(stage / "metrics.json", metric_data)
+        files = {
+            "judgments.jsonl": {"sha256": judgments_hash, "size_bytes": (stage / "judgments.jsonl").stat().st_size},
+            "metrics.json": {"sha256": metrics_hash, "size_bytes": (stage / "metrics.json").stat().st_size},
+        }
+        evaluation_manifest = {
+            "schema_version": EVALUATION_SCHEMA_VERSION,
+            "run_id": ref.run_id,
+            "evaluation_id": evaluation_id,
+            "pack_id": PACK_ID,
+            "status": status,
+            "reason": reason,
+            "pack_release": pack.descriptor.pack_release,
+            "grader_release": GRADER_RELEASE,
+            "metric_release": METRIC_RELEASE,
+            "trust_tier": "local_unverified",
+            "files": files,
+        }
+        write_json(stage / "manifest.json", evaluation_manifest)
+        published.parent.mkdir(exist_ok=True)
+        os.replace(stage, published)
+    finally:
+        if stage.exists():
+            for child in stage.iterdir():
+                if child.is_file():
+                    child.unlink()
+            stage.rmdir()
+    return EvaluationRef(evaluation_id, published.resolve(), ref.run_id, PACK_ID, status, reason)
+
+
+def grade(run_ref: RunRef | str | Path) -> EvaluationRef:
+    """Regrade saved captures without starting or changing a candidate process."""
+
+    run_path = (run_ref.path if isinstance(run_ref, RunRef) else Path(run_ref)).expanduser().resolve()
+    try:
+        manifest = _json_object(run_path / "manifest.json")
+    except ResultError as exc:
+        return EvaluationRef(None, None, None, None, "corrupt", str(exc))
+    run_id = manifest.get("run_id")
+    if not isinstance(run_id, str) or not _RUN_ID.fullmatch(run_id) or run_path.name != run_id:
+        return EvaluationRef(None, None, None, None, "corrupt", "run identity is unreadable or differs from directory")
+    ref = RunRef(run_id, run_path, manifest.get("pack_id") if isinstance(manifest.get("pack_id"), str) else "unknown")
+    try:
+        pack = load_rag_rerank_pack()
+    except Exception as exc:
+        return EvaluationRef(None, None, run_id, ref.pack_id, "grader_error", f"installed grader/pack could not load: {exc}")
+    status, reason, successes = "completed", None, None
+    try:
+        _check_root(ref, manifest, pack)
+        results, judgments, successes = _capture_results(ref, manifest, pack)
+        if successes != len(pack.cases):
+            status = "incomplete"
+            reason = f"{len(pack.cases) - successes} candidate attempt(s) failed"
+    except _Ungradable as exc:
+        status, reason = exc.status, exc.reason
+        results, judgments = _non_gradable(pack, status, reason, run_id)
+    except (EvidenceError, ResultError, KeyError, TypeError, ValueError) as exc:
+        status, reason = "corrupt", f"capture evidence failed validation: {exc}"
+        results, judgments = _non_gradable(pack, status, reason, run_id)
+    except Exception as exc:
+        status, reason = "grader_error", f"grader failed: {type(exc).__name__}: {exc}"
+        results, judgments = _non_gradable(pack, status, reason, run_id)
+    if status not in _GRADABLE:
+        successes = None
+    try:
+        aggregate_case_metrics(results)
+    except Exception as exc:
+        status, reason, successes = "grader_error", f"grader aggregation failed: {type(exc).__name__}: {exc}", None
+        results, judgments = _non_gradable(pack, status, reason, run_id)
+    return _publish(ref, manifest, pack, status, reason, results, judgments, successes)
+
+
+def open_evaluation(value: EvaluationRef | str | Path) -> EvaluationRef:
+    """Open a published evaluation path without reading candidate code."""
+
+    if isinstance(value, EvaluationRef) and value.path is None:
+        return value
+    path = (value.path if isinstance(value, EvaluationRef) else Path(value)).expanduser().resolve()
+    manifest = _json_object(path / "manifest.json")
+    evaluation_id = manifest.get("evaluation_id")
+    run_id = manifest.get("run_id")
+    if not isinstance(evaluation_id, str) or not _EVAL_ID.fullmatch(evaluation_id) or path.name != evaluation_id:
+        raise ResultError("evaluation ID differs from directory")
+    if not isinstance(run_id, str) or not _RUN_ID.fullmatch(run_id) or path.parent.name != "evaluations" or path.parent.parent.name != run_id:
+        raise ResultError("evaluation run identity differs from directory")
+    return EvaluationRef(evaluation_id, path, run_id, manifest.get("pack_id"), manifest.get("status"), manifest.get("reason"))
+
+
+def verify_evaluation(value: EvaluationRef | str | Path) -> EvaluationRef:
+    """Validate a complete published evaluation and its exact hashed inventory."""
+
+    ref = open_evaluation(value)
+    if ref.path is None:
+        raise ResultError(ref.reason or "evaluation was not durably published")
+    manifest = _json_object(ref.path / "manifest.json")
+    if set(manifest) != {"schema_version", "run_id", "evaluation_id", "pack_id", "status", "reason",
+                         "pack_release", "grader_release", "metric_release", "trust_tier", "files"}:
+        raise ResultError("evaluation manifest fields are incomplete or unsupported")
+    if manifest.get("schema_version") != EVALUATION_SCHEMA_VERSION:
+        raise ResultError("unsupported evaluation schema_version")
+    if manifest.get("pack_id") != PACK_ID or manifest.get("trust_tier") != "local_unverified":
+        raise ResultError("evaluation pack or trust tier is unsupported")
+    if manifest.get("grader_release") != GRADER_RELEASE or manifest.get("metric_release") != METRIC_RELEASE:
+        raise ResultError("evaluation grader or metric release is unsupported")
+    if manifest.get("status") not in _GRADABLE | {"corrupt", "incompatible", "unsupported_schema", "grader_error"}:
+        raise ResultError("evaluation status is unsupported")
+    if manifest.get("status") == "completed" and manifest.get("reason") is not None:
+        raise ResultError("completed evaluation cannot have a failure reason")
+    if manifest.get("status") != "completed" and not isinstance(manifest.get("reason"), str):
+        raise ResultError("non-complete evaluation needs a reason")
+    files = manifest.get("files")
+    if not isinstance(files, dict) or set(files) != _EVALUATION_FILES:
+        raise ResultError("evaluation file inventory must contain exactly judgments.jsonl and metrics.json")
+    if {item.name for item in ref.path.iterdir()} != _EVALUATION_FILES | {"manifest.json"}:
+        raise ResultError("evaluation directory has missing or unindexed files")
+    for filename in _EVALUATION_FILES:
+        item = files[filename]
+        if not isinstance(item, dict) or set(item) != {"sha256", "size_bytes"}:
+            raise ResultError(f"invalid file inventory entry: {filename}")
+        if not isinstance(item["sha256"], str) or not _DIGEST.fullmatch(item["sha256"]):
+            raise ResultError(f"invalid file digest: {filename}")
+        if type(item["size_bytes"]) is not int or item["size_bytes"] < 0:
+            raise ResultError(f"invalid file size: {filename}")
+        try:
+            raw = (ref.path / filename).read_bytes()
+        except OSError as exc:
+            raise ResultError(f"cannot read evaluation file {filename}: {exc}") from exc
+        if sha256_bytes(raw) != item["sha256"] or len(raw) != item["size_bytes"]:
+            raise ResultError(f"evaluation file hash/size mismatch: {filename}")
+    metrics = _json_object(ref.path / "metrics.json")
+    releases = metrics.get("releases")
+    identity = metrics.get("identity")
+    candidate = metrics.get("candidate")
+    counts = metrics.get("counts")
+    definitions = metrics.get("metric_definitions")
+    if any(not isinstance(item, dict) for item in (releases, identity, candidate, counts)):
+        raise ResultError("metrics release, identity, candidate, or count object is malformed")
+    if not isinstance(definitions, list) or definitions != [definition.to_dict() for definition in metric_definitions()]:
+        raise ResultError("metric definitions differ from installed release")
+    if set(candidate) != {"name", "version", "interface_version", "argv_sha256"} or any(
+        not isinstance(item, str) for item in candidate.values()
+    ):
+        raise ResultError("evaluation candidate identity is malformed")
+    if set(releases) != {"pack_release", "case_schema_version", "dataset_id", "dataset_version",
+                         "dataset_sha256", "grader_release", "metric_release"}:
+        raise ResultError("evaluation release identity is malformed")
+    pack = load_rag_rerank_pack()
+    if releases != {
+        "pack_release": pack.descriptor.pack_release,
+        "case_schema_version": pack.descriptor.case_schema_version,
+        "dataset_id": pack.cases[0].dataset_id,
+        "dataset_version": pack.cases[0].dataset_version,
+        "dataset_sha256": pack.dataset_digest,
+        "grader_release": GRADER_RELEASE,
+        "metric_release": METRIC_RELEASE,
+    }:
+        raise ResultError("evaluation release differs from installed pack/metric release")
+    if set(identity) != {"pack_id", "pack_release", "track", "cases", "execution_policy", "runner", "trust_tier"}:
+        raise ResultError("evaluation comparison identity is malformed")
+    if (identity["pack_id"] != PACK_ID or identity["pack_release"] != pack.descriptor.pack_release
+            or identity["track"] != pack.descriptor.track or identity["cases"] != _expected_cases(pack)
+            or identity["trust_tier"] != "local_unverified"):
+        raise ResultError("evaluation comparison identity differs from installed pack")
+    if not isinstance(identity["execution_policy"], dict) or not isinstance(identity["runner"], dict):
+        raise ResultError("evaluation execution/runner identity is malformed")
+    if metrics.get("track") != pack.descriptor.track or manifest.get("pack_release") != pack.descriptor.pack_release:
+        raise ResultError("evaluation track or pack release differs from manifest")
+    if any((
+        metrics.get("schema_version") != METRICS_SCHEMA_VERSION,
+        metrics.get("run_id") != ref.run_id,
+        metrics.get("evaluation_id") != ref.evaluation_id,
+        metrics.get("pack_id") != PACK_ID,
+        metrics.get("status") != ref.status,
+        metrics.get("reason") != ref.reason,
+        metrics.get("trust_tier") != "local_unverified",
+        metrics.get("illustrative") is not True,
+        releases.get("grader_release") != GRADER_RELEASE,
+        releases.get("metric_release") != METRIC_RELEASE,
+    )):
+        raise ResultError("metrics identity, release, or status differs from evaluation manifest")
+    if not isinstance(metrics.get("metrics"), dict) or set(metrics["metrics"]) != set(_METRIC_IDS):
+        raise ResultError("evaluation metric set is incomplete")
+    try:
+        judgments = [json.loads(line) for line in (ref.path / "judgments.jsonl").read_text(encoding="utf-8").splitlines()]
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise ResultError(f"cannot read evaluation judgments: {exc}") from exc
+    cases = identity["cases"]
+    if len(judgments) != len(cases):
+        raise ResultError("judgment count does not match evaluated cases")
+    if any(not isinstance(row, dict) or row.get("schema_version") != JUDGMENT_SCHEMA_VERSION
+           or row.get("run_id") != ref.run_id or row.get("evaluation_id") != ref.evaluation_id
+           or row.get("case_id") != case.get("case_id") or row.get("pool_digest") != case.get("pool_digest")
+           or not isinstance(row.get("values"), dict) or set(row["values"]) != set(_METRIC_IDS)
+           for row, case in zip(judgments, cases)):
+        raise ResultError("judgment schema, order, metric set, or identity mismatch")
+    if any(type(row.get("answerable")) is not bool or not isinstance(row.get("undefined_reasons"), dict)
+           or row.get("trial_index") != 1 or row.get("status") not in {
+               "scored", "candidate_failure", "corrupt", "incompatible", "unsupported_schema", "grader_error"
+           } for row in judgments):
+        raise ResultError("judgment status, eligibility, trial, or missingness is malformed")
+    if any(row["status"] != manifest["status"] for row in judgments) and manifest["status"] not in _GRADABLE:
+        raise ResultError("non-gradable judgment status differs from evaluation")
+    if manifest["status"] in _GRADABLE and any(row["status"] not in {"scored", "candidate_failure"} for row in judgments):
+        raise ResultError("gradable evaluation contains a non-gradable judgment")
+    results: list[CaseMetricResult] = []
+    for row in judgments:
+        terminal = "ok" if row["status"] == "scored" else row["candidate_status"] if row["status"] == "candidate_failure" else row["status"]
+        if not isinstance(terminal, str) or not terminal:
+            raise ResultError("judgment candidate terminal status is missing")
+        results.append(CaseMetricResult(
+            row["case_id"], row["pool_digest"], terminal, row["answerable"],
+            tuple(row["values"].items()), tuple(row["undefined_reasons"].items()),
+        ))
+    try:
+        recomputed = aggregate_case_metrics(results)
+    except (TypeError, ValueError, KeyError) as exc:
+        raise ResultError(f"judgment metric facts are malformed: {exc}") from exc
+    if metrics["metrics"] != {key: value.to_dict() for key, value in recomputed.items()}:
+        raise ResultError("aggregate metrics differ from case judgments")
+    primary = recomputed["ndcg_at_5"]
+    expected_counts = {
+        "attempted_count": len(pack.cases) if manifest["status"] in _GRADABLE else None,
+        "success_count": sum(row["status"] == "scored" for row in judgments) if manifest["status"] in _GRADABLE else None,
+        "failure_count": sum(row["status"] == "candidate_failure" for row in judgments) if manifest["status"] in _GRADABLE else None,
+        "eligible_count": primary.eligible_count,
+        "observed_count": primary.observed_count,
+        "missing_count": primary.missing_count,
+        "undefined_count": primary.undefined_count,
+    }
+    if counts != expected_counts:
+        raise ResultError("evaluation counts differ from case judgments")
+    if manifest["status"] == "completed" and expected_counts["failure_count"] != 0:
+        raise ResultError("completed evaluation contains candidate failures")
+    if manifest["status"] == "incomplete" and expected_counts["failure_count"] == 0:
+        raise ResultError("incomplete evaluation has no candidate failure")
+    if manifest["status"] in _GRADABLE:
+        run_path = ref.path.parent.parent
+        try:
+            run_manifest = _json_object(run_path / "manifest.json")
+            source_ref = RunRef(ref.run_id, run_path, PACK_ID)
+            _check_root(source_ref, run_manifest, pack)
+            source_results, source_judgments, source_successes = _capture_results(source_ref, run_manifest, pack)
+            source_aggregates = aggregate_case_metrics(source_results)
+        except (_Ungradable, EvidenceError, ResultError, TypeError, ValueError, KeyError) as exc:
+            raise ResultError(f"evaluation source run failed verification: {exc}") from exc
+        source_status = "completed" if source_successes == len(pack.cases) else "incomplete"
+        source_reason = None if source_status == "completed" else f"{len(pack.cases) - source_successes} candidate attempt(s) failed"
+        if (manifest["status"], manifest["reason"]) != (source_status, source_reason):
+            raise ResultError("evaluation status or reason differs from verified source captures")
+        for source_judgment in source_judgments:
+            source_judgment["evaluation_id"] = ref.evaluation_id
+        if judgments != source_judgments:
+            raise ResultError("evaluation case judgments differ from verified raw captures and installed pack")
+        if metrics["metrics"] != {key: value.to_dict() for key, value in source_aggregates.items()}:
+            raise ResultError("evaluation aggregate metrics differ from verified source captures")
+        if candidate != run_manifest["candidate"]:
+            raise ResultError("evaluation candidate identity differs from source run")
+        if identity != _identity(run_manifest, pack):
+            raise ResultError("evaluation runner, execution policy, or case identity differs from source run")
+        source_primary = source_aggregates["ndcg_at_5"]
+        source_counts = {
+            "attempted_count": len(pack.cases),
+            "success_count": source_successes,
+            "failure_count": len(pack.cases) - source_successes,
+            "eligible_count": source_primary.eligible_count,
+            "observed_count": source_primary.observed_count,
+            "missing_count": source_primary.missing_count,
+            "undefined_count": source_primary.undefined_count,
+        }
+        if counts != source_counts:
+            raise ResultError("evaluation counts differ from verified source captures")
+    return ref
+
+
+def report(evaluation_ref: EvaluationRef | str | Path) -> Scorecard:
+    """Return a JSON/Markdown scorecard for a hash-verified evaluation."""
+
+    ref = verify_evaluation(evaluation_ref)
+    assert ref.path is not None
+    data = _json_object(ref.path / "metrics.json")
+    latency = data["metrics"]["runner_latency_ms"]
+    data["runner_latency"] = {
+        "unit": latency["unit"],
+        "p50": latency["value"],
+        "p95": latency["p95"],
+        "eligible_count": latency["eligible_count"],
+        "observed_count": latency["observed_count"],
+        "missing_count": latency["missing_count"],
+        "complete": latency["complete"],
+    }
+    return Scorecard(data)
+
+
+def _ref_or_error(value: object) -> tuple[EvaluationRef | None, str | None]:
+    if isinstance(value, FastSearchSmokeResult):
+        return None, "Fast Search smoke/latency result has no labeled frozen-pool quality evaluation"
+    if not isinstance(value, (EvaluationRef, str, os.PathLike)):
+        return None, f"unsupported evaluation reference type: {type(value).__name__}"
+    try:
+        return verify_evaluation(value), None
+    except (ResultError, TypeError, OSError) as exc:
+        return None, str(exc)
+
+
+def compare(evaluation_ref_a: object, evaluation_ref_b: object) -> Comparison:
+    """Compare only matching saved workloads and report paired B-minus-A deltas."""
+
+    ref_a, error_a = _ref_or_error(evaluation_ref_a)
+    ref_b, error_b = _ref_or_error(evaluation_ref_b)
+    name_a = ref_a.evaluation_id if ref_a else evaluation_ref_a.evaluation_id if isinstance(evaluation_ref_a, EvaluationRef) else None
+    name_b = ref_b.evaluation_id if ref_b else evaluation_ref_b.evaluation_id if isinstance(evaluation_ref_b, EvaluationRef) else None
+    base: dict[str, Any] = {
+        "schema_version": "sphynx.comparison.v1",
+        "evaluation_a": name_a,
+        "evaluation_b": name_b,
+        "compatible": False,
+        "reason": None,
+        "trust_tier": "local_unverified",
+        "illustrative": True,
+        "metrics": {},
+    }
+    if error_a or error_b:
+        base["reason"] = f"evaluation {'A' if error_a else 'B'} failed integrity validation: {error_a or error_b}"
+        return Comparison(base)
+    assert ref_a is not None and ref_b is not None and ref_a.path is not None and ref_b.path is not None
+    a = _json_object(ref_a.path / "metrics.json")
+    b = _json_object(ref_b.path / "metrics.json")
+    rows_a = [json.loads(line) for line in (ref_a.path / "judgments.jsonl").read_text(encoding="utf-8").splitlines()]
+    rows_b = [json.loads(line) for line in (ref_b.path / "judgments.jsonl").read_text(encoding="utf-8").splitlines()]
+    return Comparison(compare_validated_records(a, b, rows_a, rows_b))