context-engineering-core 0.12.0__py3-none-any.whl

context_engine/evaluation/reporting.py

@@ -0,0 +1,294 @@
+"""Deterministic scorecard, leaderboard and PNG figures from saved benchmark evidence only."""
+
+import hashlib
+import io
+import json
+from importlib import metadata
+from pathlib import Path
+from statistics import median
+
+from ..errors import BenchmarkError
+from .execution import validate_execution
+from .grading import proportion
+from .results import assess_targets, statistics
+
+
+def scorecard(bundle, frozen, payload):
+    manifest, records, validation = validate_execution(bundle, frozen, payload)
+    receipt_by_id = {e["probe_id"]: e["receipt"] for e in payload["entries"] if e["receipt"]}
+    groups = statistics(manifest, records)
+    for group in groups:
+        selected = [
+            r
+            for r in records
+            if (r["model"], r["budget"], r["variant"])
+            == (group["model"], group["budget"], group["variant"])
+        ]
+        receipts = [
+            receipt_by_id[r["probe_id"]] for r in selected if r["probe_id"] in receipt_by_id
+        ]
+        costs = [r["result"]["new_cost_microusd"] for r in receipts]
+        group["new_cost_microusd"] = sum(c for c in costs if c is not None)
+        group["cost_complete"] = (
+            bool(receipts) and all(c is not None for c in costs) and group["pending"] == 0
+        )
+        group["unknown_cost_outcomes"] = sum(c is None for c in costs)
+        group["replays"] = sum(r["result"]["status"] == "replay" for r in receipts)
+        group["median_estimated_input_tokens"] = (
+            median(r["estimate"]["estimated_tokens"] for r in selected) if selected else None
+        )
+        group["median_estimated_input_reduction"] = (
+            median(
+                1 - r["estimate"]["estimated_tokens"] / r["raw_estimated_tokens"] for r in selected
+            )
+            if selected
+            else None
+        )
+        attempts = [
+            a
+            for receipt in receipts
+            for a in receipt["ledger_rows"]
+            if a["id"] in receipt["result"]["attempt_ids"]
+        ]
+        usages = [json.loads(a["usage_json"]) for a in attempts if a["usage_json"]]
+        group["observed_attempt_usage"] = {
+            "input_tokens": sum(u["input_tokens"] for u in usages),
+            "output_tokens": sum(u["output_tokens"] for u in usages),
+            "cached_input_tokens_known": sum(u["cached_input_tokens"] or 0 for u in usages),
+            "reasoning_tokens_known": sum(u["reasoning_tokens"] or 0 for u in usages),
+            "cached_details_unknown": sum(u["cached_input_tokens"] is None for u in usages),
+            "reasoning_details_unknown": sum(u["reasoning_tokens"] is None for u in usages),
+            "attempts_with_usage": len(usages),
+            "attempts": len(attempts),
+            "unknown_usage_attempts": sum(a["cost"] is None for a in attempts),
+        }
+        group["zones"] = {}
+        for zone in ("early", "middle", "late"):
+            slots = [
+                s
+                for s in manifest.slots
+                if (s["model"], s["budget"], s["variant"])
+                == (group["model"], group["budget"], group["variant"])
+                and bundle.fact(s["fact_id"])["zone"] == zone
+            ]
+            rows = [r for r in selected if bundle.fact(r["fact_id"])["zone"] == zone]
+            group["zones"][zone] = {
+                "retained": proportion(
+                    sum(r["state"] != "non_fit" and r["retained_evidence"] for r in rows),
+                    len(slots),
+                ),
+                "answer_correct": proportion(
+                    sum(r["answer_correct"] is True for r in rows), len(slots)
+                ),
+                "completed": sum(r["state"] == "success" for r in rows),
+            }
+            if not any(r["state"] in ("success", "error", "truncated") for r in rows):
+                group["zones"][zone]["answer_correct"] = {
+                    "count": None,
+                    "denominator": len(slots),
+                    "rate": None,
+                    "wilson95": None,
+                }
+    return {
+        "schema_version": 1,
+        "execution_id": payload["execution_id"],
+        "snapshot_digest": payload["digest"],
+        "mode": payload["profile"]["mode"],
+        "validity": validation,
+        "quality_acceptance": assess_targets(manifest, records, validation),
+        "configuration": manifest.data,
+        "provider_profile": payload["profile"],
+        "groups": groups,
+        "report_inference_calls": 0,
+        "warning": "Synthetic UNKNOWN responses and prices; not model-quality or API-cost evidence"
+        if payload["profile"]["mode"] == "offline"
+        else (
+            "Owner-confirmed free-tier pricing; zero configured charges are not a billing "
+            "attestation or paid-plan savings measurement; replay is not an independent answer"
+            if all(
+                c["quota"].get("billing_mode") == "free_tier"
+                for c in payload["profile"]["clients"].values()
+            )
+            else "Recorded usage and configured prices; replay is not an independent model answer"
+        ),
+    }
+
+
+def leaderboard(card):
+    lines = [
+        "# Benchmark leaderboard",
+        "",
+        f"Validity: {card['validity']['status']}. Mode: {card['mode']}.",
+        "",
+        card["warning"],
+        "",
+        "Rows are in protocol order, not ranked across incompatible budgets or models.",
+        "",
+        "| Model | Budget | Variant | Fit | Retained | Correct / planned | Correct / completed "
+        "| Pending / errors / truncated | New cost (micro-USD) |",
+        "|---|---:|---|---|---|---|---|---|---:|",
+    ]
+
+    def count(value):
+        numerator = value["count"] if value["count"] is not None else "unknown"
+        return f"{numerator} / {value['denominator']}"
+
+    for g in card["groups"]:
+        lines.append(
+            f"| {g['model']} | {g['budget']} | {g['variant']} | {count(g['fit_all_planned'])} | "
+            f"{count(g['retained_fitting_all_planned'])} | "
+            f"{count(g['answer_correct_all_planned'])} | {count(g['answer_correct_completed'])} | "
+            f"{g['pending']} / {g['errors']} / {g['truncated']} | "
+            f"{g['new_cost_microusd']}{'' if g['cost_complete'] else ' (known subtotal)'} |"
+        )
+    lines += [
+        "",
+        f"Quality acceptance: {card['quality_acceptance']['status']}.",
+        "",
+        "Retention requires fitting, source-backed evidence; it is not answer accuracy. "
+        "Unknown usage remains unknown.",
+        "Small correlated synthetic samples do not establish real-workload performance.",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def figures(card):
+    try:
+        from matplotlib.backends.backend_agg import FigureCanvasAgg
+        from matplotlib.figure import Figure
+    except ImportError:
+        raise BenchmarkError("PNG generation requires the reports extra") from None
+    panels = list(dict.fromkeys((g["model"], g["budget"]) for g in card["groups"]))
+    outputs = {}
+    for kind in ("context_cost", "recall_by_zone"):
+        fig = Figure(figsize=(12, 4 * len(panels)), layout="constrained")
+        FigureCanvasAgg(fig)
+        axes = fig.subplots(len(panels), 2, squeeze=False)
+        for row_index, (model, budget) in enumerate(panels):
+            groups = [g for g in card["groups"] if (g["model"], g["budget"]) == (model, budget)]
+            labels = [g["variant"] for g in groups]
+            left, right = axes[row_index]
+            title = f"{model.rsplit('/', 1)[-1]} | budget {budget}"
+            if kind == "context_cost":
+                values = [g["median_estimated_input_tokens"] or 0 for g in groups]
+                bars = left.bar(labels, values, color="#327e9e")
+                left.axhline(budget, color="#a43e3e", linestyle="--", label="Input allowance")
+                left.bar_label(
+                    bars,
+                    [
+                        "N/A" if g["median_estimated_input_tokens"] is None else str(round(v))
+                        for g, v in zip(groups, values, strict=True)
+                    ],
+                    fontsize=8,
+                )
+                left.set_ylabel("Median estimated input tokens")
+                left.legend(fontsize=8)
+                costs = [g["new_cost_microusd"] for g in groups]
+                bars = right.bar(labels, costs, color="#86734b")
+                right.bar_label(
+                    bars,
+                    [
+                        str(c) + ("*" if not g["cost_complete"] else "")
+                        for g, c in zip(groups, costs, strict=True)
+                    ],
+                    fontsize=8,
+                )
+                right.set_ylabel(
+                    "Synthetic micro-USD"
+                    if card["mode"] == "offline"
+                    else "New estimated charge (micro-USD)"
+                )
+                right.set_title("New cost; * known subtotal, not full billing")
+            else:
+                for axis, metric in ((left, "retained"), (right, "answer_correct")):
+                    for i, zone in enumerate(("early", "middle", "late")):
+                        metrics = [g["zones"][zone][metric] for g in groups]
+                        xs = [x + (i - 1) * 0.25 for x in range(len(groups))]
+                        bars = axis.bar(
+                            xs,
+                            [m["rate"] or 0 for m in metrics],
+                            width=0.24,
+                            label=zone,
+                            color=("#327e9e", "#bf8a45", "#6e9277")[i],
+                        )
+                        axis.bar_label(
+                            bars,
+                            [
+                                f"{m['count']}/{m['denominator']}"
+                                if m["denominator"] and m["count"] is not None
+                                else "N/A"
+                                for m in metrics
+                            ],
+                            fontsize=6,
+                            rotation=90,
+                            padding=3,
+                        )
+                    axis.set_xticks(range(len(labels)), labels)
+                    axis.set_ylim(0, 1.25)
+                    axis.set_yticks((0, 0.5, 1), ("0%", "50%", "100%"))
+                    axis.legend(fontsize=8, loc="upper left")
+                left.set_ylabel("Source-backed retained / planned")
+                right.set_ylabel("Correct answers / planned")
+                right.set_title(
+                    "Synthetic answers only"
+                    if card["mode"] == "offline"
+                    else "Uncompleted answers remain in denominator"
+                )
+            left.set_title(title)
+            for axis in (left, right):
+                axis.spines[["top", "right"]].set_visible(False)
+                axis.grid(axis="y", alpha=0.15)
+                axis.set_axisbelow(True)
+        fig.suptitle(
+            f"{kind.replace('_', ' ').title()} — "
+            f"{card['mode'].upper()} / {card['validity']['status']}\n"
+            "Estimates and synthetic tests are not live-provider performance evidence",
+            fontsize=13,
+        )
+        buffer = io.BytesIO()
+        fig.savefig(
+            buffer, format="png", dpi=130, metadata={"Software": "context-engine-report-v1"}
+        )
+        outputs[kind + ".png"] = buffer.getvalue()
+    return outputs
+
+
+def export_report(directory, card):
+    outputs = {
+        "scorecard.json": (
+            json.dumps(card, ensure_ascii=False, indent=2, allow_nan=False) + "\n"
+        ).encode(),
+        "leaderboard.md": leaderboard(card).encode(),
+        **figures(card),
+    }
+    for model in card["configuration"]["models"]:
+        projection = {
+            **card,
+            "groups": [g for g in card["groups"] if g["model"] == model],
+            "selected_model": model,
+            "projection_note": "Model-specific scorecard; parent manifest retained",
+        }
+        outputs[model.rsplit("/", 1)[-1] + "-results.json"] = (
+            json.dumps(projection, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
+        ).encode()
+    manifest = {
+        "execution_id": card["execution_id"],
+        "snapshot_digest": card["snapshot_digest"],
+        "report_inference_calls": 0,
+        "matplotlib": metadata.version("matplotlib"),
+        "pillow": metadata.version("pillow"),
+        "files": {name: hashlib.sha256(raw).hexdigest() for name, raw in outputs.items()},
+    }
+    outputs["report-manifest.json"] = (json.dumps(manifest, indent=2) + "\n").encode()
+    try:
+        directory = Path(directory)
+        directory.mkdir(parents=True, exist_ok=False)
+        for name, raw in outputs.items():
+            with (directory / name).open("xb") as handle:
+                handle.write(raw)
+    except OSError:
+        raise BenchmarkError(
+            "Report destination must be a new writable directory; partial exports are not complete"
+        ) from None
+    return manifest

context_engine/evaluation/results.py

@@ -0,0 +1,331 @@
+"""Request/result validity, explicit denominators and immutable resume snapshots."""
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from statistics import median
+
+from ..errors import BenchmarkError, ContextEngineError
+from ..inspection import save_inspection
+from ..models import canonical_json
+from .corpus import Bundle, aliases, strict_json
+from .grading import answer_correct, normalize, proportion
+from .harness import _prepare_validated
+from .protocol import Gate, Manifest, check_manifest, preflight
+
+RESPONSE_FIELDS = {
+    "answer",
+    "answer_correct",
+    "finish_reason",
+    "error_code",
+    "response_origin",
+    "provider_request_id",
+    "replay_of",
+    "provider_usage",
+}
+TERMINAL = {"success", "error", "truncated", "non_fit"}
+MAX_RUN_BYTES = 32_000_000
+
+
+def record_response(
+    bundle: Bundle,
+    prepared: dict,
+    *,
+    answer: str | None,
+    finish_reason: str,
+    origin: str,
+    error_code: str | None = None,
+    provider_request_id: str | None = None,
+    replay_of: str | None = None,
+    provider_usage: dict | None = None,
+) -> dict:
+    """Record supplied evidence of a response; this function never performs inference."""
+    if prepared["state"] != "prepared":
+        raise BenchmarkError("Only an admitted, unanswered probe can receive a response")
+    result = strict_json(canonical_json(prepared))
+    if finish_reason not in ("stop", "length", "error") or origin not in ("test", "live", "replay"):
+        raise BenchmarkError("Unsupported response disposition")
+    if finish_reason != "error" and not isinstance(answer, str):
+        raise BenchmarkError("Completed response requires answer text")
+    result.update(
+        state={"stop": "success", "length": "truncated", "error": "error"}[finish_reason],
+        answer=answer,
+        finish_reason=finish_reason,
+        response_origin=origin,
+        error_code=error_code,
+        provider_request_id=provider_request_id,
+        replay_of=replay_of,
+        provider_usage=provider_usage,
+        answer_correct=answer_correct(answer, aliases(bundle.fact(result["fact_id"])))
+        if finish_reason == "stop"
+        else None,
+    )
+    _check_response(bundle, result)
+    return result
+
+
+def _check_response(bundle: Bundle, row: dict) -> None:
+    state = row["state"]
+    if state in ("prepared", "non_fit"):
+        if any(row[key] is not None for key in RESPONSE_FIELDS):
+            raise BenchmarkError("Unsent probe contains fabricated generation metadata")
+        return
+    expected_finish = {"success": "stop", "truncated": "length", "error": "error"}
+    if state not in expected_finish or row["finish_reason"] != expected_finish[state]:
+        raise BenchmarkError("Inconsistent generation disposition")
+    if row["response_origin"] not in ("test", "live", "replay"):
+        raise BenchmarkError("Missing response provenance")
+    if (
+        row["response_origin"] in ("live", "replay")
+        and (
+            not isinstance(row["provider_request_id"], str)
+            or not row["provider_request_id"].strip()
+        )
+        and state != "error"
+    ):
+        raise BenchmarkError("Completed provider response lacks a request identifier")
+    if row["response_origin"] == "replay":
+        if not isinstance(row["replay_of"], str) or not row["replay_of"].strip():
+            raise BenchmarkError("Replay lacks an original response reference")
+    elif row["replay_of"] is not None:
+        raise BenchmarkError("Non-replay response claims replay ancestry")
+    if state == "error":
+        if (
+            not isinstance(row["error_code"], str)
+            or row["error_code"]
+            not in (
+                "timeout",
+                "quota",
+                "transport",
+                "rate_limit",
+                "provider_error",
+                "invalid_response",
+            )
+            or row["answer"] is not None
+            or row["answer_correct"] is not None
+        ):
+            raise BenchmarkError("Invalid structured provider error")
+    else:
+        if row["error_code"] is not None or not isinstance(row["answer"], str):
+            raise BenchmarkError("Inconsistent completed response")
+        expected = (
+            answer_correct(row["answer"], aliases(bundle.fact(row["fact_id"])))
+            if state == "success"
+            else None
+        )
+        if type(row["answer_correct"]) is not type(expected) or row["answer_correct"] != expected:
+            raise BenchmarkError("Stored answer grade does not match response")
+    usage = row["provider_usage"]
+    if usage is not None:
+        if not isinstance(usage, dict) or set(usage) != {
+            "input_tokens",
+            "output_tokens",
+            "cached_input_tokens",
+        }:
+            raise BenchmarkError("Malformed provider usage")
+        if (
+            any(type(n) is not int or n < 0 for n in usage.values())
+            or usage["cached_input_tokens"] > usage["input_tokens"]
+        ):
+            raise BenchmarkError("Inconsistent provider token usage")
+
+
+def validate_run(
+    bundle: Bundle, frozen: dict, manifest: Manifest, records: list[dict] | tuple[dict, ...]
+) -> dict:
+    gates = list(preflight(bundle, frozen))
+    try:
+        check_manifest(bundle, frozen, manifest)
+    except ContextEngineError:
+        gates = [g for g in gates if g.gate != "V5"] + [Gate("V5", "FAIL", "manifest_drift")]
+    if any(g.status == "FAIL" for g in gates):
+        gates += [
+            Gate("V3", "NOT_RUN", "preflight_failed"),
+            Gate("V4", "NOT_RUN", "preflight_failed"),
+        ]
+        return _validation_result("INVALID", gates, [])
+    slots = {slot["probe_id"]: slot for slot in manifest.slots}
+    seen = set()
+    prepared_cache = {}
+    try:
+        if not isinstance(records, (list, tuple)):
+            raise BenchmarkError("Records must be an array")
+        for row in records:
+            if not isinstance(row, dict) or row["probe_id"] in seen or row["probe_id"] not in slots:
+                raise BenchmarkError("Duplicate or unknown probe disposition")
+            seen.add(row["probe_id"])
+            slot = slots[row["probe_id"]]
+            key = (slot["budget"], slot["variant"], slot["fact_id"])
+            if key not in prepared_cache:
+                prepared_cache[key] = _prepare_validated(bundle, slot)
+            prepared = {**prepared_cache[key], **slot}
+            if set(row) != set(prepared):
+                raise BenchmarkError("Malformed result fields")
+            unchanged = set(prepared) - RESPONSE_FIELDS - {"state"}
+            if canonical_json({k: row[k] for k in unchanged}) != canonical_json(
+                {k: prepared[k] for k in unchanged}
+            ):
+                raise BenchmarkError(
+                    "Saved request, accounting or evidence differs from measured assembly"
+                )
+            if prepared["state"] == "non_fit" and row["state"] != "non_fit":
+                raise BenchmarkError("Oversized request was admitted")
+            if prepared["state"] != "non_fit" and row["state"] == "non_fit":
+                raise BenchmarkError("Fitting request mislabeled as oversized")
+            _check_response(bundle, row)
+        gates.append(Gate("V3", "PASS", "requests_reassembled_and_recounted"))
+    except (ContextEngineError, KeyError, TypeError, ValueError, IndexError):
+        gates += [
+            Gate("V3", "FAIL", "request_or_result_integrity"),
+            Gate("V4", "NOT_RUN", "request_check_failed"),
+        ]
+        return _validation_result("INVALID", gates, [])
+    terminal = {row["probe_id"] for row in records if row["state"] in TERMINAL}
+    pending = [identifier for identifier in slots if identifier not in terminal]
+    if any(row["state"] == "truncated" for row in records):
+        gates.append(Gate("V4", "FAIL", "generation_truncated"))
+        status = "INVALID"
+    elif pending:
+        any_response = any(row["state"] in ("success", "error") for row in records)
+        status = "INTERRUPTED" if any_response or len(seen) < len(slots) else "PREPARED"
+        gates.append(Gate("V4", "NOT_RUN", "pending_probe_dispositions"))
+    else:
+        gates.append(Gate("V4", "PASS", "all_probes_have_terminal_dispositions"))
+        status = (
+            "TEST_ONLY" if any(row["response_origin"] == "test" for row in records) else "VALID"
+        )
+    return _validation_result(status, gates, pending)
+
+
+def _validation_result(status, gates, pending):
+    return {
+        "status": status,
+        "gates": [asdict(g) for g in sorted(gates, key=lambda g: g.gate)],
+        "pending_probe_ids": pending,
+    }
+
+
+def statistics(manifest: Manifest, records) -> list[dict]:
+    """After integrity validation, report separate model/budget/variant groups."""
+    rows = {r["probe_id"]: r for r in records}
+    result = []
+    for model in manifest.data["models"]:
+        for budget in manifest.data["budgets"]:
+            for variant in manifest.data["variants"]:
+                planned = [
+                    s
+                    for s in manifest.slots
+                    if (s["model"], s["budget"], s["variant"]) == (model, budget, variant)
+                ]
+                measured = [rows[s["probe_id"]] for s in planned if s["probe_id"] in rows]
+                fitting = [r for r in measured if r["state"] != "non_fit"]
+                completed = [r for r in measured if r["state"] == "success"]
+                correct = sum(r["answer_correct"] for r in completed)
+                attempted = [r for r in measured if r["state"] in ("success", "error", "truncated")]
+                result.append(
+                    {
+                        "model": model,
+                        "budget": budget,
+                        "variant": variant,
+                        "planned": len(planned),
+                        "measured": len(measured),
+                        "fit_all_planned": proportion(len(fitting), len(planned)),
+                        "literal_fact_present_measured": proportion(
+                            sum(r["fact_present"] for r in measured), len(measured)
+                        ),
+                        "retained_fitting_all_planned": proportion(
+                            sum(r["retained_evidence"] for r in fitting), len(planned)
+                        ),
+                        "answer_correct_all_planned": proportion(correct, len(planned))
+                        if attempted
+                        else {
+                            "count": None,
+                            "denominator": len(planned),
+                            "rate": None,
+                            "wilson95": None,
+                        },
+                        "answer_correct_completed": proportion(correct, len(completed)),
+                        "errors": sum(r["state"] == "error" for r in measured),
+                        "truncated": sum(r["state"] == "truncated" for r in measured),
+                        "non_fit": len(measured) - len(fitting),
+                        "pending": len(planned) - sum(r["state"] in TERMINAL for r in measured),
+                        "abstentions": sum(normalize(r["answer"]) == "unknown" for r in completed),
+                        "provider_usage_unknown": sum(
+                            r["provider_usage"] is None for r in attempted
+                        ),
+                        "test_responses": sum(r["response_origin"] == "test" for r in measured),
+                        "cost": None,
+                    }
+                )
+    return result
+
+
+def assess_targets(manifest: Manifest, records, validation: dict) -> dict:
+    """Quality cannot pass on a partial run, mock responses or unapproved registration."""
+    p = manifest.data["protocol"]
+    if validation["status"] != "VALID" or p["approval"] != "owner_approved":
+        return {
+            "status": "NOT_EVALUATED",
+            "reason": "requires_valid_real_run_and_owner_registration",
+        }
+    primary = [
+        r
+        for r in records
+        if r["variant"] == "A5" and r["budget"] == 900 and r["model"] == p["models"][0]
+    ]
+    if len(primary) != 31 or len({r["fact_id"] for r in primary}) != 31:
+        return {"status": "NOT_EVALUATED", "reason": "requires_all_primary_probes"}
+    observed = {
+        "fit": sum(r["state"] != "non_fit" for r in primary),
+        "retained": sum(r["state"] != "non_fit" and r["retained_evidence"] for r in primary),
+        "correct": sum(r["answer_correct"] is True for r in primary),
+        "median_input_reduction": median(
+            1 - r["estimate"]["estimated_tokens"] / r["raw_estimated_tokens"] for r in primary
+        ),
+    }
+    passed = all(observed[key] >= p["targets"][key] for key in observed)
+    return {"status": "PASS" if passed else "FAIL", "observed": observed}
+
+
+def save_run(path: Path, bundle: Bundle, frozen: dict, manifest: Manifest, records) -> None:
+    validation = validate_run(bundle, frozen, manifest, records)
+    if validation["status"] == "INVALID":
+        raise BenchmarkError("Cannot export an invalid resume snapshot")
+    payload = {
+        "schema_version": 1,
+        "run_id": manifest.run_id,
+        "manifest": manifest.data,
+        "records": records,
+    }
+    if (
+        len((json.dumps(payload, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode())
+        > MAX_RUN_BYTES
+    ):
+        raise BenchmarkError("Run snapshot exceeds byte limit")
+    save_inspection(path, payload)
+
+
+def load_run(path: Path, bundle: Bundle, frozen: dict):
+    try:
+        with path.open("rb") as handle:
+            raw = handle.read(MAX_RUN_BYTES + 1)
+        if len(raw) > MAX_RUN_BYTES:
+            raise BenchmarkError("Run snapshot exceeds byte limit")
+        payload = strict_json(raw.decode("utf-8"), maximum=MAX_RUN_BYTES)
+        if (
+            set(payload) != {"schema_version", "run_id", "manifest", "records"}
+            or type(payload["schema_version"]) is not int
+            or payload["schema_version"] != 1
+        ):
+            raise BenchmarkError("Unsupported run snapshot schema")
+        manifest = Manifest(canonical_json(payload["manifest"]))
+        if payload["run_id"] != manifest.run_id:
+            raise BenchmarkError("Run identity mismatch")
+        validation = validate_run(bundle, frozen, manifest, payload["records"])
+        if validation["status"] == "INVALID":
+            raise BenchmarkError("Resume snapshot failed validity gates")
+        return manifest, payload["records"], validation
+    except BenchmarkError:
+        raise
+    except (OSError, UnicodeError, KeyError, TypeError, ValueError):
+        raise BenchmarkError("Cannot read a valid resume snapshot") from None