ragops 1.0.0__py3-none-any.whl

ragops/__init__.py

@@ -0,0 +1,30 @@
+"""RAGOps public API."""
+
+from ragops.engine import compare, evaluate
+from ragops.drift import detect_evaluator_drift
+from ragops.loader import load_responses, load_scenario, responses_from_data, scenario_from_dict
+from ragops.statistical import (
+    compare_replay_bundles,
+    load_replay_bundle,
+    replay_bundle_from_dict,
+)
+from ragops.provenance import diagnose_provenance
+from ragops.sequential import compare_replay_bundles_sequentially
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "compare",
+    "compare_replay_bundles_sequentially",
+    "detect_evaluator_drift",
+    "diagnose_provenance",
+    "evaluate",
+    "compare_replay_bundles",
+    "load_replay_bundle",
+    "load_responses",
+    "load_scenario",
+    "responses_from_data",
+    "replay_bundle_from_dict",
+    "scenario_from_dict",
+    "__version__",
+]

ragops/adapters/__init__.py

@@ -0,0 +1,16 @@
+"""Optional dependency-free integration adapters."""
+
+from .http import AdapterError, post_json
+from .external_metrics import (
+    ExternalMetricEvaluator,
+    load_external_metric_evaluator,
+    validate_external_metric_pair,
+)
+
+__all__ = [
+    "AdapterError",
+    "ExternalMetricEvaluator",
+    "load_external_metric_evaluator",
+    "post_json",
+    "validate_external_metric_pair",
+]

ragops/adapters/external_metrics.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import json
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ragops.loader import ContractError
+from ragops.models import EvalCase, RecordedResponse, Scenario
+from ragops.plugins import PluginResult
+
+ALLOWED_PROVIDERS = {"ragas", "deepeval", "langfuse", "custom"}
+
+
+@dataclass(frozen=True)
+class ExternalMetricEnvelope:
+    provider: str
+    metrics_by_case: dict[str, dict[str, float]]
+    metric_names: frozenset[str]
+
+
+class ExternalMetricEvaluator:
+    """Expose a portable external metric envelope through the evaluator protocol."""
+
+    def __init__(self, envelope: ExternalMetricEnvelope) -> None:
+        self.name = envelope.provider
+        self._metrics_by_case = envelope.metrics_by_case
+        self.metric_names = envelope.metric_names
+
+    def evaluate(self, case: EvalCase, response: RecordedResponse) -> PluginResult:
+        try:
+            metrics = self._metrics_by_case[case.id]
+        except KeyError as exc:
+            raise ContractError(f"External metrics are missing case {case.id!r}") from exc
+        return PluginResult(metrics=metrics)
+
+
+def load_external_metric_evaluator(
+    path: str | Path,
+    scenario: Scenario,
+) -> ExternalMetricEvaluator:
+    try:
+        data: Any = json.loads(
+            Path(path).read_text(encoding="utf-8"),
+            parse_constant=lambda value: (_ for _ in ()).throw(
+                ValueError(f"non-finite JSON number {value}")
+            ),
+        )
+    except (OSError, json.JSONDecodeError, ValueError) as exc:
+        raise ContractError(f"Cannot load external metrics from {path}: {exc}") from exc
+    envelope = external_metric_envelope_from_dict(data)
+    expected = {case.id for case in scenario.cases}
+    supplied = set(envelope.metrics_by_case)
+    if supplied != expected:
+        missing = sorted(expected - supplied)
+        unknown = sorted(supplied - expected)
+        raise ContractError(
+            f"External metric coverage mismatch; missing={missing}, unknown={unknown}"
+        )
+    return ExternalMetricEvaluator(envelope)
+
+
+def external_metric_envelope_from_dict(data: Any) -> ExternalMetricEnvelope:
+    if not isinstance(data, dict) or data.get("schema_version") != "0.1":
+        raise ContractError("External metrics must use schema version 0.1")
+    unknown_fields = set(data) - {"schema_version", "provider", "records"}
+    if unknown_fields:
+        raise ContractError(f"External metrics have unknown fields: {sorted(unknown_fields)}")
+    provider = data.get("provider")
+    if provider not in ALLOWED_PROVIDERS:
+        raise ContractError("External metric provider must be ragas, deepeval, langfuse, or custom")
+    records = data.get("records")
+    if not isinstance(records, list) or not records:
+        raise ContractError("External metrics need at least one record")
+    metrics_by_case: dict[str, dict[str, float]] = {}
+    expected_metric_names: frozenset[str] | None = None
+    for index, record in enumerate(records):
+        if not isinstance(record, dict):
+            raise ContractError(f"External metric record {index} must be an object")
+        unknown_record_fields = set(record) - {"case_id", "metrics"}
+        if unknown_record_fields:
+            raise ContractError(
+                f"External metric record {index} has unknown fields: "
+                f"{sorted(unknown_record_fields)}"
+            )
+        case_id = record.get("case_id")
+        if not isinstance(case_id, str) or not case_id:
+            raise ContractError(f"External metric record {index} needs a non-empty case_id")
+        if case_id in metrics_by_case:
+            raise ContractError(f"External metric case IDs must be unique: {case_id!r}")
+        raw_metrics = record.get("metrics")
+        if not isinstance(raw_metrics, dict) or not raw_metrics:
+            raise ContractError(f"External metric record {case_id!r} needs metrics")
+        metrics: dict[str, float] = {}
+        for name, value in raw_metrics.items():
+            if not isinstance(name, str) or not name:
+                raise ContractError(f"External metric names must be non-empty for case {case_id!r}")
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                raise ContractError(f"External metric {provider}.{name} must be numeric")
+            number = float(value)
+            if not math.isfinite(number):
+                raise ContractError(f"External metric {provider}.{name} must be finite")
+            metrics[name] = number
+        metric_names = frozenset(metrics)
+        if expected_metric_names is None:
+            expected_metric_names = metric_names
+        elif metric_names != expected_metric_names:
+            raise ContractError(
+                f"External metric names differ for case {case_id!r}; "
+                f"expected={sorted(expected_metric_names)}, actual={sorted(metric_names)}"
+            )
+        metrics_by_case[case_id] = metrics
+    return ExternalMetricEnvelope(
+        provider=provider,
+        metrics_by_case=metrics_by_case,
+        metric_names=expected_metric_names or frozenset(),
+    )
+
+
+def validate_external_metric_pair(
+    baseline: ExternalMetricEvaluator,
+    candidate: ExternalMetricEvaluator,
+) -> None:
+    if baseline.name != candidate.name:
+        raise ContractError(
+            f"External metric providers differ: {baseline.name!r} != {candidate.name!r}"
+        )
+    if baseline.metric_names != candidate.metric_names:
+        raise ContractError(
+            "External baseline and candidate metric names differ; "
+            f"baseline={sorted(baseline.metric_names)}, "
+            f"candidate={sorted(candidate.metric_names)}"
+        )

ragops/adapters/http.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import Any
+
+
+class AdapterError(RuntimeError):
+    """Raised when a remote application adapter cannot return valid JSON."""
+
+
+def post_json(
+    url: str,
+    payload: dict[str, Any],
+    *,
+    headers: dict[str, str] | None = None,
+    timeout: float = 30.0,
+) -> dict[str, Any]:
+    request_headers = {"content-type": "application/json", **(headers or {})}
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=request_headers,
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+            data = json.loads(response.read().decode("utf-8"))
+    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
+        raise AdapterError(f"HTTP adapter failed for {url}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise AdapterError(f"HTTP adapter expected a JSON object from {url}")
+    return data

ragops/adapters/provider_metrics.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+
+from ragops.loader import ContractError
+from ragops.models import MetricObservation
+
+
+OTEL_EVALUATION_NAME = "gen_ai.evaluation.name"
+OTEL_EVALUATION_SCORE = "gen_ai.evaluation.score.value"
+RAGOPS_CASE_ID = "ragops.case.id"
+RAGOPS_REPEAT_ID = "ragops.repeat.id"
+
+
+def ragas_scores_to_observations(
+    scores: list[dict],
+    case_ids: list[str],
+    *,
+    repeat_id: str,
+    metric_names: tuple[str, ...],
+) -> tuple[MetricObservation, ...]:
+    """Map ordered Ragas EvaluationResult.scores rows into replay observations."""
+    _identifiers(case_ids, repeat_id, metric_names)
+    if len(scores) != len(case_ids):
+        raise ContractError("Ragas score rows must match the ordered case IDs")
+    observations = []
+    for index, (case_id, score_row) in enumerate(zip(case_ids, scores, strict=True)):
+        if not isinstance(score_row, dict):
+            raise ContractError(f"Ragas score row {index} must be an object")
+        observations.append(
+            MetricObservation(
+                case_id=case_id,
+                repeat_id=repeat_id,
+                metrics=_selected_metrics(score_row, metric_names, f"Ragas row {index}"),
+            )
+        )
+    return tuple(observations)
+
+
+def deepeval_scores_to_observations(
+    records: list[dict],
+    *,
+    metric_names: tuple[str, ...],
+) -> tuple[MetricObservation, ...]:
+    """Map recorded DeepEval metric.score values into replay observations."""
+    if not metric_names or len(set(metric_names)) != len(metric_names):
+        raise ContractError("DeepEval metric names must be non-empty and unique")
+    observations = []
+    identities = set()
+    for index, record in enumerate(records):
+        if not isinstance(record, dict) or set(record) != {"case_id", "repeat_id", "metrics"}:
+            raise ContractError(
+                f"DeepEval record {index} must contain case_id, repeat_id, and metrics"
+            )
+        case_id = record["case_id"]
+        repeat_id = record["repeat_id"]
+        _identifiers([case_id], repeat_id, metric_names)
+        identity = (case_id, repeat_id)
+        if identity in identities:
+            raise ContractError("DeepEval case/repeat identities must be unique")
+        identities.add(identity)
+        observations.append(
+            MetricObservation(
+                case_id=case_id,
+                repeat_id=repeat_id,
+                metrics=_selected_metrics(
+                    record["metrics"], metric_names, f"DeepEval record {index}"
+                ),
+            )
+        )
+    if not observations:
+        raise ContractError("DeepEval records must not be empty")
+    return tuple(observations)
+
+
+def otel_evaluation_events_to_observations(
+    events: list[dict],
+) -> tuple[MetricObservation, ...]:
+    """Group recorded OpenTelemetry GenAI evaluation events by replay identity."""
+    grouped: dict[tuple[str, str], dict[str, float]] = defaultdict(dict)
+    if not events:
+        raise ContractError("OpenTelemetry evaluation events must not be empty")
+    for index, event in enumerate(events):
+        if not isinstance(event, dict) or not isinstance(event.get("attributes"), dict):
+            raise ContractError(f"OpenTelemetry event {index} needs an attributes object")
+        attributes = event["attributes"]
+        try:
+            case_id = attributes[RAGOPS_CASE_ID]
+            repeat_id = attributes[RAGOPS_REPEAT_ID]
+            metric_name = attributes[OTEL_EVALUATION_NAME]
+            score = attributes[OTEL_EVALUATION_SCORE]
+        except KeyError as exc:
+            raise ContractError(
+                f"OpenTelemetry event {index} is missing required attribute {exc.args[0]}"
+            ) from exc
+        _identifiers([case_id], repeat_id, (metric_name,))
+        numeric = _finite(score, f"OpenTelemetry event {index} score")
+        identity = (case_id, repeat_id)
+        if metric_name in grouped[identity]:
+            raise ContractError(
+                f"Duplicate OpenTelemetry evaluation metric {metric_name!r} for {identity}"
+            )
+        grouped[identity][metric_name] = numeric
+    metric_sets = {tuple(sorted(metrics)) for metrics in grouped.values()}
+    if len(metric_sets) != 1:
+        raise ContractError(
+            "Every OpenTelemetry replay identity must contain the same evaluation metrics"
+        )
+    return tuple(
+        MetricObservation(case_id, repeat_id, metrics)
+        for (case_id, repeat_id), metrics in sorted(grouped.items())
+    )
+
+
+def _selected_metrics(
+    raw: object, metric_names: tuple[str, ...], label: str
+) -> dict[str, float]:
+    if not isinstance(raw, dict):
+        raise ContractError(f"{label} metrics must be an object")
+    missing = sorted(set(metric_names) - set(raw))
+    if missing:
+        raise ContractError(f"{label} is missing metrics: {missing}")
+    return {name: _finite(raw[name], f"{label}.{name}") for name in metric_names}
+
+
+def _identifiers(
+    case_ids: list[str], repeat_id: str, metric_names: tuple[str, ...]
+) -> None:
+    if not case_ids or any(not isinstance(case_id, str) or not case_id for case_id in case_ids):
+        raise ContractError("Case IDs must be non-empty strings")
+    if len(set(case_ids)) != len(case_ids):
+        raise ContractError("Case IDs must be unique")
+    if not isinstance(repeat_id, str) or not repeat_id:
+        raise ContractError("Repeat ID must be a non-empty string")
+    if (
+        not metric_names
+        or len(set(metric_names)) != len(metric_names)
+        or any(not isinstance(name, str) or not name for name in metric_names)
+    ):
+        raise ContractError("Metric names must be non-empty and unique")
+
+
+def _finite(value: object, name: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ContractError(f"{name} must be numeric")
+    numeric = float(value)
+    if not math.isfinite(numeric):
+        raise ContractError(f"{name} must be finite")
+    return numeric

ragops/adapters/repeated_runs.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import json
+import math
+import os
+import stat
+import subprocess
+import tempfile
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Callable, Protocol
+
+from ragops.loader import ContractError
+from ragops.models import MetricObservation, ReplayBundle, ReplayProvenance
+from ragops.statistical import load_replay_bundle
+
+
+MAX_CASES = 10_000
+MAX_REPEATS = 1_000
+MAX_OBSERVATIONS = 100_000
+MAX_COMMAND_OUTPUT_BYTES = 1_000_000
+
+
+@dataclass(frozen=True)
+class RepeatedRunPlan:
+    schema_version: str
+    scenario_id: str
+    scenario_digest: str
+    case_ids: tuple[str, ...]
+    repeats: int
+    provenance: ReplayProvenance
+
+
+class MetricRunAdapter(Protocol):
+    def collect(self, case_id: str, repeat_id: str) -> dict[str, float]: ...
+
+
+class CommandMetricAdapter:
+    """Run one explicit command per observation without invoking a shell."""
+
+    def __init__(self, command: tuple[str, ...], *, timeout_seconds: float = 60.0) -> None:
+        if not command or any(not isinstance(part, str) or not part for part in command):
+            raise ValueError("Repeated-run command must contain non-empty arguments")
+        if not math.isfinite(timeout_seconds) or timeout_seconds <= 0:
+            raise ValueError("Repeated-run timeout must be finite and positive")
+        self.command = command
+        self.timeout_seconds = timeout_seconds
+
+    def collect(self, case_id: str, repeat_id: str) -> dict[str, float]:
+        environment = os.environ.copy()
+        environment["RAGOPS_CASE_ID"] = case_id
+        environment["RAGOPS_REPEAT_ID"] = repeat_id
+        try:
+            completed = subprocess.run(
+                self.command,
+                check=False,
+                capture_output=True,
+                env=environment,
+                timeout=self.timeout_seconds,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            raise ContractError(
+                f"Repeated-run command failed for {case_id}/{repeat_id}: {exc}"
+            ) from exc
+        if len(completed.stdout) > MAX_COMMAND_OUTPUT_BYTES:
+            raise ContractError(
+                f"Repeated-run command output exceeds {MAX_COMMAND_OUTPUT_BYTES} bytes"
+            )
+        if completed.returncode != 0:
+            error = completed.stderr[:2000].decode("utf-8", errors="replace").strip()
+            raise ContractError(
+                f"Repeated-run command exited {completed.returncode} for "
+                f"{case_id}/{repeat_id}: {error}"
+            )
+        try:
+            payload = json.loads(completed.stdout.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise ContractError(
+                f"Repeated-run command returned invalid JSON for {case_id}/{repeat_id}"
+            ) from exc
+        if not isinstance(payload, dict) or set(payload) != {"metrics"}:
+            raise ContractError("Repeated-run command output must contain only a metrics object")
+        return _metric_map(payload["metrics"], f"command output {case_id}/{repeat_id}")
+
+
+def load_repeated_run_plan(path: str | Path) -> RepeatedRunPlan:
+    try:
+        data = json.loads(
+            Path(path).read_text(encoding="utf-8"),
+            parse_constant=lambda value: (_ for _ in ()).throw(
+                ValueError(f"non-finite JSON number {value}")
+            ),
+        )
+    except (OSError, json.JSONDecodeError, ValueError) as exc:
+        raise ContractError(f"Cannot load repeated-run plan from {path}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ContractError("Repeated-run plan must be an object")
+    unknown = set(data) - {
+        "schema_version",
+        "scenario_id",
+        "scenario_digest",
+        "case_ids",
+        "repeats",
+        "provenance",
+    }
+    if unknown:
+        raise ContractError(f"Unknown repeated-run plan fields: {sorted(unknown)}")
+    try:
+        provenance_data = data["provenance"]
+        if not isinstance(provenance_data, dict):
+            raise TypeError("provenance must be an object")
+        if set(provenance_data) != {
+            "dataset",
+            "evidence",
+            "evaluator",
+            "application",
+            "model",
+            "model_config",
+            "environment",
+        }:
+            raise ValueError("provenance fields must match the replay-bundle contract")
+        plan = RepeatedRunPlan(
+            schema_version=data["schema_version"],
+            scenario_id=data["scenario_id"],
+            scenario_digest=data["scenario_digest"],
+            case_ids=tuple(data["case_ids"]),
+            repeats=data["repeats"],
+            provenance=ReplayProvenance(**provenance_data),
+        )
+    except (KeyError, TypeError, ValueError) as exc:
+        raise ContractError(f"Invalid repeated-run plan contract: {exc}") from exc
+    _validate_plan(plan)
+    return plan
+
+
+def collect_repeated_runs(
+    plan: RepeatedRunPlan,
+    adapter: MetricRunAdapter,
+    *,
+    existing: ReplayBundle | None = None,
+    checkpoint: Callable[[ReplayBundle], None] | None = None,
+    stop_when: Callable[[ReplayBundle], bool] | None = None,
+) -> ReplayBundle:
+    _validate_plan(plan)
+    records = list(existing.records) if existing is not None else []
+    if existing is not None:
+        _validate_resume(plan, existing)
+    completed = {(record.case_id, record.repeat_id) for record in records}
+    expected_metric_names = set(records[0].metrics) if records else None
+    for repeat_number in range(1, plan.repeats + 1):
+        for case_id in plan.case_ids:
+            repeat_id = f"run-{repeat_number:04d}"
+            if (case_id, repeat_id) in completed:
+                continue
+            metrics = _metric_map(
+                adapter.collect(case_id, repeat_id), f"adapter result {case_id}/{repeat_id}"
+            )
+            if expected_metric_names is None:
+                expected_metric_names = set(metrics)
+            elif set(metrics) != expected_metric_names:
+                raise ContractError("Every repeated-run observation must use the same metrics")
+            records.append(MetricObservation(case_id, repeat_id, metrics))
+            bundle = _bundle(plan, tuple(records))
+            if checkpoint is not None:
+                checkpoint(bundle)
+            if stop_when is not None and stop_when(bundle):
+                return bundle
+    return _bundle(plan, tuple(records))
+
+
+def load_resume_bundle(path: str | Path, *, resume: bool) -> ReplayBundle | None:
+    destination = Path(path)
+    if not destination.exists():
+        return None
+    if destination.is_symlink():
+        raise ContractError("Repeated-run output must not be a symlink")
+    if not resume:
+        raise ContractError("Repeated-run output already exists; pass --resume to continue")
+    return load_replay_bundle(destination)
+
+
+def write_replay_bundle(path: str | Path, bundle: ReplayBundle) -> None:
+    destination = Path(path)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    if destination.is_symlink():
+        raise ContractError("Repeated-run output must not be a symlink")
+    payload = json.dumps(asdict(bundle), ensure_ascii=False, indent=2) + "\n"
+    temporary_name: str | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            encoding="utf-8",
+            dir=destination.parent,
+            prefix=f".{destination.name}.",
+            delete=False,
+        ) as temporary:
+            temporary_name = temporary.name
+            os.chmod(temporary_name, stat.S_IRUSR | stat.S_IWUSR)
+            temporary.write(payload)
+            temporary.flush()
+            os.fsync(temporary.fileno())
+        os.replace(temporary_name, destination)
+    finally:
+        if temporary_name and Path(temporary_name).exists():
+            Path(temporary_name).unlink()
+
+
+def _validate_plan(plan: RepeatedRunPlan) -> None:
+    if plan.schema_version != "0.1":
+        raise ContractError(f"Unsupported repeated-run plan schema: {plan.schema_version}")
+    for name, value in (
+        ("scenario_id", plan.scenario_id),
+        ("scenario_digest", plan.scenario_digest),
+        ("provenance.dataset", plan.provenance.dataset),
+        ("provenance.evidence", plan.provenance.evidence),
+        ("provenance.evaluator", plan.provenance.evaluator),
+        ("provenance.application", plan.provenance.application),
+        ("provenance.model", plan.provenance.model),
+        ("provenance.model_config", plan.provenance.model_config),
+        ("provenance.environment", plan.provenance.environment),
+    ):
+        if not isinstance(value, str) or not value:
+            raise ContractError(f"{name} must be a non-empty string")
+    if not plan.case_ids or len(plan.case_ids) > MAX_CASES:
+        raise ContractError(f"Repeated-run plan needs between 1 and {MAX_CASES} cases")
+    if any(not isinstance(case_id, str) or not case_id for case_id in plan.case_ids):
+        raise ContractError("Repeated-run case IDs must be non-empty strings")
+    if len(set(plan.case_ids)) != len(plan.case_ids):
+        raise ContractError("Repeated-run case IDs must be unique")
+    if isinstance(plan.repeats, bool) or not isinstance(plan.repeats, int):
+        raise ContractError("Repeated-run repeats must be an integer")
+    if not 1 <= plan.repeats <= MAX_REPEATS:
+        raise ContractError(f"Repeated-run repeats must be between 1 and {MAX_REPEATS}")
+    if len(plan.case_ids) * plan.repeats > MAX_OBSERVATIONS:
+        raise ContractError(
+            f"Repeated-run plan exceeds the {MAX_OBSERVATIONS} observation limit"
+        )
+
+
+def _validate_resume(plan: RepeatedRunPlan, existing: ReplayBundle) -> None:
+    if (
+        existing.scenario_id != plan.scenario_id
+        or existing.scenario_digest != plan.scenario_digest
+        or existing.provenance != plan.provenance
+    ):
+        raise ContractError("Resume bundle metadata does not match the repeated-run plan")
+    allowed_cases = set(plan.case_ids)
+    for record in existing.records:
+        if record.case_id not in allowed_cases:
+            raise ContractError(f"Resume bundle contains unknown case {record.case_id!r}")
+        if not record.repeat_id.startswith("run-"):
+            raise ContractError("Resume bundle repeat IDs must use run-NNNN format")
+        try:
+            repeat_number = int(record.repeat_id.removeprefix("run-"))
+        except ValueError as exc:
+            raise ContractError("Resume bundle repeat IDs must use run-NNNN format") from exc
+        if not 1 <= repeat_number <= plan.repeats:
+            raise ContractError("Resume bundle contains a repeat outside the plan")
+
+
+def _metric_map(value: object, name: str) -> dict[str, float]:
+    if not isinstance(value, dict) or not value:
+        raise ContractError(f"{name} must be a non-empty metric object")
+    metrics: dict[str, float] = {}
+    for metric_name, metric_value in value.items():
+        if not isinstance(metric_name, str) or not metric_name:
+            raise ContractError(f"{name} metric names must be non-empty strings")
+        if isinstance(metric_value, bool) or not isinstance(metric_value, (int, float)):
+            raise ContractError(f"{name}.{metric_name} must be numeric")
+        numeric = float(metric_value)
+        if not math.isfinite(numeric):
+            raise ContractError(f"{name}.{metric_name} must be finite")
+        metrics[metric_name] = numeric
+    return metrics
+
+
+def _bundle(plan: RepeatedRunPlan, records: tuple[MetricObservation, ...]) -> ReplayBundle:
+    return ReplayBundle(
+        schema_version="0.1",
+        scenario_id=plan.scenario_id,
+        scenario_digest=plan.scenario_digest,
+        provenance=plan.provenance,
+        records=records,
+    )