agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/replay.py

@@ -0,0 +1,753 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from fractions import Fraction
+from typing import Any
+
+from agentloop.markdown import markdown_code_span, markdown_table_cell
+from agentloop.tokens import is_token_basis_evaluable
+
+
+@dataclass(frozen=True)
+class ReplayGates:
+    max_cost_regression_pct: float = 0.0
+    max_latency_regression_pct: float = 0.0
+    min_latency_improvement_pct: float = 0.0
+    min_cost_improvement_pct: float = 0.0
+    require_retry_non_increase: bool = True
+    require_schema_valid: bool = False
+    min_quality_score: float | None = None
+
+
+def build_replay_report(
+    baseline_trace: Any,
+    candidate_trace: Any,
+    *,
+    gates: ReplayGates | None = None,
+    quality_report: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Compare a baseline and candidate trace and evaluate the configured gates.
+
+    A cost gate is only evaluated when both its inputs are trustworthy: every
+    model call has a known rate, *and* the token counts those rates multiply were
+    counted rather than approximated. When either is missing the cost gates go
+    indeterminate instead of comparing numbers that cannot support the comparison
+    (issue #133); ``gates.pricing_known`` and ``gates.token_basis_evaluable`` say
+    which input was missing.
+    """
+
+    from agentloop.retention import require_complete_evidence
+
+    require_complete_evidence(baseline_trace, "replay")
+    require_complete_evidence(candidate_trace, "replay")
+    from agentloop.integrations.harbor.trial_evidence import require_trial_pair
+
+    require_trial_pair(baseline_trace, candidate_trace)
+    from agentloop.interoperability.cohort_evidence import require_cohort_pair
+
+    require_cohort_pair(baseline_trace, candidate_trace)
+    gates = gates or ReplayGates()
+    baseline_report = baseline_trace.report()
+    candidate_report = candidate_trace.report()
+    baseline = _trace_summary(baseline_trace, baseline_report)
+    candidate = _trace_summary(candidate_trace, candidate_report)
+    if quality_report is not None:
+        if quality_report.get("schema_version") == "2.0":
+            from agentloop.structured_quality import decision_span_count, validate_report
+
+            quality_report = validate_report(
+                quality_report, baseline_trace=baseline_trace, candidate_trace=candidate_trace
+            )
+
+            baseline["quality_score"] = _optional_float(quality_report.get("baseline_score"))
+            candidate["quality_score"] = _optional_float(quality_report.get("candidate_score"))
+            baseline["decision_count"] = decision_span_count(baseline_trace)
+            candidate["decision_count"] = decision_span_count(candidate_trace)
+            baseline["error_span_count"] = sum(
+                event.status == "error" for event in baseline_trace.events
+            )
+            candidate["error_span_count"] = sum(
+                event.status == "error" for event in candidate_trace.events
+            )
+        else:
+            baseline["quality_score"] = float(quality_report.get("baseline_score", 0.0))
+            candidate["quality_score"] = float(quality_report.get("candidate_score", 0.0))
+    pricing_known = not (
+        bool(baseline.get("has_unknown_cost")) or bool(candidate.get("has_unknown_cost"))
+    )
+    token_basis_evaluable = is_token_basis_evaluable(
+        baseline.get("token_status")
+    ) and is_token_basis_evaluable(candidate.get("token_status"))
+    cost_evaluable = pricing_known and token_basis_evaluable
+    deltas = _deltas(baseline, candidate)
+    if "external_trial" in baseline or "external_trial" in candidate:
+        nullable = {
+            "input_tokens": ("input_tokens_delta", "input_token_improvement_pct"),
+            "output_tokens": ("output_tokens_delta", "output_token_improvement_pct"),
+            "retry_count": ("retry_count_delta", "retry_improvement_pct"),
+            "tool_call_count": ("tool_call_count_delta", "tool_call_improvement_pct"),
+            "model_call_count": ("model_call_count_delta", "model_call_improvement_pct"),
+        }
+        for metric, fields in nullable.items():
+            if baseline.get(metric) is None or candidate.get(metric) is None:
+                for key in fields:
+                    deltas[key] = None
+    versioned_quality = quality_report is not None and quality_report.get("schema_version") == "2.0"
+    if versioned_quality or "quality_evidence" in baseline or "quality_evidence" in candidate:
+        if baseline.get("quality_score") is None or candidate.get("quality_score") is None:
+            deltas["quality_score_delta"] = None
+        else:
+            deltas["quality_score_delta"] = float(
+                Fraction(str(candidate["quality_score"])) - Fraction(str(baseline["quality_score"]))
+            )
+        if "decision_count" in baseline and "decision_count" in candidate:
+            deltas["decision_count_delta"] = (
+                candidate["decision_count"] - baseline["decision_count"]
+            )
+            deltas["error_span_count_delta"] = (
+                candidate["error_span_count"] - baseline["error_span_count"]
+            )
+    if not cost_evaluable:
+        # The cost totals are lower bounds when a model is unpriced, and a rate
+        # times a word estimate when the token basis is approximate. Either way
+        # their delta and percentages are not a valid comparison — expose them as
+        # unavailable (None) rather than let a consumer read that arithmetic as a
+        # real number. The gates stay indeterminate below.
+        deltas["cost_usd_delta"] = None
+        deltas["cost_improvement_pct"] = None
+        deltas["cost_regression_pct"] = None
+    gate_results = _gate_results(
+        deltas,
+        baseline,
+        candidate,
+        gates,
+        quality_report=quality_report,
+        cost_evaluable=cost_evaluable,
+        pricing_known=pricing_known,
+    )
+    if "external_trial" in baseline:
+        outcomes = [summary["external_trial"]["outcome"] for summary in (baseline, candidate)]
+        available = all(outcome["quality_pass"] is not None for outcome in outcomes)
+        correct = available and all(
+            outcome["quality_pass"] is True and outcome["execution_status"] == "completed"
+            for outcome in outcomes
+        )
+        gate_results.append(
+            _gate(
+                "external_task_correctness",
+                correct,
+                "matching externally supplied verifier criteria passed"
+                if correct
+                else "external task correctness failed or is unavailable",
+                indeterminate=not available,
+            )
+        )
+    passed = all(item["passed"] for item in gate_results)
+    indeterminate_gates = [item["name"] for item in gate_results if item.get("indeterminate")]
+
+    return {
+        "baseline": baseline,
+        "candidate": candidate,
+        "deltas": deltas,
+        "gates": {
+            "passed": passed,
+            "cost_evaluable": cost_evaluable,
+            # Split out so a failing/indeterminate cost gate says *which* input
+            # was missing: an unpriced model, or token counts that were only
+            # approximated (issue #133).
+            "pricing_known": pricing_known,
+            "token_basis_evaluable": token_basis_evaluable,
+            "baseline_token_status": baseline.get("token_status"),
+            "candidate_token_status": candidate.get("token_status"),
+            "indeterminate": indeterminate_gates,
+            "config": {
+                "max_cost_regression_pct": gates.max_cost_regression_pct,
+                "max_latency_regression_pct": gates.max_latency_regression_pct,
+                "min_latency_improvement_pct": gates.min_latency_improvement_pct,
+                "min_cost_improvement_pct": gates.min_cost_improvement_pct,
+                "require_retry_non_increase": gates.require_retry_non_increase,
+                "require_schema_valid": gates.require_schema_valid,
+                "min_quality_score": gates.min_quality_score,
+            },
+            "results": gate_results,
+        },
+        "quality": quality_report,
+        "summary": _summary(
+            deltas, passed, cost_evaluable=cost_evaluable, pricing_known=pricing_known
+        ),
+    }
+
+
+def replay_report_to_markdown(report: dict[str, Any]) -> str:
+    baseline = report["baseline"]
+    candidate = report["candidate"]
+    deltas = report["deltas"]
+    gates = report["gates"]
+    status = "passed" if gates["passed"] else "failed"
+
+    lines = [
+        "# AgentLoop Replay Report",
+        "",
+        f"- Status: {status}",
+        f"- Baseline: {markdown_code_span(baseline['name'])} "
+        f"{markdown_code_span(baseline['run_id'])}",
+        f"- Candidate: {markdown_code_span(candidate['name'])} "
+        f"{markdown_code_span(candidate['run_id'])}",
+        "",
+        "## Metric Deltas",
+        "",
+        "| Metric | Baseline | Candidate | Delta | Improvement |",
+        "|---|---:|---:|---:|---:|",
+        _metric_row(
+            "Runtime",
+            baseline["runtime_ms"],
+            candidate["runtime_ms"],
+            deltas["runtime_ms_delta"],
+            deltas["latency_improvement_pct"],
+            "ms",
+        ),
+        _cost_metric_row(baseline, candidate, deltas),
+        _metric_row(
+            "Input tokens",
+            baseline["input_tokens"],
+            candidate["input_tokens"],
+            deltas["input_tokens_delta"],
+            deltas["input_token_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Output tokens",
+            baseline["output_tokens"],
+            candidate["output_tokens"],
+            deltas["output_tokens_delta"],
+            deltas["output_token_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Retries",
+            baseline["retry_count"],
+            candidate["retry_count"],
+            deltas["retry_count_delta"],
+            deltas["retry_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Tool calls",
+            baseline["tool_call_count"],
+            candidate["tool_call_count"],
+            deltas["tool_call_count_delta"],
+            deltas["tool_call_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Model calls",
+            baseline["model_call_count"],
+            candidate["model_call_count"],
+            deltas["model_call_count_delta"],
+            deltas["model_call_improvement_pct"],
+            "count",
+        ),
+    ]
+    if (
+        baseline.get("schema_validity_pct") is not None
+        or candidate.get("schema_validity_pct") is not None
+    ):
+        lines.append(
+            _metric_row(
+                "Schema validity",
+                baseline.get("schema_validity_pct") or 0.0,
+                candidate.get("schema_validity_pct") or 0.0,
+                deltas["schema_validity_delta_pct"],
+                deltas["schema_validity_improvement_pct"],
+                "pct",
+            )
+        )
+    if "decision_count_delta" in deltas:
+        lines.append(
+            _metric_row(
+                "Recorded decision spans",
+                baseline["decision_count"],
+                candidate["decision_count"],
+                deltas["decision_count_delta"],
+                None,
+                "count",
+            )
+        )
+        lines.append(
+            _metric_row(
+                "Recorded error spans",
+                baseline["error_span_count"],
+                candidate["error_span_count"],
+                deltas["error_span_count_delta"],
+                None,
+                "count",
+            )
+        )
+    versioned_quality = (
+        report.get("quality", {}).get("schema_version") == "2.0"
+        if report.get("quality") is not None
+        else False
+    )
+    versioned_quality = (
+        versioned_quality or "quality_evidence" in baseline or "quality_evidence" in candidate
+    )
+    if (
+        baseline.get("quality_score") is not None
+        or candidate.get("quality_score") is not None
+        or versioned_quality
+    ):
+        lines.append(
+            _metric_row(
+                "Quality score",
+                baseline.get("quality_score")
+                if versioned_quality
+                else baseline.get("quality_score") or 0.0,
+                candidate.get("quality_score")
+                if versioned_quality
+                else candidate.get("quality_score") or 0.0,
+                deltas["quality_score_delta"],
+                deltas["quality_score_improvement_pct"],
+                "exact_score" if versioned_quality else "score",
+            )
+        )
+    if report.get("quality"):
+        quality = report["quality"]
+        lines.extend(
+            [
+                "",
+                "## Quality",
+                "",
+                f"- Cases: {quality['case_count']}",
+                f"- Candidate score: {_format_metric(quality['candidate_score'], 'exact_score' if quality.get('schema_version') == '2.0' else 'score')}",
+                f"- Quality delta: {_format_metric(quality['quality_delta'], 'exact_score' if quality.get('schema_version') == '2.0' else 'score')}",
+                f"- Failed cases: {quality['failed_case_count']}",
+            ]
+        )
+    lines.extend(
+        [
+            "",
+            "## Gates",
+            "",
+            "| Gate | Status | Detail |",
+            "|---|---|---|",
+        ]
+    )
+    for item in gates["results"]:
+        if item.get("indeterminate"):
+            gate_status = "indeterminate"
+        else:
+            gate_status = "pass" if item["passed"] else "fail"
+        lines.append(
+            f"| {markdown_table_cell(item['name'])} | {gate_status} | "
+            f"{markdown_table_cell(item['detail'])} |"
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _trace_summary(trace: Any, report: dict[str, Any]) -> dict[str, Any]:
+    metadata = getattr(trace, "metadata", {}) or {}
+    cost = report.get("cost_breakdown") or {}
+    result = {
+        **(
+            {
+                "quality_evidence": report["quality_evidence"],
+                "decision_count": report["decision_count"],
+                "error_span_count": report["error_span_count"],
+            }
+            if "quality_evidence" in report
+            else {}
+        ),
+        **({"execution": report["execution"]} if "execution" in report else {}),
+        **({"stages": report["stages"]} if "stages" in report else {}),
+        "run_id": trace.run_id,
+        "name": trace.name,
+        "runtime_ms": float(report.get("total_runtime_ms", 0.0) or 0.0),
+        "estimated_cost_usd": float(report.get("estimated_cost_usd", 0.0) or 0.0),
+        # Carried so the cost gates can tell "no cost" apart from "cost unknown":
+        # estimated_cost_usd is the *known* portion, has_unknown_cost flags that
+        # some model calls had no available rate, and cost_status summarizes
+        # completeness (see costs.py / metrics.py).
+        "has_unknown_cost": bool(cost.get("has_unknown_cost", False)),
+        "cost_status": report.get("cost_status", cost.get("cost_status", "complete")),
+        # How exact the token totals below are, and therefore how exact any cost
+        # calculated from them is. "unspecified" means the trace predates token
+        # provenance; agentloop.tokens.is_token_basis_evaluable() explains why
+        # that still gates.
+        "token_status": report.get("token_status", cost.get("token_status", "unspecified")),
+        "operation_counts": report.get("operation_counts", {}),
+        "input_tokens": int(report.get("input_tokens", 0) or 0),
+        "output_tokens": int(report.get("output_tokens", 0) or 0),
+        "retry_count": int(report.get("retry_count", 0) or 0),
+        "tool_call_count": int(report.get("tool_call_count", 0) or 0),
+        "model_call_count": int(report.get("model_call_count", 0) or 0),
+        "schema_validity_pct": _optional_float(
+            report.get("schema_validity_pct")
+            if report.get("schema_validity_pct") is not None
+            else metadata.get("schema_validity_pct")
+        ),
+        "schema_valid": _optional_bool(
+            report.get("schema_valid")
+            if report.get("schema_valid") is not None
+            else metadata.get("schema_valid")
+        ),
+        "quality_score": _optional_float(
+            report.get("quality_score")
+            if "quality_evidence" in report or report.get("quality_score") is not None
+            else metadata.get("quality_score")
+        ),
+    }
+    from agentloop.integrations.harbor.trial_evidence import read_trial, trial_study_values
+
+    trial = read_trial(trace)
+    if trial is not None:
+        measurement = trial["measurement"]
+        result.update(
+            external_trial=trial,
+            runtime_ms=measurement["runtime_ms"],
+            estimated_cost_usd=measurement["cost_usd"],
+            has_unknown_cost=measurement["cost_usd"] is None,
+            input_tokens=measurement["input_tokens"],
+            output_tokens=measurement["output_tokens"],
+            quality_score=trial_study_values(trial)[0],
+            retry_count=None,
+            tool_call_count=None,
+            model_call_count=None,
+            quality_basis="configured_binary_acceptance",
+            cost_basis="external_reported_agent_context",
+        )
+    return result
+
+
+def _deltas(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
+    base_runtime = float(baseline.get("runtime_ms", baseline.get("total_runtime_ms", 0.0)) or 0.0)
+    cand_runtime = float(candidate.get("runtime_ms", candidate.get("total_runtime_ms", 0.0)) or 0.0)
+    base_cost = float(baseline.get("estimated_cost_usd", 0.0) or 0.0)
+    cand_cost = float(candidate.get("estimated_cost_usd", 0.0) or 0.0)
+    base_input = int(baseline.get("input_tokens", 0) or 0)
+    cand_input = int(candidate.get("input_tokens", 0) or 0)
+    base_output = int(baseline.get("output_tokens", 0) or 0)
+    cand_output = int(candidate.get("output_tokens", 0) or 0)
+    base_retries = int(baseline.get("retry_count", 0) or 0)
+    cand_retries = int(candidate.get("retry_count", 0) or 0)
+    base_tools = int(baseline.get("tool_call_count", 0) or 0)
+    cand_tools = int(candidate.get("tool_call_count", 0) or 0)
+    base_models = int(baseline.get("model_call_count", 0) or 0)
+    cand_models = int(candidate.get("model_call_count", 0) or 0)
+    base_schema = _optional_float(baseline.get("schema_validity_pct"))
+    if base_schema is None:
+        base_schema = _schema_validity_from_bool(baseline.get("schema_valid"))
+    cand_schema = _optional_float(candidate.get("schema_validity_pct"))
+    if cand_schema is None:
+        cand_schema = _schema_validity_from_bool(candidate.get("schema_valid"))
+    base_quality = _optional_float(baseline.get("quality_score"))
+    cand_quality = _optional_float(candidate.get("quality_score"))
+
+    return {
+        "runtime_ms_delta": round(cand_runtime - base_runtime, 3),
+        "latency_improvement_pct": _improvement_pct(base_runtime, cand_runtime),
+        "latency_regression_pct": _regression_pct(base_runtime, cand_runtime),
+        "cost_usd_delta": round(cand_cost - base_cost, 6),
+        "cost_improvement_pct": _improvement_pct(base_cost, cand_cost),
+        "cost_regression_pct": _regression_pct(base_cost, cand_cost),
+        "input_tokens_delta": cand_input - base_input,
+        "input_token_improvement_pct": _improvement_pct(base_input, cand_input),
+        "output_tokens_delta": cand_output - base_output,
+        "output_token_improvement_pct": _improvement_pct(base_output, cand_output),
+        "retry_count_delta": cand_retries - base_retries,
+        "retry_improvement_pct": _improvement_pct(base_retries, cand_retries),
+        "tool_call_count_delta": cand_tools - base_tools,
+        "tool_call_improvement_pct": _improvement_pct(base_tools, cand_tools),
+        "model_call_count_delta": cand_models - base_models,
+        "model_call_improvement_pct": _improvement_pct(base_models, cand_models),
+        "schema_validity_delta_pct": round((cand_schema or 0.0) - (base_schema or 0.0), 3),
+        "schema_validity_improvement_pct": _relative_gain_pct(base_schema, cand_schema),
+        "quality_score_delta": round((cand_quality or 0.0) - (base_quality or 0.0), 4),
+        "quality_score_improvement_pct": _relative_gain_pct(base_quality, cand_quality),
+    }
+
+
+def _gate_results(
+    deltas: dict[str, Any],
+    baseline: dict[str, Any],
+    candidate: dict[str, Any],
+    gates: ReplayGates,
+    *,
+    quality_report: dict[str, Any] | None = None,
+    cost_evaluable: bool = True,
+    pricing_known: bool = True,
+) -> list[dict[str, Any]]:
+    if cost_evaluable:
+        cost_regression_gate = _gate(
+            "cost_regression",
+            deltas["cost_regression_pct"] <= gates.max_cost_regression_pct,
+            f"{deltas['cost_regression_pct']:.2f}% regression "
+            f"<= {gates.max_cost_regression_pct:.2f}% allowed",
+        )
+        cost_improvement_gate = _gate(
+            "cost_improvement",
+            deltas["cost_improvement_pct"] >= gates.min_cost_improvement_pct,
+            f"{deltas['cost_improvement_pct']:.2f}% improvement "
+            f">= {gates.min_cost_improvement_pct:.2f}% required",
+        )
+    else:
+        # A missing rate — or a token count that was only approximated — is not
+        # itself evidence of a regression, so the regression gate stays
+        # non-failing rather than blocking every latency-only optimization that
+        # uses an unpriced model. But a *required* cost improvement can't be
+        # verified, so it must fail — we won't claim an improvement we can't
+        # compute.
+        reason = _cost_indeterminate_reason(pricing_known)
+        cost_regression_gate = _indeterminate_cost_gate(
+            "cost_regression", fail=False, reason=reason
+        )
+        cost_improvement_gate = _indeterminate_cost_gate(
+            "cost_improvement", fail=gates.min_cost_improvement_pct > 0, reason=reason
+        )
+    results = [
+        _gate(
+            "latency_regression",
+            deltas["latency_regression_pct"] <= gates.max_latency_regression_pct,
+            f"{deltas['latency_regression_pct']:.2f}% regression <= {gates.max_latency_regression_pct:.2f}% allowed",
+        ),
+        cost_regression_gate,
+        _gate(
+            "latency_improvement",
+            deltas["latency_improvement_pct"] >= gates.min_latency_improvement_pct,
+            f"{deltas['latency_improvement_pct']:.2f}% improvement >= {gates.min_latency_improvement_pct:.2f}% required",
+        ),
+        cost_improvement_gate,
+    ]
+    if gates.require_retry_non_increase:
+        if "external_trial" in baseline and (
+            baseline.get("retry_count") is None or candidate.get("retry_count") is None
+        ):
+            results.append(
+                _gate(
+                    "retry_non_increase",
+                    False,
+                    "source retry counts are unavailable",
+                    indeterminate=True,
+                )
+            )
+        else:
+            base_retries = int(baseline.get("retry_count", 0) or 0)
+            cand_retries = int(candidate.get("retry_count", 0) or 0)
+            results.append(
+                _gate(
+                    "retry_non_increase",
+                    cand_retries <= base_retries,
+                    f"{cand_retries} candidate retries <= {base_retries} baseline retries",
+                )
+            )
+    if gates.require_schema_valid:
+        schema_valid = _candidate_schema_valid(candidate)
+        results.append(
+            _gate(
+                "schema_valid",
+                schema_valid,
+                "candidate schema validity passed"
+                if schema_valid
+                else "candidate schema validity missing or failed",
+            )
+        )
+    if quality_report is not None:
+        fixture_passed = bool(quality_report.get("passed"))
+        failed_count = int(quality_report.get("failed_case_count", 0) or 0)
+        versioned = quality_report.get("schema_version") == "2.0"
+        unavailable = quality_report.get("indeterminate_case_count", 0) if versioned else 0
+        results.append(
+            _gate(
+                "quality_fixtures",
+                fixture_passed,
+                "supplied quality fixtures passed"
+                if fixture_passed
+                else (
+                    f"structured quality criteria failed; {failed_count} nonpassing cases, {unavailable} indeterminate cases; configured aggregate threshold retained"
+                    if versioned
+                    else f"{failed_count} supplied quality fixture case(s) failed"
+                ),
+                indeterminate=bool(unavailable),
+            )
+        )
+    if quality_report is None and "quality_evidence" in candidate:
+        evidence = candidate["quality_evidence"]
+        results.append(
+            _gate(
+                "quality_evidence",
+                evidence.get("status") == "complete" and evidence.get("passed") is True,
+                "attached structured quality evidence passed"
+                if evidence.get("passed")
+                else "attached structured quality evidence is nonpassing or unavailable",
+                indeterminate=evidence.get("status") != "complete",
+            )
+        )
+    if gates.min_quality_score is not None:
+        quality_score = _optional_float(candidate.get("quality_score"))
+        passed = quality_score is not None and quality_score >= gates.min_quality_score
+        detail = (
+            f"{quality_score:.4f} candidate quality >= {gates.min_quality_score:.4f} required"
+            if quality_score is not None
+            else f"candidate quality score missing; {gates.min_quality_score:.4f} required"
+        )
+        versioned = (
+            quality_report is not None
+            and quality_report.get("schema_version") == "2.0"
+            or "quality_evidence" in candidate
+        )
+        results.append(
+            _gate(
+                "quality_score", passed, detail, indeterminate=versioned and quality_score is None
+            )
+        )
+    return results
+
+
+def _gate(name: str, passed: bool, detail: str, *, indeterminate: bool = False) -> dict[str, Any]:
+    return {"name": name, "passed": passed, "detail": detail, "indeterminate": indeterminate}
+
+
+def _cost_indeterminate_reason(pricing_known: bool) -> str:
+    """Explain which missing input made a cost comparison indeterminate."""
+
+    if not pricing_known:
+        return "a model call has no known pricing (see cost_breakdown.unknown_models)"
+    return (
+        "token counts were approximated from word counts, so any cost derived "
+        "from them is not a measurement (see token_status)"
+    )
+
+
+def _indeterminate_cost_gate(
+    name: str, *, fail: bool, reason: str = "a model call has no known pricing"
+) -> dict[str, Any]:
+    """A cost gate that couldn't be evaluated from the evidence available.
+
+    Two things can make a cost comparison meaningless: an unpriced model (the
+    totals are lower bounds) or token counts that were only approximated (the
+    totals are a rate times a guess). Rather than silently comparing those
+    numbers, the gate is marked ``indeterminate``. It fails only when the caller
+    explicitly required a cost outcome (``fail``) that therefore can't be
+    verified; otherwise it is a non-failing informational gate.
+    """
+    detail = f"cost could not be evaluated: {reason}. " + (
+        "failing because a cost outcome was required." if fail else "not gating."
+    )
+    return _gate(name, passed=not fail, detail=detail, indeterminate=True)
+
+
+def _summary(
+    deltas: dict[str, Any],
+    passed: bool,
+    *,
+    cost_evaluable: bool = True,
+    pricing_known: bool = True,
+) -> str:
+    status = "passed" if passed else "failed"
+    if cost_evaluable:
+        cost_phrase = f"cost improvement {deltas['cost_improvement_pct']:.2f}%"
+    elif not pricing_known:
+        cost_phrase = "cost improvement unavailable (unknown model pricing)"
+    else:
+        cost_phrase = "cost improvement unavailable (token counts are estimates)"
+    return (
+        f"Replay {status}: latency improvement {deltas['latency_improvement_pct']:.2f}%, "
+        f"{cost_phrase}, "
+        f"retry delta {deltas['retry_count_delta']}."
+    )
+
+
+def _improvement_pct(baseline: float, candidate: float) -> float:
+    if baseline == 0:
+        return 0.0 if candidate == 0 else -100.0
+    return round(((baseline - candidate) / baseline) * 100, 2)
+
+
+def _relative_gain_pct(baseline: float | None, candidate: float | None) -> float:
+    if baseline is None:
+        return 0.0 if candidate is None else 100.0
+    if baseline == 0:
+        return 0.0 if not candidate else 100.0
+    return round((((candidate or 0.0) - baseline) / baseline) * 100, 2)
+
+
+def _regression_pct(baseline: float, candidate: float) -> float:
+    if baseline == 0:
+        return 0.0 if candidate == 0 else 100.0
+    return round(max(0.0, ((candidate - baseline) / baseline) * 100), 2)
+
+
+def _metric_row(
+    metric: str,
+    baseline: float,
+    candidate: float,
+    delta: float | None,
+    improvement: float | None,
+    kind: str,
+) -> str:
+    improvement_cell = "unavailable" if improvement is None else f"{improvement:.2f}%"
+    return (
+        f"| {metric} | {_format_metric(baseline, kind)} | {_format_metric(candidate, kind)} | "
+        f"{_format_metric(delta, kind)} | {improvement_cell} |"
+    )
+
+
+def _cost_metric_row(
+    baseline: dict[str, Any], candidate: dict[str, Any], deltas: dict[str, Any]
+) -> str:
+    from agentloop.costs import format_cost_usd
+
+    improvement = deltas["cost_improvement_pct"]
+    improvement_cell = "unavailable" if improvement is None else f"{improvement:.2f}%"
+    return (
+        "| Cost | "
+        f"{format_cost_usd(baseline.get('estimated_cost_usd'), baseline.get('cost_status'), decimals=6)} | "
+        f"{format_cost_usd(candidate.get('estimated_cost_usd'), candidate.get('cost_status'), decimals=6)} | "
+        f"{_format_metric(deltas['cost_usd_delta'], 'usd')} | {improvement_cell} |"
+    )
+
+
+def _format_metric(value: float | None, kind: str) -> str:
+    if value is None:
+        return "unavailable"
+    if kind == "usd":
+        return f"${value:.6f}"
+    if kind == "ms":
+        return f"{value:.3f}ms"
+    if kind == "pct":
+        return f"{value:.3f}%"
+    if kind == "score":
+        return f"{value:.4f}"
+    if kind == "exact_score":
+        return str(value)
+    return str(int(value))
+
+
+def _optional_float(value: Any) -> float | None:
+    if value is None or value == "":
+        return None
+    return float(value)
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "pass", "passed", "valid"}
+    return bool(value)
+
+
+def _schema_validity_from_bool(value: Any) -> float | None:
+    parsed = _optional_bool(value)
+    if parsed is None:
+        return None
+    return 100.0 if parsed else 0.0
+
+
+def _candidate_schema_valid(candidate: dict[str, Any]) -> bool:
+    schema_valid = _optional_bool(candidate.get("schema_valid"))
+    if schema_valid is not None:
+        return schema_valid
+    validity_pct = _optional_float(candidate.get("schema_validity_pct"))
+    return validity_pct is not None and validity_pct >= 100.0