world-model-optimizer 0.2.0__py3-none-any.whl

wmo/evals/failover.py

@@ -0,0 +1,96 @@
+"""Same-model failover for the eval grid (never silently switches models).
+
+Main's failover architecture (`wmo.providers.provider_or_chain` + `.wmo/fallback.toml`) rides a
+config-driven chain for *world-model* calls and keeps the *judge* pinned. The grid needs one thing
+that config can't express: a **programmatic, per-cell, same-model** chain. Its judge is a single
+pinned model and each cell's target is a distinct model, so there is no static named chain to point
+at - yet Bedrock Anthropic models throttle hard, and the direct Anthropic API is the *identical*
+model on an un-throttled endpoint. Failing over Bedrock -> direct-Anthropic (same model), or across
+Bedrock regions (same model), keeps what's measured unchanged, so it honours main's
+never-silently-switch-models invariant while surviving capacity pressure over an 80-cell grid.
+
+This is deliberately NOT in `wmo.providers`: main removed the programmatic `FallbackProvider` in
+favour of `.wmo/fallback.toml`, and this seam exists only because the grid composes pre-built
+providers (metered/capped wrappers, injected fakes in tests) into same-model chains. Capacity
+classification is reused from the shared llm-waterfall package (`is_capacity_error`), not
+re-implemented here.
+"""
+
+from __future__ import annotations
+
+import re
+
+from llm_waterfall import is_capacity_error
+
+from wmo.providers.base import Completion, Message, Provider, ProviderConfig
+
+
+def anthropic_direct_id(bedrock_model: str) -> str | None:
+    """The direct-Anthropic-API model id for a Bedrock Anthropic model id, or None otherwise.
+
+    Bedrock Anthropic models are heavily capacity-constrained; the direct Anthropic API is the SAME
+    model on a different endpoint (and a key that isn't Bedrock-rate-limited), so failing over to it
+    keeps what's measured identical. Strips the `us.anthropic.`/`anthropic.` prefix and any
+    dated/versioned tail: `us.anthropic.claude-opus-4-8` -> `claude-opus-4-8`;
+    `us.anthropic.claude-haiku-4-5-20251001-v1:0` -> `claude-haiku-4-5`.
+    """
+    for prefix in ("us.anthropic.", "anthropic."):
+        if bedrock_model.startswith(prefix):
+            m = bedrock_model[len(prefix) :]
+            m = re.sub(r"-\d{8}.*$", "", m)  # drop a -YYYYMMDD... date+version tail
+            m = re.sub(r"-v\d+(:\d+)?$", "", m)  # drop a -v1 / -v1:0 tail
+            return m
+    return None
+
+
+class SameModelFailover:
+    """Try a chain of pre-built providers in order; fail over only on capacity errors.
+
+    Every rung MUST serve the same underlying model (a Bedrock model, its direct-Anthropic twin, or
+    the same model in another region) - the chain spreads throttling load without changing what is
+    measured. Capacity errors (throttling, transient 5xx, timeouts - see llm-waterfall's
+    `is_capacity_error`) spill to the next rung; a real error (bad request, auth) propagates
+    immediately, and the last rung's error surfaces when the whole chain is capacity-constrained.
+    """
+
+    def __init__(self, chain: list[Provider]) -> None:
+        if not chain:
+            raise ValueError("SameModelFailover needs at least one provider")
+        self._chain = chain
+        self.config: ProviderConfig = chain[0].config
+
+    def complete(
+        self,
+        system: str,
+        messages: list[Message],
+        *,
+        temperature: float = 0.7,
+        max_tokens: int = 8192,
+    ) -> Completion:
+        last: Exception | None = None
+        for provider in self._chain:
+            try:
+                return provider.complete(
+                    system, messages, temperature=temperature, max_tokens=max_tokens
+                )
+            except Exception as exc:  # noqa: BLE001 - classify, then re-raise or fall over
+                if not is_capacity_error(exc):
+                    raise
+                last = exc
+        assert last is not None  # noqa: S101 - the loop ran at least once (chain is non-empty)
+        raise last
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return self._chain[0].embed(texts)
+
+    def verify(self):  # noqa: ANN201 - delegate to the primary; unused on the eval path
+        return self._chain[0].verify()
+
+
+def same_model_chain(
+    configs: list[ProviderConfig],
+    factory,  # noqa: ANN001 - (ProviderConfig) -> Provider, injectable for tests
+) -> Provider:
+    """Build a `SameModelFailover` over `configs`; a single-config chain is returned unwrapped."""
+    chain = [factory(c) for c in configs]
+    return chain[0] if len(chain) == 1 else SameModelFailover(chain)

wmo/evals/gold.py

@@ -0,0 +1,127 @@
+"""Gold-assertion judging: did the agent's run satisfy the task's success conditions?
+
+Closed-loop eval scores *task success*, not per-step fidelity (docs/reference/closed_loop.md).
+Success is defined by the task's `gold` assertions — semantic post-conditions checked against the
+transcript by an LLM judge, so they are robust to wording (exact-match rules systematically
+under-report success on semantically-correct answers). The verdict is always scored against the FULL
+gold list: a truncated judge reply that omits assertions cannot report success.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, ValidationError, field_validator
+
+from wmo.core.parsing import extract_json_object
+from wmo.core.text import normalize_durable_text
+from wmo.providers.base import Message, Provider
+
+GOLD_JUDGE_MARKER = "grade whether an agent completed a task"
+
+GOLD_JUDGE_SYSTEM = """You grade whether an agent completed a task. You are given the task, the
+agent's final answer, a transcript of what it did, and a list of GOLD assertions defining success.
+For each assertion, decide whether the transcript+answer show it is satisfied. Judge by MEANING, not
+wording: an assertion holds if the evidence shows the described outcome, even if phrased otherwise.
+Do not give credit for merely attempting or claiming success without supporting evidence in the
+transcript.
+
+Respond with ONLY a JSON object, no prose:
+{"assertions": [{"assertion": "<verbatim>", "passed": <bool>, "why": "<short reason>"}],
+ "passed": <bool: true iff every assertion passed>}"""
+
+
+class AssertionResult(BaseModel):
+    assertion: str
+    passed: bool
+    why: str = ""
+
+    @field_validator("assertion", "why")
+    @classmethod
+    def _normalize_text(cls, value: str) -> str:
+        return normalize_durable_text(value)
+
+
+class GoldVerdict(BaseModel):
+    """The judge's verdict on one run against its gold assertions."""
+
+    passed: bool = False  # every assertion satisfied
+    fraction: float = 0.0  # fraction of assertions satisfied (partial-credit signal)
+    assertions: list[AssertionResult] = Field(default_factory=list)
+    rationale: str = ""
+
+    @field_validator("rationale")
+    @classmethod
+    def _normalize_rationale(cls, value: str) -> str:
+        return normalize_durable_text(value)
+
+    @classmethod
+    def trivially_passed(cls) -> GoldVerdict:
+        """A run with no gold assertions cannot fail its (empty) spec; treat as passed."""
+        return cls(passed=True, fraction=1.0, rationale="no gold assertions")
+
+
+class _RawVerdict(BaseModel):
+    assertions: list[AssertionResult] = Field(default_factory=list)
+    passed: bool = False
+
+
+class GoldJudge:
+    """LLM judge that checks a run transcript against a task's gold assertions."""
+
+    def __init__(self, provider: Provider) -> None:
+        self._provider = provider
+
+    def score(self, instruction: str, answer: str, transcript: str, gold: list[str]) -> GoldVerdict:
+        if not gold:
+            return GoldVerdict.trivially_passed()
+        user = _build_prompt(instruction, answer, transcript, gold)
+        completion = self._provider.complete(
+            GOLD_JUDGE_SYSTEM,
+            [Message(role="user", content=user)],
+            temperature=0.0,
+            max_tokens=1024,
+        )
+        return _parse(completion.text, gold)
+
+
+def _build_prompt(instruction: str, answer: str, transcript: str, gold: list[str]) -> str:
+    assertions = "\n".join(f"- {g}" for g in gold)
+    return (
+        f"TASK:\n{instruction}\n\n"
+        f"AGENT FINAL ANSWER:\n{answer or '(none)'}\n\n"
+        f"TRANSCRIPT:\n{transcript or '(empty)'}\n\n"
+        f"GOLD ASSERTIONS (all must hold for success):\n{assertions}\n"
+    )
+
+
+def _parse(text: str, gold: list[str]) -> GoldVerdict:
+    """Parse the judge reply; fall back to a failed verdict if it is unusable."""
+    raw = extract_json_object(text)
+    if raw is not None:
+        try:
+            parsed = _RawVerdict.model_validate_json(raw)
+        except ValidationError:
+            parsed = None
+        if parsed is not None and parsed.assertions:
+            # Score against ALL required gold assertions, matched BY TEXT to what the judge echoed
+            # back (the prompt demands verbatim echoes). A truncated reply that omits an assertion,
+            # one that duplicates a passing assertion to pad the count, or one that echoes the same
+            # assertion as both failed and passed (a failing verdict is never discarded) cannot
+            # report success: every unmatched gold assertion counts as failed. Fail-closed by
+            # construction.
+            passed_texts = {a.assertion.strip() for a in parsed.assertions if a.passed}
+            failed_texts = {a.assertion.strip() for a in parsed.assertions if not a.passed}
+            total = len(gold)
+            n_pass = sum(
+                1 for g in gold if g.strip() in passed_texts and g.strip() not in failed_texts
+            )
+            return GoldVerdict(
+                passed=parsed.passed and n_pass == total,
+                fraction=n_pass / total if total else 1.0,
+                assertions=parsed.assertions,
+                rationale=f"{n_pass}/{total} assertions satisfied",
+            )
+    return GoldVerdict(
+        passed=False,
+        fraction=0.0,
+        rationale=f"unparseable judge response; treated as failure. raw: {text.strip()[:200]}",
+    )

wmo/evals/grid.py

@@ -0,0 +1,394 @@
+"""Model-comparison grid: run one eval suite across many (model x condition) cells.
+
+A "grid" answers a single question for a benchmark: how do different serving models fare, each under
+base / +RAG / +GEPA / +GEPA+RAG, on the SAME held-out split, scored by the SAME judge? It reuses
+the open-loop eval (`wmo.evals.open_loop.evaluate_files`) once per cell and rolls the per-file
+report into a `GridCell`. Two invariants make cells comparable:
+
+- The **judge is pinned** (a single Bedrock Opus 4.8 `RubricJudge`) across every cell, independent
+  of the target model - a Qwen target must not be judged by Qwen - and it never switches models
+  (only to the SAME model on the direct Anthropic API under Bedrock throttling; see
+  `wmo.evals.failover`). Its `JUDGE_VERSION` is stamped on the result so numbers from different
+  judge generations are never silently compared.
+- Target token **cost is metered separately** from the judge (a `MeteredProvider` wraps only the
+  target), so a cell reports target-side cost, not judge cost. Cost is `None` when the model has no
+  pricing row (see `wmo.tracking.pricing.price_for`) rather than a misleading 0.
+
+`run_grid` is provider-agnostic: each `ModelSpec` names a provider/model the registry can build, so
+a self-hosted OpenAI-compatible model (e.g. Qwen-AgentWorld on vLLM) is just `provider="openai"`
+with `OPENAI_BASE_URL` in the environment.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+
+from pydantic import BaseModel, Field
+
+from wmo.evals.failover import anthropic_direct_id, same_model_chain
+from wmo.evals.open_loop import EvalReport, evaluate_files
+from wmo.optimize.judge import JUDGE_VERSION, Judge, RubricJudge
+from wmo.providers import ProviderConfig, ProviderKind, get_provider
+from wmo.providers.base import Completion, Message, Provider
+from wmo.retrieval import HashingEmbedder
+from wmo.tracking import MeteredProvider, RunTracker
+from wmo.tracking.pricing import price_for
+
+# Regions a Bedrock TARGET fails over across (same model, so what's measured is unchanged - this
+# only spreads throttling load, it does not switch models).
+_TARGET_FALLBACK_REGIONS = ("us-east-1",)
+
+# The four prompt/retrieval conditions each model is evaluated under. `gepa`/`gepa_rag` require a
+# per-(benchmark x model) evolved prompt; they are skipped for a model with no GEPA prompt.
+CONDITIONS = ("base", "base_rag", "gepa", "gepa_rag")
+
+# Display labels (lowercase "wmo" per the chart convention).
+_CONDITION_LABELS = {
+    "base": "base",
+    "base_rag": "wmo/rag",
+    "gepa": "wmo/gepa",
+    "gepa_rag": "wmo/gepa/rag",
+}
+
+# Cap on the TARGET's generation per step. A world-model observation is short JSON; a reasoning
+# target (GPT-5.5) otherwise spends the full 8192-token budget on reasoning, making each step
+# ~80s and a whole grid many hours. 4096 leaves ample room for reasoning + the observation while
+# roughly halving worst-case latency. The judge is never capped (it needs its full rubric budget).
+DEFAULT_TARGET_MAX_TOKENS = 4096
+
+
+class CappedProvider:
+    """Wraps a target provider, clamping each completion's `max_tokens` to a ceiling.
+
+    Only the eval TARGET is wrapped (not the judge): observation prediction needs a short output, so
+    a lower ceiling bounds a reasoning model's per-step latency without affecting judge scoring.
+    """
+
+    def __init__(self, inner: Provider, cap: int) -> None:
+        self._inner = inner
+        self._cap = cap
+        self.config = inner.config
+
+    def complete(
+        self,
+        system: str,
+        messages: list[Message],
+        *,
+        temperature: float = 0.7,
+        max_tokens: int = 8192,
+    ) -> Completion:
+        return self._inner.complete(
+            system, messages, temperature=temperature, max_tokens=min(max_tokens, self._cap)
+        )
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return self._inner.embed(texts)
+
+    def verify(self):  # noqa: ANN201 - delegate; unused on the eval path
+        return self._inner.verify()
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    """A serving model to benchmark: display label + how the provider registry builds it."""
+
+    label: str  # e.g. "Opus 4.8"
+    provider: str  # ProviderKind value: "bedrock" | "openai" | ...
+    model: str  # provider model id, e.g. "us.anthropic.claude-opus-4-8"
+    region: str | None = None
+
+
+class GridCell(BaseModel):
+    """One (model x condition) result on a benchmark's held-out split."""
+
+    model_label: str
+    provider: str
+    model: str
+    condition: str  # one of CONDITIONS
+    condition_label: str  # human label incl. lowercase "wmo"
+    fidelity: float  # step-weighted mean judge score across the split
+    error_flag_acc: float  # step-weighted fraction where predicted is_error matched actual
+    n_steps: int
+    cost_usd: float | None = None  # target-side USD; None when the model has no pricing row
+
+    @property
+    def bar_label(self) -> str:
+        """Two-line x-axis label: "Opus 4.8\\nwmo/rag" (lowercase wmo)."""
+        return f"{self.model_label}\n{self.condition_label}"
+
+
+class GridResult(BaseModel):
+    """A full grid: every cell plus the split/judge metadata that makes the numbers reproducible."""
+
+    suite: str
+    judge_model: str
+    judge_provider: str
+    # The rubric-judge version every cell was scored under. Stamped so results from different judge
+    # generations are never silently compared (rubric-v1 numbers run ~0.12 higher than rubric-v2).
+    judge_version: str = JUDGE_VERSION
+    train_split: float
+    val_frac: float = 0.0  # validation fraction reserved for GEPA; test band = 1 - train - val
+    top_k: int
+    seed: int
+    sample_turns: str
+    # Retrieval phi dimensionality (`base_rag`/`gepa_rag` cells) and any dry-run holdout cap. Both
+    # change what/how cells are scored, so they are comparability fields (see `merge_results`):
+    # merging RAG cells built at different embed_dim, or a capped dry-run with a full run, would put
+    # incomparable fidelities in one chart.
+    embed_dim: int = 0
+    max_holdout_traces: int | None = None
+    total_test_steps: int = 0
+    total_test_traces: int = 0
+    cells: list[GridCell] = Field(default_factory=list)
+
+
+def merge_results(results: list[GridResult]) -> GridResult:
+    """Combine several `GridResult`s (e.g. a 4-API-model grid + a separate Qwen grid) into one.
+
+    A self-hosted model runs in its own process (its OpenAI-compatible base URL is process-global
+    via `OPENAI_BASE_URL`), so its cells arrive in a separate result JSON. All results must be the
+    same suite/split - they score the same held-out set - so metadata is taken from the first and
+    cells are concatenated. `total_test_steps`/`total_test_traces` take the max across results
+    (equal in practice; max guards against a capped dry-run being merged with a full run).
+    """
+    if not results:
+        raise ValueError("merge_results requires at least one GridResult")
+    head = results[0]
+    # Every result merged into one chart must be directly comparable: same suite, same judge
+    # (model + rubric version), and the SAME held-out split + retrieval config. Drift in any of
+    # these means the cells were scored on different bands or on different scales, so merging them
+    # would put incomparable fidelities side by side (rubric-v1 runs ~0.12 above rubric-v2; a
+    # different train_split/val_frac/seed reserves a different test band). The self-hosted grid
+    # runs in a SEPARATE process, so a flag typo there is a realistic way to get silent drift.
+    # Fail loudly instead of taking the first result's metadata and hiding the mismatch.
+    comparability_fields = (
+        "suite",
+        "judge_model",
+        "judge_version",
+        "train_split",
+        "val_frac",
+        "top_k",
+        "seed",
+        "sample_turns",
+        "embed_dim",
+        "max_holdout_traces",
+    )
+    for field in comparability_fields:
+        values = {getattr(r, field) for r in results}
+        if len(values) > 1:
+            raise ValueError(
+                f"merge_results needs one {field} (cells not comparable across values); "
+                f"got {sorted(str(v) for v in values)}"
+            )
+    merged = GridResult(
+        suite=head.suite,
+        judge_model=head.judge_model,
+        judge_provider=head.judge_provider,
+        judge_version=head.judge_version,
+        train_split=head.train_split,
+        val_frac=head.val_frac,
+        top_k=head.top_k,
+        seed=head.seed,
+        sample_turns=head.sample_turns,
+        embed_dim=head.embed_dim,
+        max_holdout_traces=head.max_holdout_traces,
+        total_test_steps=max(r.total_test_steps for r in results),
+        total_test_traces=max(r.total_test_traces for r in results),
+    )
+    for r in results:
+        merged.cells.extend(r.cells)
+    return merged
+
+
+def _make_judge(
+    judge_provider: str,
+    judge_model: str,
+    region: str | None,
+    factory,  # noqa: ANN001 - a Provider builder (ProviderConfig) -> Provider, injectable for tests
+) -> Judge:
+    """Build the pinned `RubricJudge`. The judge NEVER switches to a different model.
+
+    A judge that silently swapped models mid-grid would score cells on different scales and make
+    fidelity numbers incomparable (see `docs/reference/failover.md`). The ONLY failover allowed is
+    to the SAME model on the direct Anthropic API (unlimited key) when the primary is a throttled
+    Bedrock Anthropic model - identical model, different endpoint - so what's measured is unchanged.
+    The judge is never metered as target cost.
+    """
+    kind_enum = ProviderKind(judge_provider)
+    configs = [ProviderConfig(kind=kind_enum, model=judge_model, region=region)]
+    if kind_enum is ProviderKind.BEDROCK:
+        direct = anthropic_direct_id(judge_model)
+        if direct is not None:
+            configs.append(ProviderConfig(kind=ProviderKind.ANTHROPIC, model=direct))
+    return RubricJudge(same_model_chain(configs, factory))
+
+
+def _make_target(spec: ModelSpec, factory) -> Provider:  # noqa: ANN001 - factory injectable for tests
+    """Build a target provider. A Bedrock target fails over across `_TARGET_FALLBACK_REGIONS` (SAME
+    model - spreads throttle without changing what's measured); other providers are single."""
+    kind = ProviderKind(spec.provider)
+    if kind is ProviderKind.BEDROCK:
+        # Spread across regions only when the primary region is EXPLICIT: with region=None the
+        # primary already resolves via AWS_REGION/the boto3 chain, and appending a literal
+        # us-east-1 rung could just re-hit the same endpoint (a no-op duplicate) if that is what
+        # the ambient region resolves to. The direct-Anthropic rung below is the real resilience.
+        regions: list[str | None] = [spec.region]
+        if spec.region is not None:
+            regions += [r for r in _TARGET_FALLBACK_REGIONS if r != spec.region]
+        configs = [ProviderConfig(kind=kind, model=spec.model, region=r) for r in regions]
+        # Then fail over to the SAME model on the direct Anthropic API (unlimited key), so a target
+        # throttled across all Bedrock regions still produces real predictions on the identical
+        # model instead of scoring the step 0 - critical for Opus 4.8 under Bedrock load.
+        direct = anthropic_direct_id(spec.model)
+        if direct is not None:
+            configs.append(ProviderConfig(kind=ProviderKind.ANTHROPIC, model=direct))
+        return same_model_chain(configs, factory)
+    return factory(ProviderConfig(kind=kind, model=spec.model, region=spec.region))
+
+
+def _target_cost(model: str, tracker: RunTracker) -> float | None:
+    """Target-side USD from the metered tracker, or None when the model has no pricing row."""
+    if price_for(model) is None:
+        return None
+    return tracker.totals().cost_usd
+
+
+def _aggregate(report: EvalReport) -> tuple[float, float, int]:
+    """(fidelity, step-weighted error-flag accuracy, total steps) from an EvalReport."""
+    total = report.total_steps
+    if total == 0:
+        return 0.0, 0.0, 0
+    err = sum(r.error_flag_accuracy * r.n_steps for r in report.per_file.values()) / total
+    return report.overall_fidelity, err, total
+
+
+def run_grid(
+    *,
+    suite_name: str,
+    files: list[str],
+    models: list[ModelSpec],
+    gepa_prompts: dict[str, str] | None,
+    base_prompt: str,
+    judge_provider: str,
+    judge_model: str,
+    judge_region: str | None,
+    train_split: float,
+    top_k: int,
+    seed: int,
+    sample_turns: str,
+    embed_dim: int,
+    val_frac: float | None = None,
+    max_holdout_traces: int | None = None,
+    target_max_tokens: int = DEFAULT_TARGET_MAX_TOKENS,
+    provider_factory=get_provider,  # noqa: ANN001 - injectable for tests (no network)
+) -> GridResult:
+    """Run every (model x condition) cell of the grid and return the rolled-up result.
+
+    `gepa_prompts` maps a `ModelSpec.label` to an evolved-prompt file path; a model absent from it
+    skips the `gepa`/`gepa_rag` conditions. The judge is built once (a pinned `RubricJudge`) and
+    shared across all cells.
+
+    Cells are scored on the reserved **test** band of the SAME 3-way `train/val/test` split GEPA
+    used to evolve its prompts (`val_frac` defaults to `(1 - train_split) / 2`, matching
+    `wmo build`), so a `+GEPA` cell is never scored on the `val` traces its prompt was selected on.
+    """
+    from pathlib import Path
+
+    from wmo.engine.build import split_traces, split_traces_3way
+    from wmo.ingest import get_adapter
+
+    if val_frac is None:
+        val_frac = (1.0 - train_split) / 2
+    # A 3-way split needs a strictly positive val band that still leaves a non-empty test band. When
+    # `train_split` leaves no room (e.g. train_split >= 1.0, or a --val-frac that overflows the
+    # [0,1) line) fall back to the plain 2-way split instead of crashing in `split_traces_3way`.
+    use_3way = val_frac > 0 and train_split + val_frac < 1
+    if not use_3way:
+        val_frac = 0.0
+    judge = _make_judge(judge_provider, judge_model, judge_region, provider_factory)
+    result = GridResult(
+        suite=suite_name,
+        judge_model=judge_model,
+        judge_provider=judge_provider,
+        train_split=train_split,
+        val_frac=val_frac,
+        top_k=top_k,
+        seed=seed,
+        sample_turns=sample_turns,
+        embed_dim=embed_dim,
+        max_holdout_traces=max_holdout_traces,
+    )
+    paths = [Path(f) for f in files]
+
+    # Held-out trace count (for reporting) - the reserved TEST band each cell scores, after any cap.
+    # Uses the same 3-way split as the scorer so the count matches what is actually evaluated.
+    adapter = get_adapter("otel-genai")
+    for path in paths:
+        traces = adapter.from_file(str(path))
+        if use_3way:
+            _, _, holdout = split_traces_3way(traces, train_split, val_frac)
+        else:
+            _, holdout = split_traces(traces, train_split)
+        holdout = holdout or traces
+        if max_holdout_traces is not None:
+            holdout = holdout[:max_holdout_traces]
+        result.total_test_traces += len(holdout)
+    for spec in models:
+        gepa_prompt_file = (gepa_prompts or {}).get(spec.label)
+        gepa_prompt: str | None = None
+        if gepa_prompt_file is not None:
+            text = Path(gepa_prompt_file).read_text(encoding="utf-8")
+            # A GEPA run that finds no improvement returns the base prompt verbatim. Scoring its
+            # `gepa`/`gepa_rag` cells would just re-run `base`/`base_rag` and report the judge/
+            # sampling noise between the two runs as a spurious "GEPA lift". Treat a base-identical
+            # evolved prompt as NO evolved prompt so the grid never presents that noise as a delta.
+            # Compare stripped so a prompt that differs only by trailing whitespace (a common
+            # artifact of hand-edited/exported prompt files) is still caught as a no-op.
+            gepa_prompt = text if text.strip() != base_prompt.strip() else None
+        for condition in CONDITIONS:
+            uses_gepa = condition in ("gepa", "gepa_rag")
+            if uses_gepa and gepa_prompt is None:
+                continue  # no (real) evolved prompt for this model -> skip its GEPA cells
+            prompt = gepa_prompt if uses_gepa else base_prompt
+            assert prompt is not None  # noqa: S101 - narrowed by the guard above
+            use_rag = condition in ("base_rag", "gepa_rag")
+            # Meter ONLY the target so cost is target-side, never judge cost. The target itself may
+            # be a region-fallback chain (Bedrock); MeteredProvider records whichever entry served.
+            tracker = RunTracker(run_id=uuid.uuid4().hex, kind="eval-grid")
+            capped = CappedProvider(_make_target(spec, provider_factory), target_max_tokens)
+            target: Provider = MeteredProvider(capped, tracker)
+            embedder = HashingEmbedder(dim=embed_dim) if use_rag else None
+            with tracker.timed():
+                report = evaluate_files(
+                    paths,
+                    prompt,
+                    target,
+                    judge,
+                    embedder=embedder,
+                    train_split=train_split,
+                    val_frac=val_frac,
+                    top_k=top_k,
+                    sample_turns=sample_turns,
+                    seed=seed,
+                    max_holdout_traces=max_holdout_traces,
+                )
+            fidelity, err, steps = _aggregate(report)
+            # Every cell scores the same held-out band with the same sampling, so `steps` is
+            # invariant across cells; take the max defensively so the reported count can never
+            # under-report if a future per-model option makes one cell score fewer steps.
+            result.total_test_steps = max(result.total_test_steps, steps)
+            result.cells.append(
+                GridCell(
+                    model_label=spec.label,
+                    provider=spec.provider,
+                    model=spec.model,
+                    condition=condition,
+                    condition_label=_CONDITION_LABELS[condition],
+                    fidelity=fidelity,
+                    error_flag_acc=err,
+                    n_steps=steps,
+                    cost_usd=_target_cost(spec.model, tracker),
+                )
+            )
+    return result