coder-eval 0.11.2__tar.gz → 0.11.4__tar.gz

{coder_eval-0.11.2 → coder_eval-0.11.4}/.github/CODEOWNERS

@@ -7,11 +7,11 @@
 # dedicated GitHub team (e.g. @UiPath/coder-eval-maintainers) once one exists.
 
 # Default owners for everything in the repo.
-*       @akshaylive @tmatup @uipreliga @bai-uipath
+*       @akshaylive @tmatup @uipreliga @bai-uipath @CarlesUIPath
 
 # Community-health, license, and security files warrant extra scrutiny.
-/LICENSE            @akshaylive @tmatup @uipreliga @bai-uipath
-/NOTICE             @akshaylive @tmatup @uipreliga @bai-uipath
-/SECURITY.md        @akshaylive @tmatup @uipreliga @bai-uipath
-/CODE_OF_CONDUCT.md @akshaylive @tmatup @uipreliga @bai-uipath
-/.github/           @akshaylive @tmatup @uipreliga @bai-uipath
+/LICENSE            @akshaylive @tmatup @uipreliga @bai-uipath @CarlesUIPath
+/NOTICE             @akshaylive @tmatup @uipreliga @bai-uipath @CarlesUIPath
+/SECURITY.md        @akshaylive @tmatup @uipreliga @bai-uipath @CarlesUIPath
+/CODE_OF_CONDUCT.md @akshaylive @tmatup @uipreliga @bai-uipath @CarlesUIPath
+/.github/           @akshaylive @tmatup @uipreliga @bai-uipath @CarlesUIPath

{coder_eval-0.11.2 → coder_eval-0.11.4}/.github/workflows/claude-pr-review.yml

@@ -118,7 +118,7 @@ jobs:
           # so this stacks on the tool allowlist below, it does not replace it.
           # Keep this list in sync with the global CODEOWNERS maintainers
           # (.github/CODEOWNERS `*` line).
-          include_comments_by_actor: "akshaylive,tmatup,uipreliga,bai-uipath"
+          include_comments_by_actor: "akshaylive,tmatup,uipreliga,bai-uipath,CarlesUIPath"
           # Drop all bot comments (dependabot, renovate, etc.). Exclusion wins
           # over the include list if an actor somehow matches both.
           exclude_comments_by_actor: "*[bot]"

{coder_eval-0.11.2 → coder_eval-0.11.4}/.github/workflows/pr-checks.yml

@@ -80,7 +80,7 @@ jobs:
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
 
       # PHASE 1: Fast checks (fail early)
       - name: Check code formatting (ruff format)
@@ -385,7 +385,7 @@ jobs:
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
 
       - name: Check code formatting (ruff format)
         run: .venv/Scripts/ruff format --check src/ tests/
@@ -843,8 +843,8 @@ jobs:
           python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
-      - name: Install project dependencies (with codex extra)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex
+      - name: Install project dependencies (with codex + litellm extras)
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
 
       - name: Verify required secrets are present
         run: |
@@ -854,14 +854,17 @@ jobs:
           fi
           echo "CODEX_API_KEY present."
 
-      - name: Run Codex live tests
+      - name: Run Codex + litellm-judge live tests
         run: |
           mkdir -p tmp
           # Run serially: `-n0` overrides the global `-n auto` (addopts).
           # Parallel xdist workers share ~/.codex and race the Codex SQLite
           # state migration (`duplicate column name: thread_id`); serial init
-          # migrates the fresh DB exactly once.
-          .venv/bin/pytest tests/test_codex_agent_live.py \
+          # migrates the fresh DB exactly once. test_litellm_judge_live.py
+          # reuses these same CODEX_* secrets to exercise
+          # checker_context.api_route.route: litellm end-to-end (PR #137
+          # review: "nothing in the repo exercises the feature").
+          .venv/bin/pytest tests/test_codex_agent_live.py tests/test_litellm_judge_live.py \
             -m live -n0 -v --tb=short --strict-markers -ra \
             --junit-xml=tmp/junit-codex-live.xml
 
@@ -878,7 +881,7 @@ jobs:
           passed = total - skipped - errors - failures
           print(f"codex-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
           if passed < 1:
-              sys.exit("test_codex_agent_live.py reported zero PASSED tests (missing API key / silent skip?)")
+              sys.exit("Live Codex/litellm-judge tests reported zero PASSED tests (missing API key / silent skip?)")
           PY
 
       - name: Upload Codex live-test artifacts on failure

{coder_eval-0.11.2 → coder_eval-0.11.4}/.github/workflows/release.yml

@@ -140,7 +140,14 @@ jobs:
           fi
 
       - name: Install build + release tools
-        run: uv tool install python-semantic-release && uv tool install twine
+        # Pinned: python-semantic-release declares gitpython~=3.0, so an
+        # unpinned install can resolve GitPython 3.1.60+, which removed
+        # git.Actor.name_email_regex and breaks semantic-release's `version`
+        # command outright ("type object 'Actor' has no attribute
+        # 'name_email_regex'"). Pin both explicitly until upstream adapts.
+        run: |
+          uv tool install python-semantic-release==10.6.1 --with gitpython==3.1.59
+          uv tool install twine
 
       - name: Run semantic-release (bump + tag, no push yet)
         id: release
@@ -157,7 +164,7 @@ jobs:
           # version to pyproject.toml + __init__.py, tags it, and regenerates the
           # changelog, but does not push yet (--no-push) so we can regenerate
           # uv.lock and amend before sending.
-          PSR="uv tool run --from python-semantic-release semantic-release"
+          PSR="uv tool run --from python-semantic-release==10.6.1 --with gitpython==3.1.59 semantic-release"
           $PSR version "--$BUMP" --no-vcs-release --no-push --changelog
           # A dispatch always cuts a release; report the just-published version.
           echo "version=$($PSR version --print)" >> "$GITHUB_OUTPUT"

{coder_eval-0.11.2 → coder_eval-0.11.4}/.gitignore

@@ -72,3 +72,6 @@ refs/
 
 # Derived pyright config for the CE036 contract engine (tests/lint/pyright_config.py)
 .pyright-tests.json
+
+# Local evalboard dataset for `next dev` (EVALBOARD_LOCAL_RUNS_DIR)
+evalboard/.local-runs/

{coder_eval-0.11.2 → coder_eval-0.11.4}/CHANGELOG.md

@@ -2,6 +2,105 @@
 
 <!-- version list -->
 
+## v0.11.4 (2026-08-27)
+
+### Chores
+
+- Add @CarlesUIPath as a code owner ([#140](https://github.com/UiPath/coder_eval/pull/140),
+  [`8fff865`](https://github.com/UiPath/coder_eval/commit/8fff8651589b2e43922321c689a66bb3aa41330d))
+
+- Sync PR-review comment allowlist with CODEOWNERS
+  ([#140](https://github.com/UiPath/coder_eval/pull/140),
+  [`8fff865`](https://github.com/UiPath/coder_eval/commit/8fff8651589b2e43922321c689a66bb3aa41330d))
+
+### Features
+
+- **docker**: Bake litellm extra into the docker image
+  ([#142](https://github.com/UiPath/coder_eval/pull/142),
+  [`3f4ae36`](https://github.com/UiPath/coder_eval/commit/3f4ae363b15024e2a033cf92ccda6befaef3a8b9))
+
+
+## v0.11.3 (2026-08-27)
+
+### Bug Fixes
+
+- **checker-context**: Typed model, reject litellm+agent_judge/sim, live test
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **eval-routing**: Restore DEFAULT_JUDGE_MODEL floor + add litellm judge transport
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **evalboard**: Exclude carried-forward passes from the wall-clock aggregates
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **release**: Pin python-semantic-release + GitPython to unbreak version bump
+  ([#139](https://github.com/UiPath/coder_eval/pull/139),
+  [`ef74014`](https://github.com/UiPath/coder_eval/commit/ef74014d3dd4b389f794e0cba743fc67ed430df8))
+
+- **routing**: Make _resolve_backend_route's match exhaustive
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+### Continuous Integration
+
+- Retrigger checks (GH Actions appeared stalled repo-wide)
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- Retrigger checks (previous push did not trigger CI)
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **fix**: Install litellm extra for pyright, address CodeQL findings
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+### Documentation
+
+- **timing**: Trim justification prose from the wall-clock comments
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+### Features
+
+- **eval-routing**: Decouple judge/agent_judge backend+model from the agent's own route
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **eval-routing**: Decouple judge/agent_judge backend+model from the agent's route
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **evalboard**: Chart seconds per passed task on the overview
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **evalboard**: Read the time ratio without hovering
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **evalboard**: Replace the turn-budget signal with time per passed task
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **evalboard**: Run the wall-clock signal beside the turn budget, behind tabs
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **litellm-judge**: Support arbitrary litellm kwargs via params/auth
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+### Refactoring
+
+- **litellm-judge**: Drop settings coupling, rename auth to env_params
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+
 ## v0.11.2 (2026-08-24)
 
 ### Bug Fixes

{coder_eval-0.11.2 → coder_eval-0.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coder-eval
-Version: 0.11.2
+Version: 0.11.4
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -58,6 +58,8 @@ Requires-Dist: pytest-mock>=3.15.1; extra == 'dev'
 Requires-Dist: pytest-xdist[psutil]>=3.0.0; extra == 'dev'
 Requires-Dist: pytest>=9.0.2; extra == 'dev'
 Requires-Dist: ruff>=0.15.7; extra == 'dev'
+Provides-Extra: litellm
+Requires-Dist: litellm<2.0.0,>=1.95.0; extra == 'litellm'
 Provides-Extra: uipath
 Requires-Dist: uipath>=2.10.31; extra == 'uipath'
 Description-Content-Type: text/markdown

{coder_eval-0.11.2 → coder_eval-0.11.4}/action.yml

@@ -41,7 +41,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.11.2"   # <-- kept in sync with releases by release.yml
+    default: "0.11.4"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

{coder_eval-0.11.2 → coder_eval-0.11.4}/docker/Dockerfile

@@ -48,12 +48,16 @@ COPY src/ ./src/
 # experiments/default.yaml is force-included by hatchling per pyproject.toml.
 COPY experiments/default.yaml ./experiments/default.yaml
 
-# Codex and Antigravity are always baked into the image -- peers to the
-# claude-code agent installed above -- so all built-in agents ship in every
-# build. `openai-codex` (+ its pinned cli-bin) and `google-antigravity` (which
-# bundles its `localharness` binary as a manylinux wheel) come from public PyPI,
-# so this needs no private-index credentials. The RUN below always passes
-# `--extra codex --extra antigravity`.
+# Codex, Antigravity, and litellm are always baked into the image -- codex/
+# antigravity are peers to the claude-code agent installed above, so all
+# built-in agents ship in every build; litellm backs the `checker_context.
+# api_route.route: litellm` judge dispatch (see pyproject.toml's `litellm`
+# extra), which every DockerRunner-isolated task needs available IN-container
+# since the checker runs there too, not just on the host. `openai-codex`
+# (+ its pinned cli-bin), `google-antigravity` (which bundles its
+# `localharness` binary as a manylinux wheel), and `litellm` all come from
+# public PyPI, so this needs no private-index credentials. The RUN below
+# always passes `--extra codex --extra antigravity --extra litellm`.
 #
 # CODER_EVAL_UV_EXTRAS carries ADDITIONAL opt-in extras on top of those; it
 # defaults to none. `make docker-image-full` passes `--extra uipath`, which
@@ -67,10 +71,10 @@ ARG CODER_EVAL_UV_EXTRAS=""
 # newer than the gate window); callers may override.
 ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,google-antigravity"
 
-# All extras (codex, antigravity, and the opt-in uipath) resolve from public
-# PyPI per uv.lock, so the build needs no private-index credentials.
+# All extras (codex, antigravity, litellm, and the opt-in uipath) resolve from
+# public PyPI per uv.lock, so the build needs no private-index credentials.
 RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS}" && \
-    uv export --frozen --extra codex --extra antigravity ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin
+    uv export --frozen --extra codex --extra antigravity --extra litellm ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin
 
 # Sanity check: the in-container entrypoint subcommand must be wired up.
 RUN coder-eval _run-task-internal --help > /dev/null

{coder_eval-0.11.2 → coder_eval-0.11.4}/docs/AB_EXPERIMENTS.md

@@ -130,6 +130,7 @@ From `ExperimentVariant` (`coder_eval/models/experiment.py`):
 | `initial_prompt_file` | str                | Prompt replacement loaded from a file                                                  |
 | `run_limits`          | block              | Per-key cap overrides (`max_turns`, `task_timeout`, token/USD budgets)                 |
 | `driver`              | `tempdir`/`docker` | Sandbox driver — enables tempdir-vs-docker arms                                        |
+| `checker_context`     | dict               | Backend/model override for the evaluation side (judge, simulator) — see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context); **not** currently `-D`-reachable |
 
 The `agent` dict is the lever for most A/B tests. Anything on `AgentConfig` is
 fair game: `model`, `permission_mode`, `allowed_tools`, `disallowed_tools`,

{coder_eval-0.11.2 → coder_eval-0.11.4}/docs/DIALOG_MODE.md

@@ -60,9 +60,11 @@ The mechanics:
 - The simulator is a **tools-disabled Claude Code agent** with `allowed_tools: []`, an explicit
   deny-list, and no plugins or settings sources. It is pure text-in / text-out, and it **cannot see
   the sandbox** — no files, no terminal, no agent reasoning. Only what the agent writes in the chat.
-- The simulator shares the coding agent's resolved `ApiRoute`, so backend and model come from the
-  run's routing (`--backend direct` / `--backend bedrock`) rather than from the `simulation:` block.
-  There is no model field here to set.
+- The simulator runs on the run's resolved *evaluation* `ApiRoute` — the coding agent's own route
+  (`--backend direct` / `--backend bedrock`) unless `checker_context.api_route.route` overrides it
+  (see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context)). The **model** is separately
+  pinned by `simulation.model` (see [Simulation](TASK_DEFINITION_GUIDE.md#simulation)), not inherited
+  from the route — so an A/B varying the subject model doesn't silently vary the simulated user too.
 
 **Agent-kind constraint.** The *subject* agent can be any registered kind — the dialog driver only
 calls the agent's `communicate()`, so Codex and plugin agents work. The *simulator*, however, is

{coder_eval-0.11.2 → coder_eval-0.11.4}/docs/TASK_DEFINITION_GUIDE.md

@@ -35,6 +35,7 @@ Complete reference for defining evaluation tasks in Coder Eval.
   - [llm_judge](#llm_judge)
   - [agent_judge](#agent_judge)
   - [skill_triggered](#skill_triggered)
+- [Checker Context](#checker-context)
 - [Reference Solutions](#reference-solutions)
 - [Pre-Run Commands](#pre-run-commands)
 - [Post-Run Commands](#post-run-commands)
@@ -63,6 +64,7 @@ reference: { ... }                    # Optional reference solution (a directory
 pre_run: [ ... ]                      # Optional pre-run commands (before agent starts)
 post_run: [ ... ]                     # Optional post-run commands
 dataset: { ... }                      # Optional dataset fan-out (one task -> N row-tasks)
+checker_context: { ... }              # Optional: backend/model for the evaluation side (judge, simulator)
 ```
 
 ### `dataset`
@@ -1295,6 +1297,40 @@ Observed label is `"yes"` when either signal is found, else `"no"`. Expected lab
 
 **Typical pattern.** Label each dataset row with its true skill (`expected_skill`, `""` for negatives) and stack one `skill_triggered` criterion per skill against the same dataset — each gets its own confusion matrix from the same agent traces. This is the natural companion to a skill A/B experiment (skill plugin on vs. off); see the [A/B Experiment Guide](AB_EXPERIMENTS.md#recipe-ab-a-skill).
 
+## Checker Context
+
+`checker_context` carries task-authored config for the success-checking side, namespaced by reserved key. Currently the only recognized namespace is **`api_route`**:
+
+```yaml
+success_criteria:
+  - type: llm_judge
+    prompt: "Grade the fix for correctness."
+
+checker_context:
+  api_route:
+    route: bedrock    # which backend the whole eval side (llm_judge/agent_judge/simulator) uses
+    model: claude-haiku-4-5   # model override for that route
+```
+
+- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object. For `direct`/`bedrock` credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. Unlike the other two backends, `route: litellm` has NO implicit env-var fallback — see `params`/`env_params` below, which is how it's configured. `model` is required for `route: litellm` (there is no default open-weight/gateway model).
+- `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason.
+- `params`/`env_params` (**`route: litellm` only**) are how the call is actually configured — there is no fallback to the agent's own `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` env vars, since a gateway-routed judge model rarely reuses the agent's own LiteLLM proxy/credential. They also cover any of the dozens of other provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`:
+  ```yaml
+  checker_context:
+    api_route:
+      route: litellm
+      model: azure/gpt-5.6-luna
+      params:                       # arbitrary literal passthrough kwargs to litellm.acompletion
+        api_version: "2024-05-01"
+      env_params:                   # param name -> ENV VAR NAME (never the secret itself)
+        api_base: LITELLM_BASE_URL
+        api_key: LITELLM_AUTH_TOKEN
+  ```
+  `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `env_params` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's config, including secrets (IAM keys, an Azure AD token, a service-account path, ...), is representable without a dedicated field per provider. `env_params` is resolved after `params`, so it always wins for the same key. Rejected at task-load time if given without `route: litellm`. For an Azure deployment, pin `api_version` via `params` to whatever API version the agent side is actually configured for (e.g. Codex's `CODEX_API_VERSION`) — the judge has no way to inherit it, and a mismatched version can hit a different shape of the same endpoint.
+  **`route: litellm` is `llm_judge`-only** — `agent_judge` and the simulator run as real Claude Code CLI subprocesses that speak the Anthropic Messages protocol, so pointing them at an arbitrary litellm-fronted gateway (which may speak an entirely different wire protocol) isn't representable. The orchestrator rejects the combination at resolution time (a clear error, not a silent misroute) if the task has an enabled `agent_judge` criterion or `simulation.enabled: true` alongside `route: litellm` — use `route: bedrock`/`direct` for those instead.
+
+`checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML.
+
 ## Reference Solutions
 
 A reference solution is always a **directory**, given relative to the task YAML's own directory:
@@ -1577,7 +1613,7 @@ simulation:
 | `check_criteria` | `end_of_dialog` | `end_of_dialog`, `every_turn`, or `both`. |
 | `model` | `anthropic.claude-sonnet-4-6` | Model that plays the simulated user. Auto-translated to the run's backend (Bedrock inference profile / bare Anthropic alias), the same way [`llm_judge`](#llm_judge)'s `model` is. |
 
-The simulator runs as a tools-disabled Claude Code agent sharing the coding agent's `ApiRoute`, so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured.
+The simulator runs as a tools-disabled Claude Code agent on the run's resolved *evaluation* `ApiRoute` (the coding agent's own route unless [`checker_context.api_route.route`](#checker-context) overrides it), so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent by default) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured.
 
 **Semantics:**
 

coder_eval-0.11.4/evalboard/app/_components/col-help.tsx

@@ -0,0 +1,10 @@
+// Column definitions, surfaced as native `title` tooltips on table headers.
+// Shared by the run-page task grid and the task-page message timeline so the two
+// describe the same number the same way. Wording is page-neutral (no "this task"
+// / "this call") because both surfaces read the same string.
+export const TOKEN_COLUMN_HELP = {
+    input: "Input tokens (uncached): fresh prompt input billed at the full input rate — the slice that was neither written to nor read from the prompt cache.",
+    output: "Output tokens: text, code, tool arguments and reasoning the model generated.",
+    cw: "Cache-write tokens: context newly written into the prompt cache (cache_creation_input_tokens).",
+    cr: "Cache-read tokens: cached input re-billed on every later call (cache_read_input_tokens). Usually the dominant cost line.",
+} satisfies Record<string, string>;

coder_eval-0.11.4/evalboard/app/_overview/__tests__/efficiency-charts.test.tsx

@@ -0,0 +1,80 @@
+import { describe, expect, test } from "vitest";
+import { fireEvent, render, screen } from "@testing-library/react";
+import type { RunPoint } from "@/lib/overview";
+import { EfficiencyCharts } from "../efficiency-charts";
+
+function point(overrides: Partial<RunPoint> = {}): RunPoint {
+    return {
+        runId: "2026-08-18_04-51-58",
+        timestamp: Date.UTC(2026, 7, 18),
+        harness: "codex",
+        successRate: 96,
+        turnBudgetRate: 87,
+        withinExpectedTimeRate: 78,
+        timePerPassedTask: 192,
+        ...overrides,
+    };
+}
+
+function renderCharts(scoped = false) {
+    return render(
+        <EfficiencyCharts
+            data={[point()]}
+            harnesses={["codex"]}
+            windowStart={Date.UTC(2026, 7, 1)}
+            windowEnd={Date.UTC(2026, 7, 31)}
+            scoped={scoped}
+        />,
+    );
+}
+
+describe("EfficiencyCharts", () => {
+    test("opens on the wall-clock metric", () => {
+        renderCharts();
+        expect(screen.getByRole("heading")).toHaveTextContent(
+            "Time per Passed Task",
+        );
+        expect(
+            screen.getByRole("tab", { name: "Time" }),
+        ).toHaveAttribute("aria-selected", "true");
+    });
+
+    test("the Turns tab still reaches the retired turn-budget chart", () => {
+        // The turn budget is kept visible while the derived expected-time line
+        // is being watched; retiring it is deleting this tab.
+        renderCharts();
+        fireEvent.click(screen.getByRole("tab", { name: "Turns" }));
+        expect(screen.getByRole("heading")).toHaveTextContent(
+            "Within Expected Turns (%)",
+        );
+        expect(screen.getByText(/1.5× their expected turns/)).toBeInTheDocument();
+        expect(
+            screen.getByRole("tab", { name: "Turns" }),
+        ).toHaveAttribute("aria-selected", "true");
+        expect(screen.getByRole("tab", { name: "Time" })).toHaveAttribute(
+            "aria-selected",
+            "false",
+        );
+    });
+
+    test("switching back restores the wall-clock blurb", () => {
+        renderCharts();
+        fireEvent.click(screen.getByRole("tab", { name: "Turns" }));
+        fireEvent.click(screen.getByRole("tab", { name: "Time" }));
+        expect(
+            screen.getByText(/the number that passed/),
+        ).toBeInTheDocument();
+    });
+
+    test("says so when the numbers are filter-scoped", () => {
+        renderCharts(true);
+        expect(
+            screen.getByText(/scoped to the active filter/),
+        ).toBeInTheDocument();
+    });
+
+    test("no filter note when the window is unscoped", () => {
+        renderCharts();
+        expect(screen.queryByText(/scoped to the active filter/)).toBeNull();
+    });
+});

{coder_eval-0.11.2 → coder_eval-0.11.4}/evalboard/app/_overview/__tests__/harness-legend.test.tsx

@@ -37,6 +37,44 @@ describe("HarnessLegend", () => {
 describe("HarnessTooltip", () => {
     const s = series("claude-code", "codex");
 
+    test("formats the value with the chart's own formatter", () => {
+        render(
+            <HarnessTooltip
+                active
+                label={1_700_000_000_000}
+                series={s}
+                suffix="per passed task"
+                emptyText="no passing tasks"
+                format={(v) => `${Math.round(v)}s`}
+                payload={[{ dataKey: s[1].dataKey, value: 192.3 }]}
+            />,
+        );
+        expect(screen.getByText("192s per passed task")).toBeInTheDocument();
+    });
+
+    test("adds the secondary line for the hovered point only", () => {
+        // The within-expected rate belongs to one run, so it rides the hover on
+        // that run's point instead of being summarized over the whole chart.
+        render(
+            <HarnessTooltip
+                active
+                label={1_700_000_000_000}
+                series={s}
+                suffix="per passed task"
+                emptyText="no passing tasks"
+                secondary={(harness) =>
+                    harness === "codex" ? "78% within 2× expected" : null
+                }
+                payload={[
+                    { dataKey: s[0].dataKey, value: 300 },
+                    { dataKey: s[1].dataKey, value: 192 },
+                ]}
+            />,
+        );
+        expect(screen.getByText("78% within 2× expected")).toBeInTheDocument();
+        expect(screen.queryAllByText(/within/)).toHaveLength(1);
+    });
+
     test("shows only the harnesses that actually ran at the hovered x", () => {
         // Recharts hands over every series, including the ones with no value
         // here. Listing those would invent runs that never happened.

{coder_eval-0.11.2 → coder_eval-0.11.4}/evalboard/app/_overview/__tests__/harness-series.test.ts

@@ -12,14 +12,16 @@ function point(
     harness: string,
     timestamp: number,
     successRate: number | null,
-    turnBudgetRate: number | null = null,
+    withinExpectedTimeRate: number | null = null,
 ): RunPoint {
     return {
         runId: `run-${timestamp}`,
         timestamp,
         harness,
         successRate,
-        turnBudgetRate,
+        turnBudgetRate: null,
+        withinExpectedTimeRate,
+        timePerPassedTask: null,
     };
 }
 
@@ -99,7 +101,7 @@ describe("pivotByHarness", () => {
             timestamp: 100,
             [key]: 90,
         });
-        expect(pivotByHarness(pts, ["codex"], "turnBudgetRate").rows[0]).toEqual(
+        expect(pivotByHarness(pts, ["codex"], "withinExpectedTimeRate").rows[0]).toEqual(
             { timestamp: 100, [key]: 40 },
         );
     });
@@ -111,7 +113,7 @@ describe("pivotByHarness", () => {
         const { rows } = pivotByHarness(
             [point("codex", 100, 90, null)],
             ["codex"],
-            "turnBudgetRate",
+            "withinExpectedTimeRate",
         );
         expect(rows).toEqual([]);
     });