coder-eval 0.11.2__tar.gz → 0.11.3__tar.gz

{coder_eval-0.11.2 → coder_eval-0.11.3}/.github/workflows/pr-checks.yml

@@ -80,7 +80,7 @@ jobs:
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
 
       # PHASE 1: Fast checks (fail early)
       - name: Check code formatting (ruff format)
@@ -385,7 +385,7 @@ jobs:
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
 
       - name: Check code formatting (ruff format)
         run: .venv/Scripts/ruff format --check src/ tests/
@@ -843,8 +843,8 @@ jobs:
           python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
-      - name: Install project dependencies (with codex extra)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex
+      - name: Install project dependencies (with codex + litellm extras)
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
 
       - name: Verify required secrets are present
         run: |
@@ -854,14 +854,17 @@ jobs:
           fi
           echo "CODEX_API_KEY present."
 
-      - name: Run Codex live tests
+      - name: Run Codex + litellm-judge live tests
         run: |
           mkdir -p tmp
           # Run serially: `-n0` overrides the global `-n auto` (addopts).
           # Parallel xdist workers share ~/.codex and race the Codex SQLite
           # state migration (`duplicate column name: thread_id`); serial init
-          # migrates the fresh DB exactly once.
-          .venv/bin/pytest tests/test_codex_agent_live.py \
+          # migrates the fresh DB exactly once. test_litellm_judge_live.py
+          # reuses these same CODEX_* secrets to exercise
+          # checker_context.api_route.route: litellm end-to-end (PR #137
+          # review: "nothing in the repo exercises the feature").
+          .venv/bin/pytest tests/test_codex_agent_live.py tests/test_litellm_judge_live.py \
             -m live -n0 -v --tb=short --strict-markers -ra \
             --junit-xml=tmp/junit-codex-live.xml
 
@@ -878,7 +881,7 @@ jobs:
           passed = total - skipped - errors - failures
           print(f"codex-live passed={passed} skipped={skipped} errors={errors} failures={failures}")
           if passed < 1:
-              sys.exit("test_codex_agent_live.py reported zero PASSED tests (missing API key / silent skip?)")
+              sys.exit("Live Codex/litellm-judge tests reported zero PASSED tests (missing API key / silent skip?)")
           PY
 
       - name: Upload Codex live-test artifacts on failure

{coder_eval-0.11.2 → coder_eval-0.11.3}/.github/workflows/release.yml

@@ -140,7 +140,14 @@ jobs:
           fi
 
       - name: Install build + release tools
-        run: uv tool install python-semantic-release && uv tool install twine
+        # Pinned: python-semantic-release declares gitpython~=3.0, so an
+        # unpinned install can resolve GitPython 3.1.60+, which removed
+        # git.Actor.name_email_regex and breaks semantic-release's `version`
+        # command outright ("type object 'Actor' has no attribute
+        # 'name_email_regex'"). Pin both explicitly until upstream adapts.
+        run: |
+          uv tool install python-semantic-release==10.6.1 --with gitpython==3.1.59
+          uv tool install twine
 
       - name: Run semantic-release (bump + tag, no push yet)
         id: release
@@ -157,7 +164,7 @@ jobs:
           # version to pyproject.toml + __init__.py, tags it, and regenerates the
           # changelog, but does not push yet (--no-push) so we can regenerate
           # uv.lock and amend before sending.
-          PSR="uv tool run --from python-semantic-release semantic-release"
+          PSR="uv tool run --from python-semantic-release==10.6.1 --with gitpython==3.1.59 semantic-release"
           $PSR version "--$BUMP" --no-vcs-release --no-push --changelog
           # A dispatch always cuts a release; report the just-published version.
           echo "version=$($PSR version --print)" >> "$GITHUB_OUTPUT"

{coder_eval-0.11.2 → coder_eval-0.11.3}/.gitignore

@@ -72,3 +72,6 @@ refs/
 
 # Derived pyright config for the CE036 contract engine (tests/lint/pyright_config.py)
 .pyright-tests.json
+
+# Local evalboard dataset for `next dev` (EVALBOARD_LOCAL_RUNS_DIR)
+evalboard/.local-runs/

{coder_eval-0.11.2 → coder_eval-0.11.3}/CHANGELOG.md

@@ -2,6 +2,87 @@
 
 <!-- version list -->
 
+## v0.11.3 (2026-08-27)
+
+### Bug Fixes
+
+- **checker-context**: Typed model, reject litellm+agent_judge/sim, live test
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **eval-routing**: Restore DEFAULT_JUDGE_MODEL floor + add litellm judge transport
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **evalboard**: Exclude carried-forward passes from the wall-clock aggregates
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **release**: Pin python-semantic-release + GitPython to unbreak version bump
+  ([#139](https://github.com/UiPath/coder_eval/pull/139),
+  [`ef74014`](https://github.com/UiPath/coder_eval/commit/ef74014d3dd4b389f794e0cba743fc67ed430df8))
+
+- **routing**: Make _resolve_backend_route's match exhaustive
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+### Continuous Integration
+
+- Retrigger checks (GH Actions appeared stalled repo-wide)
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- Retrigger checks (previous push did not trigger CI)
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **fix**: Install litellm extra for pyright, address CodeQL findings
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+### Documentation
+
+- **timing**: Trim justification prose from the wall-clock comments
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+### Features
+
+- **eval-routing**: Decouple judge/agent_judge backend+model from the agent's own route
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **eval-routing**: Decouple judge/agent_judge backend+model from the agent's route
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+- **evalboard**: Chart seconds per passed task on the overview
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **evalboard**: Read the time ratio without hovering
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **evalboard**: Replace the turn-budget signal with time per passed task
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **evalboard**: Run the wall-clock signal beside the turn budget, behind tabs
+  ([#125](https://github.com/UiPath/coder_eval/pull/125),
+  [`dd918e6`](https://github.com/UiPath/coder_eval/commit/dd918e68a454f39c8254f74fe2e4f346cff3a16f))
+
+- **litellm-judge**: Support arbitrary litellm kwargs via params/auth
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+### Refactoring
+
+- **litellm-judge**: Drop settings coupling, rename auth to env_params
+  ([#137](https://github.com/UiPath/coder_eval/pull/137),
+  [`727bb7b`](https://github.com/UiPath/coder_eval/commit/727bb7bee459b43c51af93067907687c5e04cfc8))
+
+
 ## v0.11.2 (2026-08-24)
 
 ### Bug Fixes

{coder_eval-0.11.2 → coder_eval-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coder-eval
-Version: 0.11.2
+Version: 0.11.3
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -58,6 +58,8 @@ Requires-Dist: pytest-mock>=3.15.1; extra == 'dev'
 Requires-Dist: pytest-xdist[psutil]>=3.0.0; extra == 'dev'
 Requires-Dist: pytest>=9.0.2; extra == 'dev'
 Requires-Dist: ruff>=0.15.7; extra == 'dev'
+Provides-Extra: litellm
+Requires-Dist: litellm<2.0.0,>=1.95.0; extra == 'litellm'
 Provides-Extra: uipath
 Requires-Dist: uipath>=2.10.31; extra == 'uipath'
 Description-Content-Type: text/markdown

{coder_eval-0.11.2 → coder_eval-0.11.3}/action.yml

@@ -41,7 +41,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.11.2"   # <-- kept in sync with releases by release.yml
+    default: "0.11.3"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

{coder_eval-0.11.2 → coder_eval-0.11.3}/docs/AB_EXPERIMENTS.md

@@ -130,6 +130,7 @@ From `ExperimentVariant` (`coder_eval/models/experiment.py`):
 | `initial_prompt_file` | str                | Prompt replacement loaded from a file                                                  |
 | `run_limits`          | block              | Per-key cap overrides (`max_turns`, `task_timeout`, token/USD budgets)                 |
 | `driver`              | `tempdir`/`docker` | Sandbox driver — enables tempdir-vs-docker arms                                        |
+| `checker_context`     | dict               | Backend/model override for the evaluation side (judge, simulator) — see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context); **not** currently `-D`-reachable |
 
 The `agent` dict is the lever for most A/B tests. Anything on `AgentConfig` is
 fair game: `model`, `permission_mode`, `allowed_tools`, `disallowed_tools`,

{coder_eval-0.11.2 → coder_eval-0.11.3}/docs/DIALOG_MODE.md

@@ -60,9 +60,11 @@ The mechanics:
 - The simulator is a **tools-disabled Claude Code agent** with `allowed_tools: []`, an explicit
   deny-list, and no plugins or settings sources. It is pure text-in / text-out, and it **cannot see
   the sandbox** — no files, no terminal, no agent reasoning. Only what the agent writes in the chat.
-- The simulator shares the coding agent's resolved `ApiRoute`, so backend and model come from the
-  run's routing (`--backend direct` / `--backend bedrock`) rather than from the `simulation:` block.
-  There is no model field here to set.
+- The simulator runs on the run's resolved *evaluation* `ApiRoute` — the coding agent's own route
+  (`--backend direct` / `--backend bedrock`) unless `checker_context.api_route.route` overrides it
+  (see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context)). The **model** is separately
+  pinned by `simulation.model` (see [Simulation](TASK_DEFINITION_GUIDE.md#simulation)), not inherited
+  from the route — so an A/B varying the subject model doesn't silently vary the simulated user too.
 
 **Agent-kind constraint.** The *subject* agent can be any registered kind — the dialog driver only
 calls the agent's `communicate()`, so Codex and plugin agents work. The *simulator*, however, is

{coder_eval-0.11.2 → coder_eval-0.11.3}/docs/TASK_DEFINITION_GUIDE.md

@@ -35,6 +35,7 @@ Complete reference for defining evaluation tasks in Coder Eval.
   - [llm_judge](#llm_judge)
   - [agent_judge](#agent_judge)
   - [skill_triggered](#skill_triggered)
+- [Checker Context](#checker-context)
 - [Reference Solutions](#reference-solutions)
 - [Pre-Run Commands](#pre-run-commands)
 - [Post-Run Commands](#post-run-commands)
@@ -63,6 +64,7 @@ reference: { ... }                    # Optional reference solution (a directory
 pre_run: [ ... ]                      # Optional pre-run commands (before agent starts)
 post_run: [ ... ]                     # Optional post-run commands
 dataset: { ... }                      # Optional dataset fan-out (one task -> N row-tasks)
+checker_context: { ... }              # Optional: backend/model for the evaluation side (judge, simulator)
 ```
 
 ### `dataset`
@@ -1295,6 +1297,40 @@ Observed label is `"yes"` when either signal is found, else `"no"`. Expected lab
 
 **Typical pattern.** Label each dataset row with its true skill (`expected_skill`, `""` for negatives) and stack one `skill_triggered` criterion per skill against the same dataset — each gets its own confusion matrix from the same agent traces. This is the natural companion to a skill A/B experiment (skill plugin on vs. off); see the [A/B Experiment Guide](AB_EXPERIMENTS.md#recipe-ab-a-skill).
 
+## Checker Context
+
+`checker_context` carries task-authored config for the success-checking side, namespaced by reserved key. Currently the only recognized namespace is **`api_route`**:
+
+```yaml
+success_criteria:
+  - type: llm_judge
+    prompt: "Grade the fix for correctness."
+
+checker_context:
+  api_route:
+    route: bedrock    # which backend the whole eval side (llm_judge/agent_judge/simulator) uses
+    model: claude-haiku-4-5   # model override for that route
+```
+
+- `route` selects which backend the WHOLE evaluation side calls (`llm_judge`, `agent_judge`, and the simulator all share one resolved eval route per run — this isn't per-criterion), **decoupled from the agent's own route** (a Claude agent can be graded by a differently-backed judge, or vice versa). This is a backend *name* (`direct` / `bedrock` / `litellm`), not a route object. For `direct`/`bedrock` credentials are never read from the task, always from the matching environment variables (`ANTHROPIC_API_KEY` for `direct`, `AWS_BEARER_TOKEN_BEDROCK`/`AWS_REGION` for `bedrock`). An unconfigured or unknown backend name raises at dispatch rather than silently falling back. **`route: litellm` dispatches `llm_judge` through the `litellm` library** (the `coder-eval[litellm]` extra, `litellm.acompletion`) rather than assuming one wire protocol — a gateway-routed judge model (e.g. an Azure AI `/openai/v1` deployment) rarely speaks Anthropic Messages, so this lets `model` carry its own provider hint (e.g. `azure/gpt-5.6-luna`) and get that provider's actual request/response shape handled by the library. Unlike the other two backends, `route: litellm` has NO implicit env-var fallback — see `params`/`env_params` below, which is how it's configured. `model` is required for `route: litellm` (there is no default open-weight/gateway model).
+- `model` overrides the model that resolved route uses for **`llm_judge` only** — when the criterion itself leaves `model:` unset (precedence: an explicit per-criterion `model:` always wins; below that, `checker_context.api_route.model`; below that, the built-in `DEFAULT_JUDGE_MODEL`). This floor is deliberate and never the agent's own model — an unpinned judge must grade identically regardless of which model the agent under test is using, so `resolve_evaluation_route` never lets the agent's env-configured model (e.g. `BEDROCK_MODEL`) leak into `route.model` on its own; `route.model` is set only when this override was actually given. This works because every `ApiRoute` (`DirectRoute`/`BedrockRoute`/`LiteLLMRoute`) carries its own `model` field; the orchestrator bakes the override into the resolved route's `model` before any criterion runs, so `llm_judge` just reads `context.route.model` — it never reads `checker_context` directly. **`agent_judge` and the simulator do not honor this override** — `agent_judge`'s sub-agent model comes from the criterion's own `agent:` block (defaulted to a fixed judge model), and the simulator's model is pinned by `SimulationConfig.model` (see [Simulation](#simulation) below) — both independent of `checker_context.api_route.model` by design, for the same "measuring instrument stays fixed" reason.
+- `params`/`env_params` (**`route: litellm` only**) are how the call is actually configured — there is no fallback to the agent's own `LITELLM_BASE_URL`/`LITELLM_AUTH_TOKEN` env vars, since a gateway-routed judge model rarely reuses the agent's own LiteLLM proxy/credential. They also cover any of the dozens of other provider-specific kwargs `litellm.acompletion` accepts (`aws_access_key_id`, `vertex_project`, `api_version`, ...), which have no dedicated field on `LiteLLMRoute`:
+  ```yaml
+  checker_context:
+    api_route:
+      route: litellm
+      model: azure/gpt-5.6-luna
+      params:                       # arbitrary literal passthrough kwargs to litellm.acompletion
+        api_version: "2024-05-01"
+      env_params:                   # param name -> ENV VAR NAME (never the secret itself)
+        api_base: LITELLM_BASE_URL
+        api_key: LITELLM_AUTH_TOKEN
+  ```
+  `params` is merged straight into the `litellm.acompletion(**kwargs)` call — litellm validates the param names itself, so there's no allowlist to keep in sync here. `env_params` maps a kwarg name to the *name* of an environment variable; the value is resolved right before the call, so no secret is ever written into task/experiment YAML — this is how an arbitrary provider's config, including secrets (IAM keys, an Azure AD token, a service-account path, ...), is representable without a dedicated field per provider. `env_params` is resolved after `params`, so it always wins for the same key. Rejected at task-load time if given without `route: litellm`. For an Azure deployment, pin `api_version` via `params` to whatever API version the agent side is actually configured for (e.g. Codex's `CODEX_API_VERSION`) — the judge has no way to inherit it, and a mismatched version can hit a different shape of the same endpoint.
+  **`route: litellm` is `llm_judge`-only** — `agent_judge` and the simulator run as real Claude Code CLI subprocesses that speak the Anthropic Messages protocol, so pointing them at an arbitrary litellm-fronted gateway (which may speak an entirely different wire protocol) isn't representable. The orchestrator rejects the combination at resolution time (a clear error, not a silent misroute) if the task has an enabled `agent_judge` criterion or `simulation.enabled: true` alongside `route: litellm` — use `route: bedrock`/`direct` for those instead.
+
+`checker_context` merges shallow-per-namespace across `default_experiment.defaults.checker_context` → `experiment.defaults.checker_context` → `task.checker_context` → `variant.checker_context` (same 4-layer precedence as `agent`/`simulation`). So a judge-model A/B, or a judge-backend A/B, is a variant-level config change, not an edit to every task YAML.
+
 ## Reference Solutions
 
 A reference solution is always a **directory**, given relative to the task YAML's own directory:
@@ -1577,7 +1613,7 @@ simulation:
 | `check_criteria` | `end_of_dialog` | `end_of_dialog`, `every_turn`, or `both`. |
 | `model` | `anthropic.claude-sonnet-4-6` | Model that plays the simulated user. Auto-translated to the run's backend (Bedrock inference profile / bare Anthropic alias), the same way [`llm_judge`](#llm_judge)'s `model` is. |
 
-The simulator runs as a tools-disabled Claude Code agent sharing the coding agent's `ApiRoute`, so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured.
+The simulator runs as a tools-disabled Claude Code agent on the run's resolved *evaluation* `ApiRoute` (the coding agent's own route unless [`checker_context.api_route.route`](#checker-context) overrides it), so temperature and sampling are resolved at the route level (same `-b` flag as the coding agent by default) and are not configured on this block. The **model is not**: it is pinned by `model` above. Inheriting it from the route meant `BEDROCK_MODEL` decided who the simulated user was, so an A/B varying the subject model silently varied its interlocutor too. Hold `model` fixed across variants for the same reason you hold a judge model fixed — the simulator is part of the measuring instrument, not the thing being measured.
 
 **Semantics:**
 

coder_eval-0.11.3/evalboard/app/_components/col-help.tsx

@@ -0,0 +1,10 @@
+// Column definitions, surfaced as native `title` tooltips on table headers.
+// Shared by the run-page task grid and the task-page message timeline so the two
+// describe the same number the same way. Wording is page-neutral (no "this task"
+// / "this call") because both surfaces read the same string.
+export const TOKEN_COLUMN_HELP = {
+    input: "Input tokens (uncached): fresh prompt input billed at the full input rate — the slice that was neither written to nor read from the prompt cache.",
+    output: "Output tokens: text, code, tool arguments and reasoning the model generated.",
+    cw: "Cache-write tokens: context newly written into the prompt cache (cache_creation_input_tokens).",
+    cr: "Cache-read tokens: cached input re-billed on every later call (cache_read_input_tokens). Usually the dominant cost line.",
+} satisfies Record<string, string>;

coder_eval-0.11.3/evalboard/app/_overview/__tests__/efficiency-charts.test.tsx

@@ -0,0 +1,80 @@
+import { describe, expect, test } from "vitest";
+import { fireEvent, render, screen } from "@testing-library/react";
+import type { RunPoint } from "@/lib/overview";
+import { EfficiencyCharts } from "../efficiency-charts";
+
+function point(overrides: Partial<RunPoint> = {}): RunPoint {
+    return {
+        runId: "2026-08-18_04-51-58",
+        timestamp: Date.UTC(2026, 7, 18),
+        harness: "codex",
+        successRate: 96,
+        turnBudgetRate: 87,
+        withinExpectedTimeRate: 78,
+        timePerPassedTask: 192,
+        ...overrides,
+    };
+}
+
+function renderCharts(scoped = false) {
+    return render(
+        <EfficiencyCharts
+            data={[point()]}
+            harnesses={["codex"]}
+            windowStart={Date.UTC(2026, 7, 1)}
+            windowEnd={Date.UTC(2026, 7, 31)}
+            scoped={scoped}
+        />,
+    );
+}
+
+describe("EfficiencyCharts", () => {
+    test("opens on the wall-clock metric", () => {
+        renderCharts();
+        expect(screen.getByRole("heading")).toHaveTextContent(
+            "Time per Passed Task",
+        );
+        expect(
+            screen.getByRole("tab", { name: "Time" }),
+        ).toHaveAttribute("aria-selected", "true");
+    });
+
+    test("the Turns tab still reaches the retired turn-budget chart", () => {
+        // The turn budget is kept visible while the derived expected-time line
+        // is being watched; retiring it is deleting this tab.
+        renderCharts();
+        fireEvent.click(screen.getByRole("tab", { name: "Turns" }));
+        expect(screen.getByRole("heading")).toHaveTextContent(
+            "Within Expected Turns (%)",
+        );
+        expect(screen.getByText(/1.5× their expected turns/)).toBeInTheDocument();
+        expect(
+            screen.getByRole("tab", { name: "Turns" }),
+        ).toHaveAttribute("aria-selected", "true");
+        expect(screen.getByRole("tab", { name: "Time" })).toHaveAttribute(
+            "aria-selected",
+            "false",
+        );
+    });
+
+    test("switching back restores the wall-clock blurb", () => {
+        renderCharts();
+        fireEvent.click(screen.getByRole("tab", { name: "Turns" }));
+        fireEvent.click(screen.getByRole("tab", { name: "Time" }));
+        expect(
+            screen.getByText(/the number that passed/),
+        ).toBeInTheDocument();
+    });
+
+    test("says so when the numbers are filter-scoped", () => {
+        renderCharts(true);
+        expect(
+            screen.getByText(/scoped to the active filter/),
+        ).toBeInTheDocument();
+    });
+
+    test("no filter note when the window is unscoped", () => {
+        renderCharts();
+        expect(screen.queryByText(/scoped to the active filter/)).toBeNull();
+    });
+});

{coder_eval-0.11.2 → coder_eval-0.11.3}/evalboard/app/_overview/__tests__/harness-legend.test.tsx

@@ -37,6 +37,44 @@ describe("HarnessLegend", () => {
 describe("HarnessTooltip", () => {
     const s = series("claude-code", "codex");
 
+    test("formats the value with the chart's own formatter", () => {
+        render(
+            <HarnessTooltip
+                active
+                label={1_700_000_000_000}
+                series={s}
+                suffix="per passed task"
+                emptyText="no passing tasks"
+                format={(v) => `${Math.round(v)}s`}
+                payload={[{ dataKey: s[1].dataKey, value: 192.3 }]}
+            />,
+        );
+        expect(screen.getByText("192s per passed task")).toBeInTheDocument();
+    });
+
+    test("adds the secondary line for the hovered point only", () => {
+        // The within-expected rate belongs to one run, so it rides the hover on
+        // that run's point instead of being summarized over the whole chart.
+        render(
+            <HarnessTooltip
+                active
+                label={1_700_000_000_000}
+                series={s}
+                suffix="per passed task"
+                emptyText="no passing tasks"
+                secondary={(harness) =>
+                    harness === "codex" ? "78% within 2× expected" : null
+                }
+                payload={[
+                    { dataKey: s[0].dataKey, value: 300 },
+                    { dataKey: s[1].dataKey, value: 192 },
+                ]}
+            />,
+        );
+        expect(screen.getByText("78% within 2× expected")).toBeInTheDocument();
+        expect(screen.queryAllByText(/within/)).toHaveLength(1);
+    });
+
     test("shows only the harnesses that actually ran at the hovered x", () => {
         // Recharts hands over every series, including the ones with no value
         // here. Listing those would invent runs that never happened.

{coder_eval-0.11.2 → coder_eval-0.11.3}/evalboard/app/_overview/__tests__/harness-series.test.ts

@@ -12,14 +12,16 @@ function point(
     harness: string,
     timestamp: number,
     successRate: number | null,
-    turnBudgetRate: number | null = null,
+    withinExpectedTimeRate: number | null = null,
 ): RunPoint {
     return {
         runId: `run-${timestamp}`,
         timestamp,
         harness,
         successRate,
-        turnBudgetRate,
+        turnBudgetRate: null,
+        withinExpectedTimeRate,
+        timePerPassedTask: null,
     };
 }
 
@@ -99,7 +101,7 @@ describe("pivotByHarness", () => {
             timestamp: 100,
             [key]: 90,
         });
-        expect(pivotByHarness(pts, ["codex"], "turnBudgetRate").rows[0]).toEqual(
+        expect(pivotByHarness(pts, ["codex"], "withinExpectedTimeRate").rows[0]).toEqual(
             { timestamp: 100, [key]: 40 },
         );
     });
@@ -111,7 +113,7 @@ describe("pivotByHarness", () => {
         const { rows } = pivotByHarness(
             [point("codex", 100, 90, null)],
             ["codex"],
-            "turnBudgetRate",
+            "withinExpectedTimeRate",
         );
         expect(rows).toEqual([]);
     });

coder_eval-0.11.3/evalboard/app/_overview/efficiency-charts.tsx

@@ -0,0 +1,104 @@
+"use client";
+
+// The overview's efficiency section, tabbed.
+//
+// Two signals run side by side for now. "Time" is the derived wall-clock line
+// (lib/timing.ts): expected seconds per task per harness, computed from that
+// task's own passing history by the eval runner and stamped into run.json.
+// "Turns" is the hand-written predecessor it is meant to replace, kept visible
+// while the new number is being watched rather than trusted.
+//
+// The tab strip is the seam: retiring the turn budget is deleting the "turns"
+// entry from TABS and the TurnBudgetChart import, with no other edit to this
+// page. Tab state is local — deliberately not a search param, so switching
+// charts never re-renders the server page or perturbs a shared link.
+
+import { useState } from "react";
+import type { RunPoint } from "@/lib/overview";
+import { TimePerPassedTaskChart } from "./wall-clock-chart";
+import { TurnBudgetChart } from "./turn-budget-chart";
+
+type TabKey = "time" | "turns";
+
+interface ChartProps {
+    data: RunPoint[];
+    harnesses: string[];
+    windowStart: number;
+    windowEnd: number;
+}
+
+const TABS: Array<{
+    key: TabKey;
+    label: string;
+    heading: string;
+    // Hover text on the heading: what the number actually is.
+    title: string;
+    // Shown under the heading. `scoped` appends the active-filter note.
+    blurb: (scoped: boolean) => string;
+    render: (props: ChartProps) => React.ReactNode;
+}> = [
+    {
+        key: "time",
+        label: "Time",
+        heading: "Time per Passed Task",
+        title: "Total wall clock ÷ tasks passed. Every task's seconds count, failures included; only passes count in the denominator, so a run that fails more reads slower.",
+        blurb: (scoped) =>
+            "Seconds of every task that ran ÷ the number that passed · hover a point for the share within 2× expected" +
+            (scoped ? " · scoped to the active filter" : ""),
+        render: (props) => <TimePerPassedTaskChart {...props} />,
+    },
+    {
+        key: "turns",
+        label: "Turns",
+        heading: "Within Expected Turns (%)",
+        title: "Share of tasks carrying an expected_turns budget whose visible turns stayed within 1.5× it. A budgeted task that failed counts as over budget.",
+        blurb: (scoped) =>
+            "% of budgeted tasks that stayed within 1.5× their expected turns (a budgeted task that failed counts as over budget) · runs with no budgeted task are omitted rather than plotted at 0" +
+            (scoped ? " · scoped to the active filter" : ""),
+        render: (props) => <TurnBudgetChart {...props} />,
+    },
+];
+
+export function EfficiencyCharts({
+    scoped = false,
+    ...props
+}: ChartProps & { scoped?: boolean }) {
+    const [active, setActive] = useState<TabKey>(TABS[0].key);
+    const tab = TABS.find((t) => t.key === active) ?? TABS[0];
+    return (
+        <div className="pt-4 mt-2 border-t border-dashed border-gray-200 space-y-1">
+            <div className="flex items-start justify-between gap-3">
+                <h2
+                    className="text-sm font-semibold text-gray-900"
+                    title={tab.title}
+                >
+                    {tab.heading}
+                </h2>
+                <span
+                    className="inline-flex shrink-0 overflow-hidden rounded-md border border-gray-200 text-xs"
+                    role="tablist"
+                    aria-label="Efficiency metric"
+                >
+                    {TABS.map((t) => (
+                        <button
+                            key={t.key}
+                            type="button"
+                            role="tab"
+                            aria-selected={t.key === active}
+                            onClick={() => setActive(t.key)}
+                            className={`px-2 py-0.5 transition-colors ${
+                                t.key === active
+                                    ? "bg-studio-blue text-white"
+                                    : "bg-white text-gray-600 hover:bg-gray-50"
+                            }`}
+                        >
+                            {t.label}
+                        </button>
+                    ))}
+                </span>
+            </div>
+            <p className="text-xs text-gray-500">{tab.blurb(scoped)}</p>
+            {tab.render(props)}
+        </div>
+    );
+}

{coder_eval-0.11.2 → coder_eval-0.11.3}/evalboard/app/_overview/harness-legend.tsx

@@ -57,6 +57,8 @@ export function HarnessTooltip({
     series,
     suffix,
     emptyText,
+    format = (v) => `${v.toFixed(1)}%`,
+    secondary,
 }: {
     active?: boolean;
     payload?: TooltipEntry[];
@@ -67,6 +69,12 @@ export function HarnessTooltip({
     suffix: string;
     // Shown when the hovered run produced no value for this metric at all.
     emptyText: string;
+    // How to render the value. Defaults to a percentage; a seconds chart passes
+    // its own duration formatter.
+    format?: (value: number) => string;
+    // Optional second line for a row, e.g. a companion rate the chart doesn't
+    // plot. Returning null omits it for that point.
+    secondary?: (harness: string, timestamp: number) => string | null;
 }) {
     if (!active || !payload?.length) return null;
     const byKey = new Map(series.map((s) => [s.dataKey, s]));
@@ -86,22 +94,27 @@ export function HarnessTooltip({
             ) : (
                 rows.map((e) => {
                     const s = byKey.get(String(e.dataKey))!;
+                    const sub = secondary?.(s.harness, ms) ?? null;
                     return (
-                        <div
-                            key={s.harness}
-                            className="flex items-center gap-1.5 text-gray-600 tabular-nums"
-                        >
-                            <span
-                                aria-hidden
-                                className="inline-block h-0.5 w-3 rounded-full shrink-0"
-                                style={{ backgroundColor: s.color }}
-                            />
-                            <span className="text-gray-700">
-                                {harnessShortLabel(s.harness)}
-                            </span>
-                            <span>
-                                {(e.value as number).toFixed(1)}% {suffix}
-                            </span>
+                        <div key={s.harness} className="tabular-nums">
+                            <div className="flex items-center gap-1.5 text-gray-600">
+                                <span
+                                    aria-hidden
+                                    className="inline-block h-0.5 w-3 rounded-full shrink-0"
+                                    style={{ backgroundColor: s.color }}
+                                />
+                                <span className="text-gray-700">
+                                    {harnessShortLabel(s.harness)}
+                                </span>
+                                <span>
+                                    {format(e.value as number)} {suffix}
+                                </span>
+                            </div>
+                            {sub && (
+                                <div className="pl-[1.125rem] text-gray-500">
+                                    {sub}
+                                </div>
+                            )}
                         </div>
                     );
                 })