coder-eval 0.9.0__tar.gz → 0.9.2__tar.gz

{coder_eval-0.9.0 → coder_eval-0.9.2}/.env.example

@@ -58,6 +58,17 @@ LOG_TO_FILE=false  # Set to true to enable file logging
 # GEMINI_API_KEY="AIza..."
 # ANTIGRAVITY_MODEL="gemini-3.1-pro-preview"
 
+# LiteLLM (Anthropic-compatible) open-weight backend (API_BACKEND=litellm). Point
+# the agent at a self-hosted LiteLLM proxy (see litellm/start-litellm.sh, which
+# prints these). LITELLM_COST_LOG must be the SAME path the proxy writes its
+# per-call JSONL to — that is how the run joins ACTUAL OpenRouter cost + cache back
+# onto each turn; if it is unset (or points elsewhere) the run silently falls back
+# to static rate-card pricing with 0 cache reads.
+# LITELLM_BASE_URL="http://localhost:4000"
+# LITELLM_AUTH_TOKEN="sk-..."
+# LITELLM_MODEL="zai.glm-5"
+# LITELLM_COST_LOG="./tmp/litellm-costs.jsonl"
+
 # UiPath CLI plugin-discovery pin. When unset, the sandbox auto-derives
 # the canonical `node_modules/@uipath` from the resolved `uip` binary at setup
 # time. Operators on dedicated eval hosts can pin explicitly to override.

{coder_eval-0.9.0 → coder_eval-0.9.2}/CHANGELOG.md

@@ -2,6 +2,201 @@
 
 <!-- version list -->
 
+## v0.9.2 (2026-07-31)
+
+### Bug Fixes
+
+- **cost**: A task timeout with no preserved turn is unrecorded spend, not free
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Book spend on the error and timeout paths, flag what is unpriced
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Flag every hard-killed task as a cost floor, not just the empty ones
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: Honest scoped counts, and one definition of a run's scope
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **litellm**: Gate cost_log_tags on agent capability, not route (fixes non-Claude crash)
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Make the orphaned-spend warning actually fire
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Per-attempt cost-log scoping + single run-id accessor + no-match warning
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Pin each open-weight model to a vetted provider set (no silent fallback)
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Proxy-authoritative token buckets + all-priced gate + transactional join
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Sanitize cost headers, reject non-finite cost, drop debug scaffolding
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **orchestrator**: Recover the in-flight turn's spend on a hard kill
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Add the claude-opus-5 rate so killed turns stop booking zero
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Add the five unpriced codex tiers still on OpenAI's rate card
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Correct every wrong rate-card entry and close the alias gaps
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Refresh the rate card and correct gemini-3-flash-preview
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **reports**: Count errors as misses and stop losing cost on error paths
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **reports**: Count errors as misses in one canonical pass rate
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+### Code Style
+
+- **evalboard**: Drop the swatch dots and the scope caption from the header
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+### Documentation
+
+- **cost**: Describe the per-turn backfill as the net it is
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Describe the unpriced-crash mechanism accurately and keep comments framework-general
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **litellm**: Correct the cost contract after cutting per-message distribution
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Document LITELLM_COST_LOG wiring + correct the reconciliation-cost contract
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+### Features
+
+- **cost**: Publish one accurate total on every reporting surface
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **docker**: Bind-mount the LiteLLM cost log so --driver docker joins actual cost
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **evalboard**: Compare every harness on the overview, and scope the whole page to one
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Compare harnesses on the overview, and identify each run
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Lift the harness scope to the page header, in vendor colors
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Make each turn's provider-call table a collapsed dropdown
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **evalboard**: Mark a partly-priced run total as a floor, not the bill
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: One set of pass-rate cutoffs, and a run table that pages through all history
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Per-call cost/cache table from provider_call_costs (replaces inline)
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **evalboard**: Read the canonical pass rate and surface incomplete cost
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: Say which harness, model, and framework version a run used
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **litellm**: Actual per-call cost + cache accounting for the open-weight backend
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+### Refactoring
+
+- **cost**: Correct the simulator-cost bound and drop the unread variant error share
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Cut the commentary and drop unreachable rate-card keys
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Define the unpriced-row test once, and only for new runs
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Total_cost_usd means the whole bill everywhere
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: Call the UiPath harness Delegate
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **litellm**: Cut per-message distribution; turn-level join + per-call audit record
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Drop the provider field/column — unavailable on the streaming path
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Stream the cost log + de-duplicate the OpenRouter config comment
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+### Testing
+
+- **litellm**: Cover config shape, join ordering, and defensive cost branches
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+
+## v0.9.1 (2026-07-29)
+
+### Features
+
+- **agents**: Extend cooperative early stop to codex and antigravity
+  ([`b849421`](https://github.com/UiPath/coder_eval/commit/b8494218af6870feff2a9809e3050e96da849cd8))
+
+
 ## v0.9.0 (2026-07-28)
 
 ### Features

{coder_eval-0.9.0 → coder_eval-0.9.2}/CLAUDE.md

@@ -25,6 +25,7 @@ coder_eval/
 ├── logging_config.py              # Structured logging setup
 ├── path_utils.py                  # Run ID generation, path utilities
 ├── pricing.py                     # Model pricing / cost calculation (ModelPricing, calculate_cost, register_pricing)
+├── litellm_cost.py                # Join proxy-captured ACTUAL per-call cost/cache onto turns (LiteLLM backend; apply_actual_cost)
 ├── utils.py                       # Version info helpers
 │
 ├── agents/
@@ -41,7 +42,7 @@ coder_eval/
 │   ├── routing.py                 # ApiRoute (DirectRoute/BedrockRoute)
 │   ├── sandbox.py                 # SandboxConfig, ResourceLimits
 │   ├── tasks.py                   # TaskDefinition, AgentConfig, Dataset (dataset fan-out + sample)
-│   ├── telemetry.py               # CommandTelemetry, CommandStatistics, TokenUsage, ReconciliationMessage, TranscriptMessage
+│   ├── telemetry.py               # CommandTelemetry, CommandStatistics, TokenUsage, ProviderCallCost, ReconciliationMessage, TranscriptMessage
 │   └── templates.py               # RepoSource, TemplateDirSource, StarterFilesSource
 │
 ├── criteria/                      # Criterion checker plugins (one file per type)
@@ -138,7 +139,7 @@ action.yml                         # Published composite GitHub Action (coder-ev
 - **Dataset fan-out**: `TaskDefinition.dataset` (inline rows or JSONL path) expands a single task into N row-tasks with `${row.<field>}` substitution in `initial_prompt` and `success_criteria` string fields. Expansion runs in `task_loader.expand_dataset` **before** variant resolution, so variants cannot override the dataset. Row sampling: CLI `--sample N` (fixed-seed uniform-random N over the whole dataset) overrides `--sample-per-stratum N` / `dataset.sample_per_stratum` (stratified random N-per-stratum, keyed on `stratify_field`, default `expected_skill` — for classification suites like activation). Stratified sampling (whether the N-per-stratum count comes from the **CLI** `--sample-per-stratum` flag or **YAML** `dataset.sample_per_stratum`) is **nondeterministic** by default — it re-draws each run (so the nightly activation suite broadens coverage over time). Set `dataset.sample_seed` to pin a reproducible sample; an explicit seed always wins. (Only `--sample N` uses a fixed seed, since a smoke test wants the same N rows each run.)
 - **Per-criterion aggregation**: Each `BaseCriterion` subclass exposes `aggregate(criterion, per_row_results) -> CriterionAggregate | None`. Default emits `count / mean / median / std / min / max` so every criterion is suite-thresholdable for free. Classification-style criteria return `ClassificationCriterionResult` (subclass of `CriterionResult`) and layer accuracy / P/R/F1 / confusion via the shared `overlay_classification_metrics` utility. `BaseSuccessCriterion.suite_thresholds` gates the suite on those metrics; CLI exits non-zero on any gate failure.
 - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection.
-- **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports.
+- **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports.
 - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly.
 - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block.
 - **Early stop on criterion (opt-in)**: `run_limits.stop_early` (default off) ends a single-shot Claude run early once the run's **armed** criteria are decided, so a raised `max_turns` isn't wasted on the smoke flavor. A criterion is armed by `stop_when: pass|fail|decided|auto`; only criteria that can decide from a partial trajectory may arm (non-empty `live_stop_polarities` ClassVar + `live_verdict` override — currently `skill_triggered`, `command_executed`; CE025 keeps the two consistent). `decided` arms **both** polarities; `auto` arms whichever polarities **this instance** can decide — the value for dataset-fanned criteria whose positive/distractor role flips per row. Stop rule: the pass-stop fires when every **pass-armed** criterion live-passes (fail-armed distractors are not required to pass; zero pass-armed ⇒ never pass-stops); the fail-stop fires on the first fail-armed live-fail but is **deferred while any pass-armed criterion is undecided** — a distractor misfire must not truncate a positive row's recall signal, so the latched misfire fires once the positives resolve (or the run continues to the cap). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a `stop_early: false` run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` through the Claude agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. An early-stopped run gates on the **armed subset** (`EvaluationResult.armed_criteria_passed`); a completed run gates on the full set. Every unsupported use is a hard error at resolution (plan *and* run), and a runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo`, report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. Defaults off ⇒ behavior byte-for-behavior unchanged.

{coder_eval-0.9.0 → coder_eval-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coder-eval
-Version: 0.9.0
+Version: 0.9.2
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval

{coder_eval-0.9.0 → coder_eval-0.9.2}/action.yml

@@ -33,7 +33,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.9.0"   # <-- kept in sync with releases by release.yml
+    default: "0.9.2"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

{coder_eval-0.9.0 → coder_eval-0.9.2}/docs/REPORT_SCHEMA.md

@@ -51,6 +51,21 @@ run-level summary; full per-replicate detail lives in each `task.json`.
 | `framework_version` | `str` | Coder Eval version chip. |
 | `environment_info` | `dict` | Version/dependency info (may nest, e.g. `tool_plugins`). |
 
+These are **computed**, not stored — derived from the counts and rows above on every
+serialization, so they cannot drift from what they summarize. Read them rather than
+re-deriving your own; independent re-derivations are how two consumers end up
+publishing different numbers for the same run.
+
+| Key | Type | Meaning |
+| --- | --- | --- |
+| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). |
+| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. |
+| `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. |
+| `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. |
+| `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. |
+| `tasks_cost_incomplete` | `int` | Rows whose recorded spend is missing money (unpriced model, or a hard kill that lost an in-flight turn). |
+| `cost_complete` | `bool` | `tasks_cost_incomplete == 0`. When false, every cost figure above is a **floor**, not the bill. A run is never failed for this — see [Missing cost is never fatal](#missing-cost-is-never-fatal). |
+
 ### `task_results[]` — the flat per-task row
 
 Each entry is an **untyped dict** (a denormalization, not a Pydantic model) with keys
@@ -58,7 +73,10 @@ including: `task_id`, `replicate_index`, `variant_id`, `status`
 ([`FinalStatus`](#finalstatus)), `weighted_score`, `duration`, `iteration_count`,
 `tags`, `task_path`, `model_used`, `reference_similarity`, the token buckets
 (`input_tokens` = uncached input, `output_tokens`, `cache_creation_input_tokens`,
-`cache_read_input_tokens`, `total_tokens`), `total_cost_usd`, `expected_commands`,
+`cache_read_input_tokens`, `total_tokens`), the cost fields
+(`total_cost_usd` = agent + judge + simulator, plus the `agent_cost_usd` /
+`judge_cost_usd` / `simulator_cost_usd` slices and the `cost_complete` flag),
+`expected_commands`,
 `actual_commands`, `commands_efficiency`, `agent_config`, `sdk_options`,
 `installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_turns`,
 `max_turns_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`,
@@ -66,6 +84,29 @@ including: `task_id`, `replicate_index`, `variant_id`, `status`
 turn digest (`{iteration, duration_seconds, command_count, assistant_turn_count,
 crashed, crash_reason}`) — the full transcript is in `task.json`.
 
+### Missing cost is never fatal
+
+Pricing degrades; the evaluation does not. A model absent from the rate card, a turn
+the backend never priced, a hard-killed task that lost its in-flight spend: each one
+lowers a total and sets `cost_complete: false`. None of them raises, none of them
+books a zero, and none of them changes a run's exit code.
+
+The reasoning is that the two failure modes are not symmetric. A missing cost is
+recoverable after the fact — the token counts are on the record, so a corrected rate
+card reprices the run from its artifacts. A failed run is not: the tokens are already
+spent and the only way back is to run it again. So the framework warns loudly and
+keeps going.
+
+The warning fires up front. `check_pricing_coverage` walks every model the run pins
+(subject agents and judge criteria) before the first task dispatches, and logs the
+ones the card cannot price — early enough to fix the card and restart while it is
+still cheap. After that the run is on its own: totals become floors, and
+`tasks_cost_incomplete` says how many rows are behind that floor.
+
+Consumers should treat any cost field as a lower bound whenever `cost_complete` is
+false, and must not read `None` as `0.0` — "nothing could be priced" and "it was
+free" are different facts.
+
 ---
 
 ## `task.json` — `EvaluationResult`
@@ -127,15 +168,22 @@ fields so subclass keys round-trip.
 `iteration`, `user_input`, `agent_output`, `commands` (`list[CommandTelemetry]`),
 `timestamp`, `duration_seconds`, `token_usage`, `model_used`, `assistant_turn_count`,
 `messages` (`list[TranscriptMessage]`, discriminated on `role`:
-`user`/`assistant`/`reconciliation`), `num_turns`, `max_turns_exhausted`,
+`user`/`assistant`/`reconciliation`), `provider_call_costs`
+(`list[ProviderCallCost]` — one row per real upstream call with its ACTUAL cost +
+cache buckets, captured proxy-side on the LiteLLM open-weight backend and rendered
+by the evalboard as a per-call table; empty on every other
+backend), `num_turns`, `max_turns_exhausted`,
 `result_summary` (`{is_error, subtype, stop_reason, result}`), `crashed`,
 `crash_reason`.
 
 > **Token invariant.** Summing the four token buckets across `messages`
 > (assistant + the synthetic `reconciliation` entry) equals `token_usage` exactly.
 > The `reconciliation` message carries the residual the per-message stream
-> under-reports; it has no cost and is excluded from turn/generation counts. See the
-> [Claude Code guide](agents/CLAUDE_CODE.md#telemetry).
+> under-reports; it has no cost and is excluded from turn/generation counts. The
+> LiteLLM actual-cost join writes cost at the TURN level only (`token_usage.total_cost_usd`
+> = the real bill) plus the per-call `provider_call_costs` audit record — it does
+> NOT touch the message token buckets, so this invariant holds on every backend.
+> See the [Claude Code guide](agents/CLAUDE_CODE.md#telemetry).
 
 ### EarlyStopInfo
 
@@ -247,6 +295,12 @@ respectively), checked after each completed agent turn — see
 - `TokenUsage.total_tokens` is not serialized; sum the buckets (or use the computed
   `input_tokens` + `output_tokens` + cache buckets).
 - `EarlyStopInfo` presence is itself the "stopped early" signal.
+- `total_cost_usd` is the whole bill (agent + judge + simulator) at both row and run
+  level; `agent_cost_usd` is the agent-only slice. `TokenUsage.total_cost_usd` is a
+  different thing: the cost of those tokens, so always agent-only. `run_limits.max_usd`
+  gates on that one, since judge and simulator spend is not known mid-run.
+- A cost of `None` means unpriced, not free, and any total is a floor while
+  `cost_complete` is false — see [Missing cost is never fatal](#missing-cost-is-never-fatal).
 
 ## See also
 

{coder_eval-0.9.0 → coder_eval-0.9.2}/docs/agents/ANTIGRAVITY.md

@@ -168,18 +168,15 @@ as every other agent.
 
 ## Known limitations
 
-1. **No cooperative early stop.** Antigravity does not support the `should_stop`
-   seam, so `run_limits.stop_early` is unsupported for this agent (it errors at
-   resolution, as it does for any non-Claude agent).
-2. **No endpoint routing.** Only `GEMINI_API_KEY` + `ANTIGRAVITY_MODEL` are read —
+1. **No endpoint routing.** Only `GEMINI_API_KEY` + `ANTIGRAVITY_MODEL` are read —
    there is no base-URL, project, region, or gateway override.
-3. **Default-model drift.** The runtime fallback (`gemini-3.5-flash`) may differ from
+2. **Default-model drift.** The runtime fallback (`gemini-3.5-flash`) may differ from
    what a given release's docs or example tasks pin; always set `agent.model`
    explicitly for reproducible runs.
-4. **`kill_sync()` is best-effort.** The SDK's cancel/disconnect are async-only, so
+3. **`kill_sync()` is best-effort.** The SDK's cancel/disconnect are async-only, so
    the watchdog's synchronous kill only flips agent state to `ERROR`; real teardown
    happens on the subsequent async `stop()`.
-5. **Process-global spawn lock.** The SDK spawns `localharness` via a subprocess with
+4. **Process-global spawn lock.** The SDK spawns `localharness` via a subprocess with
    no env-injection seam, so the agent transiently mutates `PATH` across the spawn
    under a process-wide lock. This serializes harness startup across concurrent
    tasks (it does not serialize the turns themselves).

{coder_eval-0.9.0 → coder_eval-0.9.2}/docs/agents/CLAUDE_CODE.md

@@ -145,13 +145,14 @@ simulator force `[]` for the same reason.)
   UiPath CLI plugin discovery; when unset the sandbox derives it from the resolved
   `uip` binary. See [User Guide → Environment Variables](../USER_GUIDE.md#environment-variables).
 
-## Early stop (Claude-only)
+## Early stop
 
-Claude Code is the only agent that supports the cooperative early-stop seam. With
+Claude Code supports the cooperative early-stop seam, as do the
+[Codex](CODEX.md) and [Antigravity](ANTIGRAVITY.md) agents. With
 `run_limits.stop_early: true`, a single-shot run ends cleanly at the next
 tool-call boundary once its **armed** criteria (`stop_when: pass|fail|decided`) are
 decided — so a raised `max_turns` isn't wasted on a smoke run. Early stop errors at
-resolution for any other agent. See the
+resolution for any agent that does not declare `supports_cooperative_stop`. See the
 [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) for the full contract.
 
 ## Telemetry

{coder_eval-0.9.0 → coder_eval-0.9.2}/docs/agents/CODEX.md

@@ -216,6 +216,7 @@ The Codex SDK is synchronous. The agent uses `_run_async()` helper to detect and
 | **Session Resume** | `--resume {session_id}` | Via thread ID |
 | **Permissions** | `permission_mode` + `allowed_tools` | `permission_mode` → sandbox/approval + `allowed_tools`/`disallowed_tools` → thread config |
 | **Tool Enforcement** | Not enforced by Coder Eval wrapper | `enabled_tools` honored; `disabled_tools` NOT enforced by the SDK |
+| **Early stop** | Supported (cooperative `should_stop`, polled between messages) | Supported — polled after each streamed notification; the in-flight turn is interrupted best-effort |
 
 ## Known Limitations
 

coder_eval-0.9.2/evalboard/app/_components/__tests__/harness-badge.test.tsx

@@ -0,0 +1,45 @@
+import { describe, expect, test } from "vitest";
+import { render, screen } from "@testing-library/react";
+import { KNOWN_HARNESSES } from "@/lib/harness";
+import { HarnessBadge, harnessShortLabel } from "../harness-badge";
+
+describe("harnessShortLabel", () => {
+    test("every known harness has a human label", () => {
+        // A missing entry falls through to the raw id, which would put
+        // "delegate-sdk" in a legend next to "Codex" and "Antigravity".
+        for (const h of KNOWN_HARNESSES) {
+            expect(harnessShortLabel(h)).not.toBe(h);
+        }
+    });
+
+    test("the UiPath harness reads as Delegate", () => {
+        // The run data's id stays `delegate-sdk` (it's the registered
+        // `agent.type`); only the label people read is the short one.
+        expect(harnessShortLabel("delegate-sdk")).toBe("Delegate");
+    });
+
+    test("an unknown harness falls back to its id rather than a wrong name", () => {
+        expect(harnessShortLabel("some-new-agent")).toBe("some-new-agent");
+    });
+});
+
+describe("HarnessBadge", () => {
+    test("names the vendor in the alt text, not just the product", () => {
+        render(<HarnessBadge harness="delegate-sdk" />);
+        expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument();
+    });
+
+    test("renders the id as text when there is no logo for it", () => {
+        // Better a raw id than another vendor's mark on someone else's run.
+        render(<HarnessBadge harness="some-new-agent" />);
+        expect(screen.getByText("some-new-agent")).toBeInTheDocument();
+    });
+
+    test("takes a size so the chart legend can sit inside 11px text", () => {
+        render(<HarnessBadge harness="codex" size={14} />);
+        expect(screen.getByAltText("Codex · OpenAI")).toHaveAttribute(
+            "width",
+            "14",
+        );
+    });
+});

coder_eval-0.9.2/evalboard/app/_components/__tests__/harness-selector.test.tsx

@@ -0,0 +1,81 @@
+import { describe, expect, test, vi } from "vitest";
+import { render, screen } from "@testing-library/react";
+import { KNOWN_HARNESSES } from "@/lib/harness";
+import { harnessShortLabel } from "../harness-badge";
+
+// The selector reads router/params hooks; stub them so it renders in jsdom
+// without a router provider.
+const replace = vi.fn();
+vi.mock("next/navigation", () => ({
+    useRouter: () => ({ replace }),
+    usePathname: () => "/",
+    useSearchParams: () => new URLSearchParams(),
+}));
+
+const { HarnessSelector } = await import("../harness-selector");
+
+describe("HarnessSelector", () => {
+    test("the all-harness segment is present and selected by default", () => {
+        render(
+            <HarnessSelector
+                current={null}
+                harnesses={[...KNOWN_HARNESSES]}
+                includeAll
+            />,
+        );
+        expect(screen.getByRole("button", { name: "All" })).toHaveAttribute(
+            "aria-pressed",
+            "true",
+        );
+    });
+
+    test("a scoped harness reads as pressed, and All does not", () => {
+        render(
+            <HarnessSelector
+                current="codex"
+                harnesses={[...KNOWN_HARNESSES]}
+                includeAll
+            />,
+        );
+        expect(screen.getByRole("button", { name: "Codex" })).toHaveAttribute(
+            "aria-pressed",
+            "true",
+        );
+        expect(screen.getByRole("button", { name: "All" })).toHaveAttribute(
+            "aria-pressed",
+            "false",
+        );
+    });
+
+    test("a deep-linked harness that has aged out still shows as a segment", () => {
+        // Otherwise `?h=delegate-sdk` after a quiet fortnight renders a control
+        // with nothing selected, which reads as an unscoped page.
+        render(
+            <HarnessSelector
+                current="delegate-sdk"
+                harnesses={["claude-code", "codex"]}
+                includeAll
+            />,
+        );
+        expect(
+            screen.getByRole("button", { name: "Delegate" }),
+        ).toHaveAttribute("aria-pressed", "true");
+    });
+
+    test("every segment is named for screen readers, not color-only", () => {
+        render(
+            <HarnessSelector
+                current={null}
+                harnesses={[...KNOWN_HARNESSES]}
+                includeAll
+            />,
+        );
+        // The accessible name comes from aria-label, so it survives the
+        // below-`sm` breakpoint that hides the visible text.
+        for (const h of KNOWN_HARNESSES) {
+            expect(
+                screen.getByRole("button", { name: harnessShortLabel(h) }),
+            ).toBeInTheDocument();
+        }
+    });
+});

{coder_eval-0.9.0 → coder_eval-0.9.2}/evalboard/app/_components/harness-badge.tsx

@@ -22,6 +22,16 @@ const HARNESS_LOGO: Record<string, { src: string; label: string; short: string }
         label: "Antigravity · Google Gemini",
         short: "Antigravity",
     },
+    // The Delegate harness is UiPath's own (the coder_eval_uipath plugin), so
+    // its vendor mark is the UiPath logo already served for the site header
+    // rather than a per-harness file under /harness. The key stays `delegate-sdk`
+    // because that is the `agent.type` the plugin registers and therefore what
+    // run.json carries; only the label people read is shortened.
+    "delegate-sdk": {
+        src: "/uipath.png",
+        label: "Delegate · UiPath",
+        short: "Delegate",
+    },
 };
 
 // Short human label for a harness id ("Claude Code"), for selectors and prose.
@@ -30,7 +40,15 @@ export function harnessShortLabel(harness: string): string {
     return HARNESS_LOGO[harness]?.short ?? harness;
 }
 
-export function HarnessBadge({ harness }: { harness?: string | null }) {
+export function HarnessBadge({
+    harness,
+    // Square edge in px. Defaults to the runs-table size; the chart legend asks
+    // for a smaller mark so the logo sits inside a line of 11px text.
+    size = 20,
+}: {
+    harness?: string | null;
+    size?: number;
+}) {
     const key = harness ?? "claude-code";
     const logo = HARNESS_LOGO[key];
     // Unknown harness: show the raw id rather than a misleading logo.
@@ -42,8 +60,8 @@ export function HarnessBadge({ harness }: { harness?: string | null }) {
             src={logo.src}
             alt={logo.label}
             title={logo.label}
-            width={20}
-            height={20}
+            width={size}
+            height={size}
             className="rounded-sm"
         />
     );