coder-eval 0.9.1__tar.gz → 0.9.3__tar.gz

{coder_eval-0.9.1 → coder_eval-0.9.3}/.env.example

@@ -58,6 +58,17 @@ LOG_TO_FILE=false  # Set to true to enable file logging
 # GEMINI_API_KEY="AIza..."
 # ANTIGRAVITY_MODEL="gemini-3.1-pro-preview"
 
+# LiteLLM (Anthropic-compatible) open-weight backend (API_BACKEND=litellm). Point
+# the agent at a self-hosted LiteLLM proxy (see litellm/start-litellm.sh, which
+# prints these). LITELLM_COST_LOG must be the SAME path the proxy writes its
+# per-call JSONL to — that is how the run joins ACTUAL OpenRouter cost + cache back
+# onto each turn; if it is unset (or points elsewhere) the run silently falls back
+# to static rate-card pricing with 0 cache reads.
+# LITELLM_BASE_URL="http://localhost:4000"
+# LITELLM_AUTH_TOKEN="sk-..."
+# LITELLM_MODEL="zai.glm-5"
+# LITELLM_COST_LOG="./tmp/litellm-costs.jsonl"
+
 # UiPath CLI plugin-discovery pin. When unset, the sandbox auto-derives
 # the canonical `node_modules/@uipath` from the resolved `uip` binary at setup
 # time. Operators on dedicated eval hosts can pin explicitly to override.

{coder_eval-0.9.1 → coder_eval-0.9.3}/CHANGELOG.md

@@ -2,6 +2,214 @@
 
 <!-- version list -->
 
+## v0.9.3 (2026-08-04)
+
+### Bug Fixes
+
+- **early-stop**: Address PR review — polarity-blind budget, pass_threshold displacement,
+  gate-semantic split ([#74](https://github.com/UiPath/coder_eval/pull/74),
+  [`800ac77`](https://github.com/UiPath/coder_eval/commit/800ac7730e53a0ce9e9128b42ea91f0030d01f89))
+
+### Chores
+
+- **deps**: Bump aiohttp 3.14.1→3.14.3, cryptography 49.0.0→50.0.0
+  ([#74](https://github.com/UiPath/coder_eval/pull/74),
+  [`800ac77`](https://github.com/UiPath/coder_eval/commit/800ac7730e53a0ce9e9128b42ea91f0030d01f89))
+
+### Features
+
+- **early-stop**: Weighted ceiling/floor bounds + decision-step budget
+  ([#74](https://github.com/UiPath/coder_eval/pull/74),
+  [`800ac77`](https://github.com/UiPath/coder_eval/commit/800ac7730e53a0ce9e9128b42ea91f0030d01f89))
+
+
+## v0.9.2 (2026-07-31)
+
+### Bug Fixes
+
+- **cost**: A task timeout with no preserved turn is unrecorded spend, not free
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Book spend on the error and timeout paths, flag what is unpriced
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Flag every hard-killed task as a cost floor, not just the empty ones
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: Honest scoped counts, and one definition of a run's scope
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **litellm**: Gate cost_log_tags on agent capability, not route (fixes non-Claude crash)
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Make the orphaned-spend warning actually fire
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Per-attempt cost-log scoping + single run-id accessor + no-match warning
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Pin each open-weight model to a vetted provider set (no silent fallback)
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Proxy-authoritative token buckets + all-priced gate + transactional join
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Sanitize cost headers, reject non-finite cost, drop debug scaffolding
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **orchestrator**: Recover the in-flight turn's spend on a hard kill
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Add the claude-opus-5 rate so killed turns stop booking zero
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Add the five unpriced codex tiers still on OpenAI's rate card
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Correct every wrong rate-card entry and close the alias gaps
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **pricing**: Refresh the rate card and correct gemini-3-flash-preview
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **reports**: Count errors as misses and stop losing cost on error paths
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **reports**: Count errors as misses in one canonical pass rate
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+### Code Style
+
+- **evalboard**: Drop the swatch dots and the scope caption from the header
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+### Documentation
+
+- **cost**: Describe the per-turn backfill as the net it is
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Describe the unpriced-crash mechanism accurately and keep comments framework-general
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **litellm**: Correct the cost contract after cutting per-message distribution
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Document LITELLM_COST_LOG wiring + correct the reconciliation-cost contract
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+### Features
+
+- **cost**: Publish one accurate total on every reporting surface
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **docker**: Bind-mount the LiteLLM cost log so --driver docker joins actual cost
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **evalboard**: Compare every harness on the overview, and scope the whole page to one
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Compare harnesses on the overview, and identify each run
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Lift the harness scope to the page header, in vendor colors
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Make each turn's provider-call table a collapsed dropdown
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **evalboard**: Mark a partly-priced run total as a floor, not the bill
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: One set of pass-rate cutoffs, and a run table that pages through all history
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **evalboard**: Per-call cost/cache table from provider_call_costs (replaces inline)
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **evalboard**: Read the canonical pass rate and surface incomplete cost
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: Say which harness, model, and framework version a run used
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **litellm**: Actual per-call cost + cache accounting for the open-weight backend
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+### Refactoring
+
+- **cost**: Correct the simulator-cost bound and drop the unread variant error share
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Cut the commentary and drop unreachable rate-card keys
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Define the unpriced-row test once, and only for new runs
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **cost**: Total_cost_usd means the whole bill everywhere
+  ([#63](https://github.com/UiPath/coder_eval/pull/63),
+  [`93c7fc0`](https://github.com/UiPath/coder_eval/commit/93c7fc086bbefe74e25625dd8b77106bef736c09))
+
+- **evalboard**: Call the UiPath harness Delegate
+  ([#69](https://github.com/UiPath/coder_eval/pull/69),
+  [`0bdac0b`](https://github.com/UiPath/coder_eval/commit/0bdac0b2bab0d9f2937a2ddc5e54b26d110a56d1))
+
+- **litellm**: Cut per-message distribution; turn-level join + per-call audit record
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Drop the provider field/column — unavailable on the streaming path
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+- **litellm**: Stream the cost log + de-duplicate the OpenRouter config comment
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+### Testing
+
+- **litellm**: Cover config shape, join ordering, and defensive cost branches
+  ([#66](https://github.com/UiPath/coder_eval/pull/66),
+  [`4131a2a`](https://github.com/UiPath/coder_eval/commit/4131a2a2983c347023927c66c04756595c3dbceb))
+
+
 ## v0.9.1 (2026-07-29)
 
 ### Features

{coder_eval-0.9.1 → coder_eval-0.9.3}/CLAUDE.md

@@ -25,6 +25,7 @@ coder_eval/
 ├── logging_config.py              # Structured logging setup
 ├── path_utils.py                  # Run ID generation, path utilities
 ├── pricing.py                     # Model pricing / cost calculation (ModelPricing, calculate_cost, register_pricing)
+├── litellm_cost.py                # Join proxy-captured ACTUAL per-call cost/cache onto turns (LiteLLM backend; apply_actual_cost)
 ├── utils.py                       # Version info helpers
 │
 ├── agents/
@@ -41,7 +42,7 @@ coder_eval/
 │   ├── routing.py                 # ApiRoute (DirectRoute/BedrockRoute)
 │   ├── sandbox.py                 # SandboxConfig, ResourceLimits
 │   ├── tasks.py                   # TaskDefinition, AgentConfig, Dataset (dataset fan-out + sample)
-│   ├── telemetry.py               # CommandTelemetry, CommandStatistics, TokenUsage, ReconciliationMessage, TranscriptMessage
+│   ├── telemetry.py               # CommandTelemetry, CommandStatistics, TokenUsage, ProviderCallCost, ReconciliationMessage, TranscriptMessage
 │   └── templates.py               # RepoSource, TemplateDirSource, StarterFilesSource
 │
 ├── criteria/                      # Criterion checker plugins (one file per type)
@@ -138,10 +139,10 @@ action.yml                         # Published composite GitHub Action (coder-ev
 - **Dataset fan-out**: `TaskDefinition.dataset` (inline rows or JSONL path) expands a single task into N row-tasks with `${row.<field>}` substitution in `initial_prompt` and `success_criteria` string fields. Expansion runs in `task_loader.expand_dataset` **before** variant resolution, so variants cannot override the dataset. Row sampling: CLI `--sample N` (fixed-seed uniform-random N over the whole dataset) overrides `--sample-per-stratum N` / `dataset.sample_per_stratum` (stratified random N-per-stratum, keyed on `stratify_field`, default `expected_skill` — for classification suites like activation). Stratified sampling (whether the N-per-stratum count comes from the **CLI** `--sample-per-stratum` flag or **YAML** `dataset.sample_per_stratum`) is **nondeterministic** by default — it re-draws each run (so the nightly activation suite broadens coverage over time). Set `dataset.sample_seed` to pin a reproducible sample; an explicit seed always wins. (Only `--sample N` uses a fixed seed, since a smoke test wants the same N rows each run.)
 - **Per-criterion aggregation**: Each `BaseCriterion` subclass exposes `aggregate(criterion, per_row_results) -> CriterionAggregate | None`. Default emits `count / mean / median / std / min / max` so every criterion is suite-thresholdable for free. Classification-style criteria return `ClassificationCriterionResult` (subclass of `CriterionResult`) and layer accuracy / P/R/F1 / confusion via the shared `overlay_classification_metrics` utility. `BaseSuccessCriterion.suite_thresholds` gates the suite on those metrics; CLI exits non-zero on any gate failure.
 - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection.
-- **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports.
+- **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports.
 - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly.
-- **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block.
-- **Early stop on criterion (opt-in)**: `run_limits.stop_early` (default off) ends a single-shot Claude run early once the run's **armed** criteria are decided, so a raised `max_turns` isn't wasted on the smoke flavor. A criterion is armed by `stop_when: pass|fail|decided|auto`; only criteria that can decide from a partial trajectory may arm (non-empty `live_stop_polarities` ClassVar + `live_verdict` override — currently `skill_triggered`, `command_executed`; CE025 keeps the two consistent). `decided` arms **both** polarities; `auto` arms whichever polarities **this instance** can decide — the value for dataset-fanned criteria whose positive/distractor role flips per row. Stop rule: the pass-stop fires when every **pass-armed** criterion live-passes (fail-armed distractors are not required to pass; zero pass-armed ⇒ never pass-stops); the fail-stop fires on the first fail-armed live-fail but is **deferred while any pass-armed criterion is undecided** — a distractor misfire must not truncate a positive row's recall signal, so the latched misfire fires once the positives resolve (or the run continues to the cap). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a `stop_early: false` run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` through the Claude agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. An early-stopped run gates on the **armed subset** (`EvaluationResult.armed_criteria_passed`); a completed run gates on the full set. Every unsupported use is a hard error at resolution (plan *and* run), and a runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo`, report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. Defaults off ⇒ behavior byte-for-behavior unchanged.
+- **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `max_steps_to_decide`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step-budget breach to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express.
+- **Early stop on criterion (opt-in)**: `run_limits.stop_early` (default off) ends a single-shot Claude run early once the run's **armed** criteria are decided, so a raised `max_turns` isn't wasted on the smoke flavor. A criterion is armed by `stop_when: pass|fail|decided|auto`; only criteria that can decide from a partial trajectory may arm — "is this criterion type live-observable" is `models.LiveSuccessCriterion` subclassing (currently `skill_triggered`, `command_executed`), the single source of truth `validate_early_stop`/`EarlyStopWatcher` check directly via `isinstance`; each subclass implements the abstract, checker-independent `live_decidable_polarities()` (a pure function of its own fields) alongside the checker's `live_verdict` override, and lint rule CE025 (`tests/test_custom_lint.py::TestCE025LiveVerdictConsistency`, a registry-based whole-tree check, not a per-file AST rule) keeps the two paired. `decided` arms **both** polarities; `auto` arms whichever polarities **this instance** can decide — the value for dataset-fanned criteria whose positive/distractor role flips per row. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing pre-weighting behavior byte-for-byte) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed subset's **ceiling** (best case for everything still undecided) can no longer reach the threshold, a pass-stop once the pass-armed subset's **floor** (worst case) already meets it — both **deferred while any pass-armed criterion is undecided**, so a distractor misfire never truncates a positive row's recall signal before the positives resolve (or the run continues to the cap). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a `stop_early: false` run. A per-criterion `max_steps_to_decide` (on `LiveSuccessCriterion` only, requires `stop_when`) caps tool-call steps spent still undecided — cumulative across retry attempts of the same turn — before `EarlyStopReason.DECISION_BUDGET_EXCEEDED` force-fails the run outright, bypassing the weighted gate (nothing to weigh a criterion that never decided against). Driven by `orchestration/early_stop.py::EarlyStopWatcher` through the Claude agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. An early-stopped run gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally (even with `stop_early: true`) gates on the full set via the strict-AND `all_criteria_passed` — weight magnitude only forgives under the former, so the weighted gate is contingent on the watcher itself firing, not solely on the configured threshold. Every unsupported use is a hard error at resolution (plan *and* run), and a runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. Defaults off ⇒ behavior byte-for-behavior unchanged.
 
 ## Success Criteria (14 types)
 

{coder_eval-0.9.1 → coder_eval-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coder-eval
-Version: 0.9.1
+Version: 0.9.3
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval

{coder_eval-0.9.1 → coder_eval-0.9.3}/action.yml

@@ -33,7 +33,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.9.1"   # <-- kept in sync with releases by release.yml
+    default: "0.9.3"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

{coder_eval-0.9.1 → coder_eval-0.9.3}/docs/EXTENDING.md

@@ -210,8 +210,16 @@ Notes:
   `count/mean/median/std/min/max`, so your criterion is suite-thresholdable for free.
   Classification-style criteria return a `ClassificationCriterionResult` and layer
   accuracy / precision / recall / F1 / confusion on top.
-- For **early stop**, implement `live_verdict(...)` and declare
-  `live_stop_polarities` — a lint rule keeps the two consistent.
+- For **early stop**, make your criterion model subclass `LiveSuccessCriterion`
+  (`models/criteria.py`) instead of `BaseSuccessCriterion`, implement its
+  abstract `live_decidable_polarities()` (a pure function of the criterion's
+  own fields — no `turn_records`, no checker instance), and override the
+  checker's `live_verdict(...)`. `LiveSuccessCriterion` subclassing is the
+  single source of truth for "is this criterion type live-observable" —
+  `validate_early_stop`/`EarlyStopWatcher` check `isinstance(c,
+  LiveSuccessCriterion)` directly, no separate checker-side flag. A lint rule
+  (`tests/test_custom_lint.py::TestCE025LiveVerdictConsistency`) keeps the
+  model subclassing and the checker's `live_verdict` override paired.
 
 > A duplicate `criterion_type` **overwrites** the earlier checker with a warning (not
 > a hard error, unlike agents) — keep type strings unique.

{coder_eval-0.9.1 → coder_eval-0.9.3}/docs/REPORT_SCHEMA.md

@@ -51,6 +51,21 @@ run-level summary; full per-replicate detail lives in each `task.json`.
 | `framework_version` | `str` | Coder Eval version chip. |
 | `environment_info` | `dict` | Version/dependency info (may nest, e.g. `tool_plugins`). |
 
+These are **computed**, not stored — derived from the counts and rows above on every
+serialization, so they cannot drift from what they summarize. Read them rather than
+re-deriving your own; independent re-derivations are how two consumers end up
+publishing different numbers for the same run.
+
+| Key | Type | Meaning |
+| --- | --- | --- |
+| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). |
+| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. |
+| `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. |
+| `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. |
+| `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. |
+| `tasks_cost_incomplete` | `int` | Rows whose recorded spend is missing money (unpriced model, or a hard kill that lost an in-flight turn). |
+| `cost_complete` | `bool` | `tasks_cost_incomplete == 0`. When false, every cost figure above is a **floor**, not the bill. A run is never failed for this — see [Missing cost is never fatal](#missing-cost-is-never-fatal). |
+
 ### `task_results[]` — the flat per-task row
 
 Each entry is an **untyped dict** (a denormalization, not a Pydantic model) with keys
@@ -58,7 +73,10 @@ including: `task_id`, `replicate_index`, `variant_id`, `status`
 ([`FinalStatus`](#finalstatus)), `weighted_score`, `duration`, `iteration_count`,
 `tags`, `task_path`, `model_used`, `reference_similarity`, the token buckets
 (`input_tokens` = uncached input, `output_tokens`, `cache_creation_input_tokens`,
-`cache_read_input_tokens`, `total_tokens`), `total_cost_usd`, `expected_commands`,
+`cache_read_input_tokens`, `total_tokens`), the cost fields
+(`total_cost_usd` = agent + judge + simulator, plus the `agent_cost_usd` /
+`judge_cost_usd` / `simulator_cost_usd` slices and the `cost_complete` flag),
+`expected_commands`,
 `actual_commands`, `commands_efficiency`, `agent_config`, `sdk_options`,
 `installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_turns`,
 `max_turns_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`,
@@ -66,6 +84,29 @@ including: `task_id`, `replicate_index`, `variant_id`, `status`
 turn digest (`{iteration, duration_seconds, command_count, assistant_turn_count,
 crashed, crash_reason}`) — the full transcript is in `task.json`.
 
+### Missing cost is never fatal
+
+Pricing degrades; the evaluation does not. A model absent from the rate card, a turn
+the backend never priced, a hard-killed task that lost its in-flight spend: each one
+lowers a total and sets `cost_complete: false`. None of them raises, none of them
+books a zero, and none of them changes a run's exit code.
+
+The reasoning is that the two failure modes are not symmetric. A missing cost is
+recoverable after the fact — the token counts are on the record, so a corrected rate
+card reprices the run from its artifacts. A failed run is not: the tokens are already
+spent and the only way back is to run it again. So the framework warns loudly and
+keeps going.
+
+The warning fires up front. `check_pricing_coverage` walks every model the run pins
+(subject agents and judge criteria) before the first task dispatches, and logs the
+ones the card cannot price — early enough to fix the card and restart while it is
+still cheap. After that the run is on its own: totals become floors, and
+`tasks_cost_incomplete` says how many rows are behind that floor.
+
+Consumers should treat any cost field as a lower bound whenever `cost_complete` is
+false, and must not read `None` as `0.0` — "nothing could be priced" and "it was
+free" are different facts.
+
 ---
 
 ## `task.json` — `EvaluationResult`
@@ -127,23 +168,33 @@ fields so subclass keys round-trip.
 `iteration`, `user_input`, `agent_output`, `commands` (`list[CommandTelemetry]`),
 `timestamp`, `duration_seconds`, `token_usage`, `model_used`, `assistant_turn_count`,
 `messages` (`list[TranscriptMessage]`, discriminated on `role`:
-`user`/`assistant`/`reconciliation`), `num_turns`, `max_turns_exhausted`,
+`user`/`assistant`/`reconciliation`), `provider_call_costs`
+(`list[ProviderCallCost]` — one row per real upstream call with its ACTUAL cost +
+cache buckets, captured proxy-side on the LiteLLM open-weight backend and rendered
+by the evalboard as a per-call table; empty on every other
+backend), `num_turns`, `max_turns_exhausted`,
 `result_summary` (`{is_error, subtype, stop_reason, result}`), `crashed`,
 `crash_reason`.
 
 > **Token invariant.** Summing the four token buckets across `messages`
 > (assistant + the synthetic `reconciliation` entry) equals `token_usage` exactly.
 > The `reconciliation` message carries the residual the per-message stream
-> under-reports; it has no cost and is excluded from turn/generation counts. See the
-> [Claude Code guide](agents/CLAUDE_CODE.md#telemetry).
+> under-reports; it has no cost and is excluded from turn/generation counts. The
+> LiteLLM actual-cost join writes cost at the TURN level only (`token_usage.total_cost_usd`
+> = the real bill) plus the per-call `provider_call_costs` audit record — it does
+> NOT touch the message token buckets, so this invariant holds on every backend.
+> See the [Claude Code guide](agents/CLAUDE_CODE.md#telemetry).
 
 ### EarlyStopInfo
 
 Present (non-`null`) iff the run stopped early — there is no separate boolean.
-Fields: `reason` (`criterion_passed` / `criterion_failed`),
+Fields: `reason` (`criterion_passed` / `criterion_failed` /
+`decision_budget_exceeded` — the last forces `FinalStatus.FAILURE` outright,
+bypassing the weighted gate),
 `deciding_criterion_type`, `deciding_criterion_description`, `armed_criteria`,
 `sdk_turn_index`, `tool_call_index` (1-based, includes the in-flight call),
-`elapsed_seconds`, `turns_remaining_at_stop`.
+`elapsed_seconds`, `turns_remaining_at_stop`, `gate_threshold` (the
+`run_limits.stop_early_gate_threshold` in effect for this stop; default `1.0`).
 
 ---
 
@@ -247,6 +298,12 @@ respectively), checked after each completed agent turn — see
 - `TokenUsage.total_tokens` is not serialized; sum the buckets (or use the computed
   `input_tokens` + `output_tokens` + cache buckets).
 - `EarlyStopInfo` presence is itself the "stopped early" signal.
+- `total_cost_usd` is the whole bill (agent + judge + simulator) at both row and run
+  level; `agent_cost_usd` is the agent-only slice. `TokenUsage.total_cost_usd` is a
+  different thing: the cost of those tokens, so always agent-only. `run_limits.max_usd`
+  gates on that one, since judge and simulator spend is not known mid-run.
+- A cost of `None` means unpriced, not free, and any total is a floor while
+  `cost_complete` is false — see [Missing cost is never fatal](#missing-cost-is-never-fatal).
 
 ## See also
 

{coder_eval-0.9.1 → coder_eval-0.9.3}/docs/TASK_DEFINITION_GUIDE.md

@@ -260,6 +260,7 @@ run_limits:
 | `count_cached_input` | `false` | — | Count `cache_read_input_tokens` toward the input/total budgets. Off by default — cached reads are typically free. |
 | `count_cache_creation` | `false` | — | Count `cache_creation_input_tokens` toward the input/total budgets. Off by default. |
 | `stop_early` | `false` | — | Opt-in master switch for early-stop-on-criterion. See [`stop_early`](#stop_early-opt-in-early-stop). |
+| `stop_early_gate_threshold` | `1.0` | `[0.0, 1.0]` (but `> 0.0` is enforced at resolution when `stop_early: true`) | Minimum weighted score over the armed subset required to gate as a pass. See [`stop_early`](#stop_early-opt-in-early-stop). |
 
 The authoritative source is `src/coder_eval/models/limits.py`. A lint rule (CE030) fails the build if
 a field defined there goes undocumented in this guide, so the table can't quietly fall behind the
@@ -376,11 +377,14 @@ Semantics:
   (e.g. `stop_when: pass` alongside a `max_count`, or `auto` on an instance that
   can decide neither) is likewise a hard error at resolution, not a silent full
   run.
-- **Verdict.** An early-stopped run is gated on the **armed subset only**; the
-  non-armed criteria become **advisory** and are clearly marked (report badge +
-  per-criterion note + `stopped_early` row). A run that completes naturally is
-  gated on the **full** set, as always. This is what lets one file serve both a
-  `smoke` flavor (`stop_early: true`) and an `e2e` flavor (`stop_early: false`) —
+- **Verdict.** Any task armed for early-stop (`stop_early: true`) is gated on
+  the **armed subset only** — the non-armed criteria become **advisory** and
+  are clearly marked (report badge + per-criterion note + `stopped_early`
+  row when the watcher actually fired) — whether or not the watcher actually
+  cut the run short; one task config maps to one gate semantic. Only a task
+  that never armed `stop_early` at all is gated on the **full** set, as
+  always. This is what lets one file serve both a `smoke` flavor
+  (`stop_early: true`) and an `e2e` flavor (`stop_early: false`) —
   see [AB_EXPERIMENTS.md](AB_EXPERIMENTS.md). Verdict parity between the flavors
   is one-sided: a **fail-stop** is verdict-preserving (the deferral above
   guarantees every pass-armed signal was allowed to resolve first), but a
@@ -390,6 +394,57 @@ Semantics:
   authoritative precision/recall belongs on the `stop_early: false` run.
 - **Fail-safe.** A live-verdict bug **fails open** to a full run (logged loudly) —
   it can never silently disable a criterion or cause a false early stop.
+- **Weighting.** `run_limits.stop_early_gate_threshold` (default `1.0`) is the
+  minimum weighted score (`Σ weight·score / Σ weight`, over the armed subset)
+  required to gate as a pass — both for the post-hoc verdict and for the live
+  stop rule itself. A fail-stop fires once the armed subset's **ceiling** (best
+  case: every still-undecided or already-passed criterion ends up scoring 1.0,
+  every live-failed one scores 0) can no longer reach the threshold — the gate
+  is mathematically guaranteed to fail regardless of how the trajectory
+  continues. A pass-stop fires once the pass-armed subset's **floor** (worst
+  case: every still-undecided one scores 0) already meets it. At the default
+  `1.0` both bounds collapse to the pre-weighting rules above exactly (any
+  single armed criterion's live-fail already drops the ceiling below 1.0, and
+  the floor only reaches 1.0 once every pass-armed criterion has actually
+  passed) — lowering it lets a low-weight armed criterion's failure be absorbed
+  without truncating the run, at the cost of the gate becoming a genuine
+  weighted average rather than a strict AND. **The armed weighted gate applies
+  whenever `stop_early: true` is set — one task config, one gate semantic —
+  regardless of whether the watcher actually fired a stop.** A task armed for
+  early-stop that instead completes naturally (the agent finishes, or
+  `max_turns` is hit, before the bound ever trips) is gated on the *same*
+  weighted armed-subset formula as an actual early stop, not the full-run
+  `all_criteria_passed`; only a task that never armed `stop_early` at all uses
+  the strict full-set gate. Each armed criterion's own `pass_threshold` still
+  decides whether it individually passed (converted to a binary 1.0/0.0
+  before weighting) — only the combination rule (weighted average vs strict
+  AND) changes, which is what makes the `gate_threshold=1.0` default an exact
+  equivalence with the pre-weighting `all(...)` rule.
+- **Decision-step budget.** `max_steps_to_decide` (per armed criterion, only
+  on `skill_triggered` / `command_executed`, requires `stop_when`) caps how
+  many tool-call steps that criterion may spend still **undecided** before the
+  run gives up on it:
+
+  ```yaml
+  success_criteria:
+    - type: skill_triggered
+      description: "date-teller must activate within 5 steps"
+      skill_name: date-teller
+      expected_skill: date-teller
+      stop_when: pass
+      max_steps_to_decide: 5
+  ```
+
+  Once the cap is exceeded (checked AFTER the normal fail-/pass-stop checks
+  each round, so a criterion that decides on that very step is never
+  penalized), the watcher fires `reason: decision_budget_exceeded` and the run
+  is forced to `FinalStatus.FAILURE` outright — bypassing
+  `stop_early_gate_threshold`'s weighted gate entirely, since a criterion that
+  never reached a verdict has nothing meaningful to weigh against the others.
+  `None` (default) = no cap; the run relies solely on `run_limits.max_turns`.
+  The step count is **cumulative across every retry attempt** of the turn —
+  including an attempt that crashed or timed out before this criterion's own
+  investigation even began — so size the budget with that headroom in mind.
 
 Observability (every early-stopped run is flagged everywhere so analysis never
 compares a truncated run against a full one):
@@ -535,6 +590,7 @@ All criteria share these fields:
 | `weight` | 1.0 | Relative importance for weighted score. `0` = **informational**: excluded from both the score and the pass/fail gate |
 | `pass_threshold` | 0.9 | Minimum score (0.0–1.0) to pass |
 | `stop_when` | `null` | Arms this criterion for early stop (`pass`/`fail`/`decided`/`auto`); requires `run_limits.stop_early: true` and an observable criterion type (`skill_triggered`, `command_executed`). `auto` arms whichever polarity this instance can decide (for dataset-fanned criteria whose positive/distractor role flips per row). See [`stop_early`](#stop_early-opt-in-early-stop). |
+| `max_steps_to_decide` | `null` | **Only on live-observable criteria** (`skill_triggered`, `command_executed`) — requires `stop_when` to be set. Caps the tool-call steps this armed criterion may spend still undecided before the run gives up and force-fails. See [`stop_early`](#stop_early-opt-in-early-stop). |
 
 **Scoring types:**
 - **Binary** (1.0 or 0.0): `file_exists`, `run_command`, `file_matches_regex`, `classification_match`, `skill_triggered`

coder_eval-0.9.3/evalboard/app/_components/__tests__/harness-badge.test.tsx

@@ -0,0 +1,45 @@
+import { describe, expect, test } from "vitest";
+import { render, screen } from "@testing-library/react";
+import { KNOWN_HARNESSES } from "@/lib/harness";
+import { HarnessBadge, harnessShortLabel } from "../harness-badge";
+
+describe("harnessShortLabel", () => {
+    test("every known harness has a human label", () => {
+        // A missing entry falls through to the raw id, which would put
+        // "delegate-sdk" in a legend next to "Codex" and "Antigravity".
+        for (const h of KNOWN_HARNESSES) {
+            expect(harnessShortLabel(h)).not.toBe(h);
+        }
+    });
+
+    test("the UiPath harness reads as Delegate", () => {
+        // The run data's id stays `delegate-sdk` (it's the registered
+        // `agent.type`); only the label people read is the short one.
+        expect(harnessShortLabel("delegate-sdk")).toBe("Delegate");
+    });
+
+    test("an unknown harness falls back to its id rather than a wrong name", () => {
+        expect(harnessShortLabel("some-new-agent")).toBe("some-new-agent");
+    });
+});
+
+describe("HarnessBadge", () => {
+    test("names the vendor in the alt text, not just the product", () => {
+        render(<HarnessBadge harness="delegate-sdk" />);
+        expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument();
+    });
+
+    test("renders the id as text when there is no logo for it", () => {
+        // Better a raw id than another vendor's mark on someone else's run.
+        render(<HarnessBadge harness="some-new-agent" />);
+        expect(screen.getByText("some-new-agent")).toBeInTheDocument();
+    });
+
+    test("takes a size so the chart legend can sit inside 11px text", () => {
+        render(<HarnessBadge harness="codex" size={14} />);
+        expect(screen.getByAltText("Codex · OpenAI")).toHaveAttribute(
+            "width",
+            "14",
+        );
+    });
+});