coder-eval 0.10.2__tar.gz → 0.11.0__tar.gz

{coder_eval-0.10.2 → coder_eval-0.11.0}/.github/workflows/pr-checks.yml

@@ -96,6 +96,15 @@ jobs:
       - name: Type check with pyright
         run: .venv/bin/pyright
 
+      # The CE036 contract engine lives under tests/, which [tool.pyright] excludes
+      # -- and `exclude` beats both a CLI file arg and an `include` entry, so it can
+      # only be reached through a config of its own, derived from [tool.pyright] so
+      # the two passes cannot drift. Mirrors `make typecheck`.
+      - name: Type check the CE036 contract engine
+        run: |
+          .venv/bin/python -m tests.lint.pyright_config .pyright-tests.json
+          .venv/bin/pyright -p .pyright-tests.json
+
       # PHASE 3: Security scanning
       - name: Security - Dependency vulnerabilities (pip-audit)
         run: .venv/bin/pip-audit --desc --skip-editable --ignore-vuln CVE-2026-4539 --ignore-vuln CVE-2026-3219 --ignore-vuln PYSEC-2025-183 # pygments 2.19.2 ReDoS + pip 26.0.1 tar/ZIP ambiguity + pyjwt 2.12.1 weak-encryption (disputed by supplier; key length is application-chosen); no fixes available on PyPI yet — revisit quarterly
@@ -387,6 +396,11 @@ jobs:
       - name: Type check with pyright
         run: .venv/Scripts/pyright
 
+      - name: Type check the CE036 contract engine
+        run: |
+          .venv/Scripts/python -m tests.lint.pyright_config .pyright-tests.json
+          .venv/Scripts/pyright -p .pyright-tests.json
+
       - name: Run test suite
         run: .venv/Scripts/pytest tests/ -v -m "not live and not lint"
 
@@ -438,7 +452,10 @@ jobs:
   e2e-smoke:
     name: E2E Smoke Tests (Real API)
     runs-on: uipath-ubuntu-latest
-    timeout-minutes: 10
+    # 15 (was 10): the bucket now includes anti_cheat_reference, a driver: docker
+    # task that spins its own container on top of the two image builds this job
+    # already does. Headroom, not an expected duration.
+    timeout-minutes: 15
     # Skip on fork PRs where secrets aren't available
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
 
@@ -452,12 +469,16 @@ jobs:
       AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
       AWS_REGION: ${{ secrets.AWS_REGION }}
       BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
-      # tasks_run for --tags smoke-pass. 6 task files (hello_date, dataset_example,
-      # smoke_llm_judge, smoke_agent_judge, byod_smoke_test, agentless_smoke_test);
-      # dataset_example fans out to 2 inline rows, so 7 sub-tasks. If you add/remove a
-      # smoke-pass task or change the dataset row count, bump these.
-      EXPECTED_SMOKE_PASS_RUN: "7"
-      EXPECTED_SMOKE_PASS_SUCCEEDED: "7"
+      # tasks_run for --tags smoke-pass. 7 task files (hello_date, dataset_example,
+      # smoke_llm_judge, smoke_agent_judge, byod_smoke_test, agentless_smoke_test,
+      # anti_cheat_reference); dataset_example fans out to 2 inline rows, so 8
+      # sub-tasks. If you add/remove a smoke-pass task or change the dataset row
+      # count, bump these.
+      #
+      # anti_cheat_reference lives in a SUBDIRECTORY, which `tasks/*.yaml` does not
+      # match — the smoke-pass step names its path explicitly. Keep that in sync.
+      EXPECTED_SMOKE_PASS_RUN: "8"
+      EXPECTED_SMOKE_PASS_SUCCEEDED: "8"
       # smoke-fail bucket: three tasks expected to fail.
       #   1. smoke_negative_path: file_contains criterion is unsatisfiable
       #      (sentinel-string regression detection for success-checker).
@@ -524,9 +545,13 @@ jobs:
       # that Bedrock rejects with 400 (no such cross-region profile). Falling
       # back to BEDROCK_MODEL — a valid pre-formatted Bedrock profile id — is
       # the same pattern live-tests uses (see test_claude_settings_enforcement_live._model_for_env).
+      # `tasks/*.yaml` is NOT recursive, so subdirectory tasks are listed
+      # explicitly. anti_cheat_reference is the adversarial probe that the agent
+      # cannot read the reference solution during its turn; it needs the
+      # coder-eval-agent image built above (it is a driver: docker task).
       - name: Run smoke-pass bucket (expect all to succeed)
         run: |
-          .venv/bin/coder-eval run tasks/*.yaml \
+          .venv/bin/coder-eval run tasks/*.yaml tasks/anti_cheat_reference/*.yaml \
             --tags smoke-pass \
             --run-dir runs/ci-smoke-pass
 

{coder_eval-0.10.2 → coder_eval-0.11.0}/.gitignore

@@ -69,3 +69,6 @@ refs/
 
 # SkillsBench tasks for testing
 /resources/
+
+# Derived pyright config for the CE036 contract engine (tests/lint/pyright_config.py)
+.pyright-tests.json

{coder_eval-0.10.2 → coder_eval-0.11.0}/CHANGELOG.md

@@ -2,6 +2,49 @@
 
 <!-- version list -->
 
+## v0.11.0 (2026-08-19)
+
+### Bug Fixes
+
+- **docker**: Restore container access after the DAC cap drop, shield the task dir
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+- **reference**: Address code review and CodeQL findings
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+- **reference**: Address PR review — scoring correctness, fail-closed anti-cheat
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+- **reference**: Clear remaining CodeQL alerts
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+### Documentation
+
+- **reference**: Record why READ_ONLY_MODE exists, and what is left to wire
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+### Features
+
+- **reference**: Directory-only references + anti-cheat permission window
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+### Testing
+
+- **early-stop**: CE036 enforces the live_verdict determinism + monotonicity contract
+  ([#126](https://github.com/UiPath/coder_eval/pull/126),
+  [`d854004`](https://github.com/UiPath/coder_eval/commit/d854004a5ec4fce0dab16d17d809a93c448fa022))
+
+- **reference**: Skip host-side chmod assertions on Windows
+  ([#106](https://github.com/UiPath/coder_eval/pull/106),
+  [`56bffad`](https://github.com/UiPath/coder_eval/commit/56bffad6acc4c0b32357bd25334bedd403c2f0d0))
+
+
 ## v0.10.2 (2026-08-18)
 
 ### Continuous Integration

{coder_eval-0.10.2 → coder_eval-0.11.0}/CLAUDE.md

@@ -24,6 +24,7 @@ coder_eval/
 ├── analysis.py                    # Command statistics aggregation
 ├── logging_config.py              # Structured logging setup
 ├── path_utils.py                  # Run ID generation, path utilities
+├── fs_permissions.py              # set_permissions: stacked chmod window (via Sandbox.set_permissions)
 ├── pricing.py                     # Model pricing / cost calculation (ModelPricing, calculate_cost, register_pricing)
 ├── litellm_cost.py                # Join proxy-captured ACTUAL per-call cost/cache onto turns (LiteLLM backend; apply_actual_cost)
 ├── utils.py                       # Version info helpers
@@ -82,7 +83,7 @@ coder_eval/
 │   ├── batch.py                   # Parallel task execution (run_batch + run_batch_resolved)
 │   ├── config.py                  # Batch run configuration
 │   ├── early_stop.py              # validate_early_stop guardrails + EarlyStopWatcher (armed live-verdict observer)
-│   ├── evaluation.py              # Evaluation helpers
+│   ├── evaluation.py              # Reference dir resolution + per-run private staging
 │   ├── experiment.py              # ExperimentRunner, resolve_task_for_variant, load_experiment
 │   └── task_loader.py             # YAML task loading
 │
@@ -142,6 +143,7 @@ action.yml                         # Published composite GitHub Action (coder-ev
 - **Per-criterion aggregation**: Each `BaseCriterion` subclass exposes `aggregate(criterion, per_row_results) -> CriterionAggregate | None`. Default emits `count / mean / median / std / min / max` so every criterion is suite-thresholdable for free. Classification-style criteria return `ClassificationCriterionResult` (subclass of `CriterionResult`) and layer accuracy / P/R/F1 / confusion via the shared `overlay_classification_metrics` utility. `BaseSuccessCriterion.suite_thresholds` gates the suite on those metrics; CLI exits non-zero on any gate failure.
 - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection.
 - **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports.
+- **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half.
 - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), and `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it). Full table + rationale: docs/agents/HARNESS_PARITY.md.
 - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly.
 - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express.
@@ -214,7 +216,9 @@ make plugin-reference  # the plugin's bundled criteria reference from the models
 
 Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction.
 
-When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033, CE034, CE035 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE035 resolves every `steps.<id>.outputs.<key>` / `needs.<job>.outputs.<key>` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/<slug>` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.)
+Recent additions, each traceable to a shipped defect: **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's).
+
+When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps.<id>.outputs.<key>` / `needs.<job>.outputs.<key>` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/<slug>` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.)
 
 Adding a user-facing field to one of the models CE030 tracks (`TaskDefinition`, `RunLimits`, `Dataset`, `SimulationConfig` — see `tests/lint/doc_schema_parity.py`) means documenting it in its guide (mention the field name as inline code) or adding an `EXEMPT` entry with a reason it is not user-authored. `make lint` fails otherwise.
 

{coder_eval-0.10.2 → coder_eval-0.11.0}/Makefile

@@ -38,6 +38,15 @@ plugin-reference:  ## Regenerate the plugin's bundled criteria reference from th
 
 typecheck:  ## Run type checking with pyright
 	uv run pyright
+	# The CE036 contract engine executes checker code and feeds the early-stop
+	# design; it is the one tests/ surface worth type-checking. It needs its own
+	# config: pyproject.toml excludes "tests", and pyright's `exclude` beats BOTH
+	# an explicitly-passed CLI file arg AND an `include` entry naming the file --
+	# either shortcut analyzes ZERO files and exits 0, a gate that checks nothing.
+	# The config below is DERIVED from [tool.pyright] (same rules, only
+	# include/exclude swapped), so the two passes cannot drift apart.
+	uv run python -m tests.lint.pyright_config .pyright-tests.json
+	uv run pyright -p .pyright-tests.json
 
 test:  ## Run test suite (excludes live + lint tests; run `make lint` for those)
 	uv run pytest -n auto -m "not live and not lint" tests/
@@ -86,13 +95,21 @@ clean:  ## Clean build artifacts and cache
 	rm -rf runs/2025-* runs/latest
 	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
 
+# Task globs. `tasks/*.yaml` does NOT recurse, so every subdirectory holding a
+# task must be listed. Kept identical to the CI e2e-smoke job's globs
+# (.github/workflows/pr-checks.yml) and pinned there by
+# tests/test_tags.py::TestCiSmokePassContract -- when the two drifted, `make
+# test-smoke` silently skipped tasks CI was gating on.
+TASK_GLOBS := tasks/*.yaml tasks/agents/*.yaml tasks/anti_cheat_reference/*.yaml
+SMOKE_GLOBS := tasks/*.yaml tasks/anti_cheat_reference/*.yaml
+
 run:	## Run coder-eval on all tasks with 8 parallel jobs
-	uv run coder-eval run tasks/*.yaml tasks/agents/*.yaml -j 8
+	uv run coder-eval run $(TASK_GLOBS) -j 8
 
 test-smoke:  ## Run e2e smoke tests with real API (mirrors CI "E2E Smoke Tests" job)
-	uv run coder-eval run tasks/*.yaml --tags smoke-pass --model claude-haiku-4-5-20251001
+	uv run coder-eval run $(SMOKE_GLOBS) --tags smoke-pass --model claude-haiku-4-5-20251001
 	@echo "--- now running smoke-fail bucket (expected to exit non-zero) ---"
-	! uv run coder-eval run tasks/*.yaml --tags smoke-fail --model claude-haiku-4-5-20251001
+	! uv run coder-eval run $(SMOKE_GLOBS) --tags smoke-fail --model claude-haiku-4-5-20251001
 
 docker-image:  ## Build the coder-eval-agent image (core + both agents baked in; no creds needed)
 	@VERSION=$$($(VERSION_CMD)); \

{coder_eval-0.10.2 → coder_eval-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coder-eval
-Version: 0.10.2
+Version: 0.11.0
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval

{coder_eval-0.10.2 → coder_eval-0.11.0}/action.yml

@@ -41,7 +41,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.10.2"   # <-- kept in sync with releases by release.yml
+    default: "0.11.0"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

coder_eval-0.11.0/comparison.md

@@ -0,0 +1,171 @@
+# Closing the grading-material leak: two approaches compared
+
+> **Scope note.** This is a design-decision document about *sandbox isolation*. It is unrelated to
+> `docs/comparison.md`, which compares coder_eval to other eval frameworks (SWE-bench, Harbor, …).
+
+- **Approach A** — [PR #88](https://github.com/UiPath/coder_eval/pull/88),
+  `fix/docker-isolation-copy-prune`: **COPY/PRUNE + GRADE-OUTSIDE**. Open, draft, 8 commits,
+  +5,754 / −214 across 35 files. Implemented and verified.
+- **Approach B** — **Declarative trust split + DAC permissions**: `input_dir` / `reference_dir` on
+  `TaskDefinition`, `$INPUT_DIR` / `$REFERENCE_DIR` substitution, agent runs as an unprivileged uid,
+  `reference_dir` is root-owned `0700`, `task.yaml` itself lives in `reference_dir`. A design, not
+  yet built.
+
+---
+
+## The problem
+
+When coder_eval grades an agent, the grading material sits on the same filesystem the agent is
+working in — **including under `--driver docker`, which is the mode users assume is safe**.
+
+On `main`, an agent in the container can reach all of the following:
+
+| Surface | Where it lives today |
+|---|---|
+| Full `success_criteria` | `/work/input/task.yaml`, staged verbatim (`docker_runner.py:605-620`) |
+| Raw task YAML text | `context.json.source_yaml` |
+| Reference solution | `reference.file` / `reference.directory`, auto-mounted (`:1253-1254`) |
+| Task-dir graders (`check_*.py`, `RESOLUTION.md`, reference agents) | host task dir bind-mounted at its own host path (`:1176`) |
+| Signposts pointing at all of the above | `TASK_DIR` / `SKILLS_REPO_PATH` in the agent's env |
+
+The agent and the harness run as the **same user** in the container — both root, since neither
+`docker/Dockerfile` nor `docker/Dockerfile.runtime` sets `USER` and `_build_argv` passes no
+`--user`. So there is no permission barrier of any kind: a single `grep -r` reaches the answer key.
+
+**This is measured, not theoretical.** PR #88 reports a nightly audit in which **~2.4% of replicates
+read the suite instead of solving the task**, and an adversarial "check `/work`" prompt drives that
+to **~100%**.
+
+A separate but adjacent defect: `~/.uipath` (including `.auth`) was mounted **read-write from the
+host original**, so an agent could overwrite operator credentials.
+
+The root cause is structural: **the harness lives inside the sandbox it is supposed to be isolating
+from.** The in-container process runs the agent *and* grades it (`run_task_internal_command.py:190-209`),
+so the rubric has to be shipped into the container at `t=0`.
+
+---
+
+## Approach A — COPY/PRUNE + GRADE-OUTSIDE (PR #88)
+
+**Closes the leak by absence, not by permission.** The container never contains grading material.
+
+Three coordinated moves:
+
+1. **COPY/PRUNE the agent's inputs.** The staged `task.yaml` is criteria-stripped via
+   `agent_safe_dump()` (`success_criteria: []`, `reference: null`) and `context.json.source_yaml` is
+   nulled. Plugins are projected to a *sanitized bundle copy* (`project_plugin_for_agent`, allowlist
+   `PLUGIN_AGENT_ALLOWED_SUBDIRS` = `skills`/`commands`/`agents`/`hooks`/`.claude-plugin`) mounted
+   `:ro` at `/work/skills`. The raw checkout, the reference, and the host task dir are **not mounted
+   at all**. A **grader-dir overlap guard** hard-errors any `template_sources` / `system_prompt_file`
+   / `extra_mounts` whose source contains the task dir.
+2. **GRADE OUTSIDE.** The container runs the agent only. After it exits, the host grades the
+   copied-out artifacts via `regrade_on_host`, using the full unstripped `TaskDefinition` it still
+   holds, with `TASK_DIR` pointing at the real host task dir. Re-grade is restricted to an allowlist
+   of final statuses (`SUCCESS` / `FAILURE` / `MAX_TURNS_EXHAUSTED`).
+3. **Copy-then-mount host state.** `~/.uipath` and `~/.claude` are forwarded as throwaway rw
+   **copies**; `~/.claude/jobs` (operator session timelines) is denylisted.
+
+Plus **harness-outside**: `pre_run`/`post_run` invoke helper scripts under the skills-repo `tests/`
+tree, which is no longer mounted. `post_run` becomes host-only; `pre_run` gains a per-command
+`runs_in: host|agent` (default `host`, seeding the workspace through a `:ro` `/work/seed` staging
+mount). The PR body flags this as a known blocker affecting **243 of 1,147 downstream tasks (~21%)**;
+the branch implements the fix, with only the ~7 `uv sync` tasks needing `runs_in: agent`.
+
+**Verification shipped with it:** Detector A (host-unchanged — daemon-less mount-shape proxy in CI
+plus an `-m live` byte-and-metadata-identical real run), Detector B (zero-grading-material in the
+agent mount view, with a positive control so "staged nothing" can't pass vacuously), a baked-image
+answer scan, and live confirmation that an adversarial "read your grader" task fails on gpt / claude
+/ gemini / kimi.
+
+**Stated costs:** early stop is **disabled under docker** (criteria are gone, so the in-container
+watcher can't arm — a warning is logged, verdict unaffected), and the design doc states plainly that
+**allowlist-by-absence has no DAC backstop** — if the prune boundary is wrong, nothing else stops the
+read.
+
+## Approach B — Declarative trust split + DAC permissions
+
+**Closes the leak by permission, with an author-declared boundary.**
+
+- `TaskDefinition` gains `input_dir` (agent-visible fixtures) and `reference_dir` (answer key),
+  substituted as `$INPUT_DIR` / `$REFERENCE_DIR` inside the task file.
+- `task.yaml` itself lives in `reference_dir`, so the rubric is protected by the same single rule as
+  everything else — no special case.
+- In the container, the agent runs as an unprivileged uid; `reference_dir` is root-owned `0700`;
+  `input_dir` and the workspace are chowned to the agent uid.
+- Effectively this **splits the existing `$TASK_DIR` token into its two halves by trust level** —
+  which is precisely why the leak exists today, since one token points at a directory mixing both.
+
+Refinements this needs to be sound:
+
+- **Scope-aware substitution.** `$REFERENCE_DIR` must be a hard validation error in agent-facing
+  fields (`initial_prompt`, `system_prompt`, mutations) — resolving it there advertises the path and
+  leaks it into logs and reports. `$INPUT_DIR` substitutes anywhere.
+- **Copy, don't mount, `reference_dir`.** The host bind mount is `:ro`, so `chmod` fails `EROFS`;
+  making it writable would mutate the user's real repo permissions with no crash-safe restore. Stage
+  a container-local root-owned copy — which also lets the symmetric same-host-path mount disappear.
+- **Static, not scoped, permissions.** A `with block_access(): …` context manager that flips modes
+  on entry and restores on exit is not crash-safe (SIGKILL between enter and exit leaves them
+  flipped). With two identities, the perms are simply set once at stage time and never restored.
+- **Dataset files default to `reference_dir`** — for activation suites the JSONL rows carry
+  `expected_skill`, which *is* the answer key.
+- The `~/.claude` credential copy must stay agent-writable, so `0700`-root cannot be applied
+  blanket — it is per-path.
+
+---
+
+## Pros and cons
+
+| | Pros | Cons |
+|---|---|---|
+| **A — COPY/PRUNE + GRADE-OUTSIDE** (PR #88) | Closes the leak **even against a root agent** — no identity work needed. Fails safe: a file that isn't there can't be read. Also closes surfaces B doesn't touch: host-credential overwrite (`~/.uipath` rw), `~/.claude/jobs` exposure, env signposts, baked-image content. **Built, tested, and empirically verified** (detectors in CI + live adversarial probe across four models). Requires **no per-task authoring changes** for the leak fix itself. Removes the harness from the sandbox — the structural root cause. | **Early stop disabled under docker** — a shipped feature silently degrades on one driver. Forced `pre_run`/`post_run` host-side, touching **~21% of downstream tasks** and adding a new `runs_in` concept. **No DAC backstop** (the PR says so): the prune allowlist is "a coder_eval-side guess about what is answer-free", so a plugin putting answers in `skills/` defeats it. Detector B catches known sentinels, not an unknown golden file. Large blast radius: **+5,754 LOC / 35 files**, restructuring `orchestrator.py`. Nothing protects `tempdir`. |
+| **B — Trust split + DAC permissions** | Supplies exactly the **DAC backstop A lacks** — a second, independent barrier. The boundary is **author-declared**, not framework-guessed, replacing A's residual risk #1 with an explicit contract. **Preserves early stop** (criteria stay in container memory, which the agent can't read) and **keeps `pre_run`/`post_run` in-container** (they run as root and can still read `reference_dir`) — no `runs_in`, no 243-task migration. Much smaller runtime change: a uid on spawn plus chowns at setup. Conceptually simple: one rule, "`reference_dir` is harness-only". | **Depends entirely on identity separation that doesn't exist yet** — both processes are root today, so `chmod 0700` is a *no-op* until the agent runs as a separate uid. Fails **open and silently**: one missed chown, one new mount, and there's no absence to fall back on. Real friction running the agent CLI unprivileged (HOME, npm cache, the `~/.claude` copy must stay writable). Pushes classification onto **1,147 downstream task authors** unless the default is "protected unless declared input". Does **not** address host-credential overwrite, baked-image content, or env signposts. **Also nothing for `tempdir`** (no second identity on the host). Unbuilt and unverified. |
+
+### Head-to-head
+
+| Axis | A — Absence | B — Permissions |
+|---|---|---|
+| Works against a root agent | ✅ | ❌ (requires uid split first) |
+| Failure mode | Fails safe (nothing to read) | Fails open (silently, if a chown is missed) |
+| Boundary defined by | Framework allowlist (a guess) | Task author (a declaration) |
+| Early stop under docker | ❌ disabled | ✅ preserved |
+| `pre_run` / `post_run` | Moved host-side; `runs_in` added; ~21% of tasks affected | Unchanged, in-container |
+| Host credential / `~/.claude/jobs` / baked image | ✅ covered | ❌ out of scope |
+| `tempdir` driver | ❌ | ❌ |
+| Per-task migration | None for the leak fix | `input_dir` / `reference_dir` across the suite |
+| Framework blast radius | +5,754 LOC, 35 files | Smaller runtime change; new schema + token rules |
+| Status | Implemented, CI-gated, live-verified | Design only |
+
+---
+
+## Assessment
+
+**These are not competing designs — B is the missing layer under A.**
+
+PR #88's own residual-risk list names its weakest point: *"Allowlist-by-absence has no DAC backstop
+… `PLUGIN_AGENT_ALLOWED_SUBDIRS` is a coder_eval-side guess about what is answer-free."* That is
+precisely the gap Approach B fills, in two independent ways:
+
+1. **As a mechanism** — a uid split gives the second barrier, so a prune-boundary miss is no longer
+   game over.
+2. **As a contract** — and this matters more. `input_dir`/`reference_dir` replaces the framework's
+   guess with an author declaration, which is the durable fix PR #88 already identifies as a
+   cross-repo follow-up ("push the agent-bundle boundary into the skills repo — a manifest declaring
+   the agent-safe surface"). Approach B *is* that manifest, expressed in the task schema.
+
+If only one ships, it should be **A**: it is built, measured against real leak rates, gated by
+detectors in CI, and it closes several surfaces B never touches. B's mechanism half is also blocked
+on identity work (non-root agent, credential-copy ownership) that A doesn't need.
+
+The sequencing that gets the most value:
+
+1. **Land A.** It is the structural fix — it removes the harness from the sandbox.
+2. **Adopt B's declarative half next**, as the durable replacement for the prune allowlist. It is
+   the higher-value part of B and is independent of the permission mechanism.
+3. **Add B's uid split as defense-in-depth** where material must remain in-container.
+4. **Revisit early stop.** A disables it under docker; the leak-free restoration is a host-side
+   watcher over the live event stream — the host already receives every `ToolStartEvent` /
+   `ToolEndEvent` with full `CommandTelemetry` (`streaming/wire.py`), and can already signal the
+   container through the heartbeat channel. Worth first confirming the feature earns its keep: every
+   `stop_early` usage in this repo is a fixture for testing early stop itself.
+5. **State plainly that `tempdir` is not cheat-resistant** under either approach. Neither has a
+   boundary there; it should be documented as a development driver.

{coder_eval-0.10.2 → coder_eval-0.11.0}/docs/DOCKER_ISOLATION.md

@@ -304,6 +304,39 @@ The host's run dir is bind-mounted **read-write** into the container at the same
 
 The host's `DockerRunner` (`coder_eval/isolation/docker_runner.py`) renders the `docker run` argv, bind-mounts task inputs at `/work/input`, allocates an output dir at `/work/output`, and tails container stdout into `docker.log` in the task's run dir.
 
+If the task declares a `reference:` block, a throwaway **copy** of its directory is bind-mounted **read-write** at `/work/references`, and an empty tmpfs is layered over its original location inside the task-directory mount so the agent cannot reach the solution through `$TASK_DIR`. The mount is read-write on purpose — a `:ro` bind mount cannot be chmod'd (EROFS), and the orchestrator holds this path at mode `000` for the duration of each agent turn. The container also runs with `--cap-drop DAC_OVERRIDE --cap-drop DAC_READ_SEARCH` (see the caveat below for why `FOWNER`/`CHOWN` are deliberately kept).
+
+> **What this does and does not guarantee.** The mode-000 window stops an agent
+> that *passively reads* the reference — `ls`, `cat`, `grep` all fail. Two
+> things it does **not** stop:
+>
+> 1. **A deliberate re-chmod.** `chmod(2)` is gated on *owner-or-`CAP_FOWNER`*,
+>    and the container runs as root, which owns the bind-mounted copy. Verified:
+>    `chmod 755 /work/references` succeeds from inside the container and the
+>    solution is then readable. Dropping `FOWNER`/`CHOWN` does **not** fix this
+>    and is deliberately not done — the in-container orchestrator that *applies*
+>    the window is the same root process with the same capabilities, so dropping
+>    `FOWNER` breaks the harness's own `chmod` wherever the bind mount preserves
+>    a non-root owner (native Linux). Verified: container root, uid-1000-owned
+>    directory, `FOWNER` dropped → `chmod: Operation not permitted`. The drop
+>    only bites on the hosts where it also disables the control.
+> 2. **Waiting the window out.** The window spans `agent.communicate`, so
+>    between turns and after the final one the reference is back at its normal
+>    mode, and nothing reaps the agent's child processes at turn end — a
+>    backgrounded read loop started during a turn succeeds once the window
+>    closes.
+>
+> The *write* half of (2) IS closed: the reference tree is hashed at staging
+> time and re-verified before grading, and a mismatch fails the run with
+> `ReferenceTamperedError` rather than scoring `reference_comparison` against a
+> file the agent wrote. **Closing the read half fully requires running the agent
+> as a non-root uid that does not own the reference, and holding the window for
+> the agent's whole lifetime** — tracked as follow-up. Treat the current control
+> as raising the cost of accidental leakage, not as a boundary against an
+> adversarial agent.
+
+See [Reference Solutions](TASK_DEFINITION_GUIDE.md#reference-solutions).
+
 Inside the container, the entrypoint invokes `coder-eval _run-task-internal` (hidden subcommand), which loads the staged YAML + context, runs the standard in-process Orchestrator (driver auto-coerced back to `tempdir`), and writes `task.json` to the output mount. Host reads it and feeds the existing aggregation pipeline.
 
 A `result_kind` discriminator on `CriterionResult` ensures `ClassificationCriterionResult` subclasses survive the JSON round-trip — without it, host-side aggregation would silently lose `observed_label`/`expected_label`.

{coder_eval-0.10.2 → coder_eval-0.11.0}/docs/EXTENDING.md

@@ -161,8 +161,11 @@ from coder_eval.models import CriterionResult
 class MyChecker(BaseCriterion[MyCriterion]):
     criterion_type = "my_criterion"          # must match the model discriminator
 
-    def _check_impl(self, criterion, sandbox, reference_code=None, *,
+    def _check_impl(self, criterion, sandbox, *,
                     turn_records=None, context=None) -> CriterionResult:
+        # `context` carries the live run state: `context.route` (for criteria
+        # that call a model) and `context.reference_dir` (the staged reference
+        # copy, for criteria that grade against a reference solution).
         ok = ...  # your logic
         return CriterionResult(
             criterion_type=self.criterion_type,
@@ -220,6 +223,21 @@ Notes:
   LiveSuccessCriterion)` directly, no separate checker-side flag. A lint rule
   (`tests/test_custom_lint.py::TestCE025LiveVerdictConsistency`) keeps the
   model subclassing and the checker's `live_verdict` override paired.
+- Your `live_verdict` must be **deterministic** (a pure function of the
+  `turn_records` prefix — no wall-clock, randomness, or hidden instance state)
+  and **monotonic** (once it returns `"pass"`/`"fail"` for some prefix, every
+  longer prefix returns that same verdict) — `EarlyStopWatcher`'s verdict
+  latching and deferred stops silently depend on both. Lint rule CE036
+  (`tests/lint/live_verdict_contract.py`) enforces this by replaying each live
+  criterion against every prefix of recorded trajectories, and **fails until
+  you add `ContractCase` fixtures** for the new type in the same change,
+  reaching every polarity its instances claim via
+  `live_decidable_polarities()`. An out-of-tree plugin criterion is invisible
+  to CE036's union walk — and the module lives under `tests/`, which is not
+  shipped in the PyPI wheel — so copy the replay pattern (a `ContractCase`-style
+  fixture plus the prefix-by-prefix determinism/monotonicity walk) into your
+  plugin's own test suite, using `tests/lint/live_verdict_contract.py` in this
+  repo as the reference implementation.
 
 > A duplicate `criterion_type` **overwrites** the earlier checker with a warning (not
 > a hard error, unlike agents) — keep type strings unique.