coder-eval 0.11.0__tar.gz → 0.11.1__tar.gz

{coder_eval-0.11.0 → coder_eval-0.11.1}/.github/workflows/pr-checks.yml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -297,7 +297,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project without [uipath] extra
@@ -381,7 +381,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -527,7 +527,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -697,7 +697,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -840,7 +840,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (with codex extra)
@@ -919,7 +919,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)

{coder_eval-0.11.0 → coder_eval-0.11.1}/CHANGELOG.md

@@ -2,6 +2,35 @@
 
 <!-- version list -->
 
+## v0.11.1 (2026-08-21)
+
+### Bug Fixes
+
+- **ci**: Pin pip floor to 26.2 to close PYSEC-2026-3721
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **codex**: Store command output whole in result_summary (+ code-review fixes)
+  ([#127](https://github.com/UiPath/coder_eval/pull/127),
+  [`3d09f0f`](https://github.com/UiPath/coder_eval/commit/3d09f0f4d966f619405058aca44aa982b1ba265a))
+
+- **deps**: Address PR #133 review — anthropic 1.0.0 dropped temperature kwarg
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **deps**: Bump anthropic to 1.0.0 and migrate Bedrock judge path to httpx2
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **deps**: Bump anthropic to 1.0.0, migrate Bedrock judge path to httpx2
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **deps**: Re-lock pip to 26.2.1 to actually close PYSEC-2026-3721
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+
 ## v0.11.0 (2026-08-19)
 
 ### Bug Fixes

{coder_eval-0.11.0 → coder_eval-0.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coder-eval
-Version: 0.11.0
+Version: 0.11.1
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -23,11 +23,12 @@ Classifier: Topic :: Software Development :: Quality Assurance
 Classifier: Topic :: Software Development :: Testing
 Classifier: Typing :: Typed
 Requires-Python: >=3.13
-Requires-Dist: anthropic>=0.86.0
+Requires-Dist: anthropic<2.0.0,>=1.0.0
 Requires-Dist: anyio>=4.13.0
 Requires-Dist: azure-monitor-opentelemetry-exporter<1.1.0,>=1.0.0b30
 Requires-Dist: claude-agent-sdk>=0.2.124
 Requires-Dist: click>=8.3.3
+Requires-Dist: httpx2<3.0.0,>=2.12.0
 Requires-Dist: jmespath>=1.1.0
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: opentelemetry-sdk<2.0.0,>=1.30.0

{coder_eval-0.11.0 → coder_eval-0.11.1}/action.yml

@@ -41,7 +41,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.11.0"   # <-- kept in sync with releases by release.yml
+    default: "0.11.1"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

{coder_eval-0.11.0 → coder_eval-0.11.1}/experiments/default.yaml

@@ -18,9 +18,11 @@ defaults:
   # Per-row dataset multiplication: a 100-row task with max_usd: 0.10 permits up to
   # $10 cumulative spend.
   run_limits:
-    # Maximum agent inner-loop turns per iteration (null = SDK default).
-    # Typical range in tasks: 3-18 depending on complexity.
-    max_turns: 20
+    # Hard safety ceiling on agent inner-loop turns per iteration (null = SDK
+    # default). Set well above the typical 3-18 turns a task needs so it only
+    # trips on runaway loops; task_timeout / turn_timeout below are the practical
+    # guards. Override per-task when a task legitimately needs more.
+    max_turns: 100
     # Soft target: when cumulative SDK turns across all iterations of a task
     # exceed this, the orchestrator logs a one-shot warning and the report
     # surfaces a badge. Does NOT abort the run (max_turns is the hard cap).

{coder_eval-0.11.0 → coder_eval-0.11.1}/plugins/coder-eval/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "coder-eval",
-  "version": "0.11.0",
+  "version": "0.11.1",
   "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.",
   "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" },
   "homepage": "https://coder-eval.com",

{coder_eval-0.11.0 → coder_eval-0.11.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "coder-eval"
-version = "0.11.0"
+version = "0.11.1"
 description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites."
 readme = "README.md"
 license = "Apache-2.0"
@@ -34,7 +34,14 @@ dependencies = [
     "click>=8.3.3",
     "rich>=14.3.3",
     "python-dotenv>=1.2.2",
-    "anthropic>=0.86.0",
+    # Cap the major: 1.0.0 already proved a major can drop call kwargs
+    # (temperature/top_p/top_k off messages.create — see judge_anthropic.py).
+    "anthropic>=1.0.0,<2.0.0",
+    # anthropic>=1.0.0 migrated its HTTP layer from httpx to httpx2 (its own
+    # exceptions, e.g. APIConnectionError, now carry an httpx2.Request). Declared
+    # explicitly since our code/tests construct httpx2 types directly. Capped to
+    # mirror anthropic's own bound on it (`httpx2<3,>=2.0.0`).
+    "httpx2>=2.12.0,<3.0.0",
     "claude-agent-sdk>=0.2.124",
     "anyio>=4.13.0",
     "radon>=6.0.1",

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/__init__.py

@@ -1,3 +1,3 @@
 """coder_eval - A framework for evaluating AI coding agents."""
 
-__version__ = "0.11.0"
+__version__ = "0.11.1"

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/agents/codex_agent.py

@@ -2178,10 +2178,15 @@ class CodexAgent(Agent[CodexAgentConfig]):
             # Determine result status from exit code
             result_status = "success" if exit_code == 0 else "error" if exit_code is not None else "unknown"
 
-            # Build result summary with output if available
+            # Build result summary with output if available. Store the output WHOLE:
+            # CommandTelemetry.result_summary is the untruncated tool-result body (its
+            # length drives CommandTelemetry.result_tokens), so truncating here would
+            # under-report tool-output size for every command (see CE043). The output is
+            # already bounded by the Codex harness's own exec-output truncation; any
+            # further trimming for display belongs in the renderers/reports, not capture.
             summary_parts = [f"Exit code: {exit_code}" if exit_code is not None else "Command executed"]
             if output and len(output.strip()) > 0:
-                summary_parts.append(f"Output: {output[:100]}")
+                summary_parts.append(f"Output: {output}")
             result_summary = " | ".join(summary_parts)
 
             # Try to parse output as JSON

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/evaluation/judge_anthropic.py

@@ -40,7 +40,14 @@ async def invoke_anthropic_judge_async(
     Returns the SDK response converted to a dict via ``model_dump`` so the
     caller can reuse ``extract_verdict_from_anthropic_response`` — Anthropic's
     native shape (content blocks with ``type: tool_use``) is identical
-    between this SDK call and the Bedrock httpx-direct call.
+    between this SDK call and the Bedrock httpx2-direct call.
+
+    ``temperature`` is forwarded via ``extra_body`` rather than as a top-level
+    kwarg: anthropic 1.0.0 dropped ``temperature``/``top_p``/``top_k`` from
+    ``AsyncMessages.create``'s typed signature, but the Messages API itself
+    still accepts ``temperature`` in the raw JSON body — the same body shape
+    the Bedrock path already sends it in — so ``extra_body`` keeps both judge
+    backends honoring ``LLMJudgeCriterion.temperature`` identically.
     """
     alias = to_anthropic_alias(model)
     client = AsyncAnthropic(timeout=timeout_seconds)
@@ -51,12 +58,17 @@ async def invoke_anthropic_judge_async(
                 system=system,
                 messages=[{"role": "user", "content": user}],
                 max_tokens=max_tokens,
-                temperature=temperature,
-                tools=[tool_spec],  # type: ignore[arg-type]
+                tools=[tool_spec],  # pyright: ignore[reportArgumentType]
                 tool_choice={"type": "tool", "name": tool_spec["name"]},
+                extra_body={"temperature": temperature},
             )
         except APIError as e:
             # The SDK already retries transient failures internally (2 attempts
             # by default) — do not add another retry loop here.
             raise JudgeInfrastructureError(f"Anthropic judge API error: {e}") from e
+        except Exception as e:
+            # A signature/contract break (e.g. a removed or renamed kwarg after an
+            # SDK bump) must not be scored as an agent failure — see CLAUDE.md's
+            # CE039 rationale: an eval-infra fault is not the agent's fault.
+            raise JudgeInfrastructureError(f"Anthropic judge call failed: {e}") from e
     return response.model_dump()

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/evaluation/judge_bedrock.py

@@ -14,7 +14,7 @@ agent_judge uses the Claude Code SDK subprocess instead — the two paths
 intentionally do not share an HTTP client.
 
 Async on purpose: this is llm_judge's only implementation of the network
-call (there is no sync twin) — ``httpx.AsyncClient`` lets the call yield the
+call (there is no sync twin) — ``httpx2.AsyncClient`` lets the call yield the
 event loop instead of blocking a thread-pool thread for the wait, so
 ``SuccessChecker.check_all_async`` awaits it directly without pinning a
 thread. (``check_all_async`` currently runs criteria sequentially; running
@@ -27,7 +27,7 @@ import asyncio
 import logging
 from typing import Any
 
-import httpx
+import httpx2
 
 from coder_eval.errors import JudgeInfrastructureError
 from coder_eval.errors.categories import RetryConfig
@@ -90,13 +90,13 @@ async def invoke_bedrock_judge_async(
     attempts = _JUDGE_RETRY.max_retries + 1
     last_failure = ""
     last_exc: Exception | None = None
-    async with httpx.AsyncClient() as client:
+    async with httpx2.AsyncClient() as client:
         for attempt in range(attempts):
             if attempt:
                 await asyncio.sleep(compute_backoff(_JUDGE_RETRY, attempt - 1))
             try:
                 response = await client.post(url, headers=headers, json=body, timeout=timeout_seconds)
-            except httpx.HTTPError as e:
+            except httpx2.HTTPError as e:
                 last_failure = f"Bedrock invoke transport error: {e}"
                 last_exc = e
                 logger.warning("Bedrock judge attempt %d/%d failed: %s", attempt + 1, attempts, last_failure)

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/evaluation/verdict_tool.py

@@ -40,7 +40,7 @@ SUBMIT_VERDICT_ANTHROPIC_TOOL: dict[str, Any] = {
     "description": _SUBMIT_VERDICT_DESCRIPTION,
     "input_schema": JudgeVerdict.model_json_schema(),
 }
-"""Anthropic-native tool spec for the Bedrock httpx-direct path and Anthropic SDK calls.
+"""Anthropic-native tool spec for the Bedrock httpx2-direct path and Anthropic SDK calls.
 
 Note: Anthropic uses ``input_schema``, not OpenAI's ``parameters``.
 """
@@ -132,7 +132,7 @@ def extract_verdict_from_anthropic_response(
     ``{"type": "tool_use", "name": "submit_verdict"}`` and validates the last
     one's ``input`` against ``JudgeVerdict``. Used by:
 
-    * The Bedrock httpx path (``invoke_bedrock_judge_async``) — raw JSON dict.
+    * The Bedrock httpx2 path (``invoke_bedrock_judge_async``) — raw JSON dict.
     * The Anthropic SDK Direct path (``invoke_anthropic_judge_async``) —
       response converted via ``Message.model_dump()``.
 

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/models/telemetry.py

@@ -410,6 +410,16 @@ class CommandTelemetry(BaseModel):
         result size from prompt-cache growth (which is unavailable when caching is
         disabled). Approximate, not the API's exact tokenizer count, but
         deterministic and always present. 0 when the tool returned no content.
+
+        This measure is only meaningful while ``result_summary`` stays whole: an
+        agent that truncates a command's output before recording it (as the Codex
+        agent once did with ``output[:100]``) silently under-reports that command's
+        result. Lint rule CE043 forbids truncating captured command output
+        (stdout/stderr) in the agents, so the "untruncated" contract holds for
+        command results across agents. (One-line summaries of non-command tool
+        items — e.g. collab/MCP status lines built by ``_summarize_tool_item`` —
+        are intentionally brief and out of scope.) Trim for DISPLAY in the
+        renderers/reports instead.
         """
         if not self.result_summary:
             return 0

{coder_eval-0.11.0 → coder_eval-0.11.1}/src/coder_eval/streaming/renderers.py

@@ -104,14 +104,18 @@ class RichStreamRenderer:
             return f"[cyan]>>> TOOL: {escape(event.tool.tool_name)}[/cyan] | {params_str}"
 
         if isinstance(event, ToolEndEvent):
-            preview = escape(event.tool.result_summary or "")
+            # result_summary is stored WHOLE (untruncated) at capture; cap it here
+            # for the terse live feed so a large command output doesn't flood the
+            # console. The reported char count reflects the true (full) length.
+            full = event.tool.result_summary or ""
+            preview = escape(_truncate(full, _MAX_RESULT_LEN))
             if event.status == ToolEndStatus.OK:
                 tag = "[green]<<< OK[/green]"
             elif event.status == ToolEndStatus.UNRESOLVED:
                 tag = "[yellow]<<< UNRESOLVED:[/yellow]"
             else:
                 tag = f"[red]<<< {escape(event.status.value.upper())}:[/red]"
-            return f"{tag} ({len(preview)} chars) {preview}"
+            return f"{tag} ({len(full)} chars) {preview}"
 
         if isinstance(event, TextChunkEvent):
             return f"[dim]{escape(event.text)}[/dim]"

coder_eval-0.11.1/tests/lint/rules/ce043_no_command_output_truncation.py

@@ -0,0 +1,75 @@
+"""CE043: Agents must not truncate a command's output when recording it.
+
+``CommandTelemetry.result_summary`` is contractually the *untruncated* tool-result
+body, and its length drives the ``result_tokens`` computed field (the cost
+simulator's cache-independent measure of tool-output size). An agent that clips a
+command's captured output before storing it silently under-reports every tool
+result — exactly the bug the Codex agent shipped with (``f"Output: {output[:100]}"``),
+which pinned ~77% of its Bash results at ~31 tokens and skewed the cost model.
+
+This rule flags, inside ``src/coder_eval/agents/``, a constant-upper-bound slice
+(``x[:N]``) applied to a value that denotes captured command output:
+
+  * a name whose id is/ends with ``output`` / ``stdout`` / ``stderr``
+    (e.g. ``output[:100]``, ``aggregated_output[:512]``, ``proc_stdout[:80]``)
+  * an attribute access ``.aggregated_output`` / ``.output`` / ``.stdout`` / ``.stderr``
+    (e.g. ``command_item.aggregated_output[:100]``)
+
+Store the output whole (it is already bounded by the harness's own exec-output
+truncation) and trim for DISPLAY in the renderers/reports instead.
+
+Add ``# noqa: CE043`` on the offending line for a genuinely non-recorded use
+(e.g. slicing stdout only to build a short crash/log message that never becomes a
+``result_summary``), with a comment explaining why.
+"""
+
+import ast
+
+from tests.lint.rules.base import BaseRule
+
+
+_OUTPUT_NAMES = {"output", "stdout", "stderr", "aggregated_output"}
+
+
+def _denotes_output(value: ast.AST) -> str | None:
+    """Return the output-ish identifier being sliced, or None."""
+    if isinstance(value, ast.Name):
+        low = value.id.lower()
+        if low in _OUTPUT_NAMES or low.endswith("_output") or low.endswith("_stdout") or low.endswith("_stderr"):
+            return value.id
+    elif isinstance(value, ast.Attribute):
+        if value.attr.lower() in _OUTPUT_NAMES:
+            return value.attr
+    return None
+
+
+def _is_const_upper_slice(sl: ast.AST) -> bool:
+    """True for ``[:N]`` / ``[:N:...]`` with a constant int upper bound."""
+    return (
+        isinstance(sl, ast.Slice)
+        and sl.lower is None
+        and isinstance(sl.upper, ast.Constant)
+        and isinstance(sl.upper.value, int)
+    )
+
+
+class NoCommandOutputTruncation(BaseRule):
+    id = "CE043"
+
+    def check(self, tree: ast.AST) -> list:
+        # Scope to the agent implementations — only they record CommandTelemetry.
+        if "/agents/" not in self.filepath.replace("\\", "/"):
+            return []
+        self.visit(tree)
+        return self.violations
+
+    def visit_Subscript(self, node: ast.Subscript) -> None:
+        name = _denotes_output(node.value)
+        if name is not None and _is_const_upper_slice(node.slice):
+            self.violation(
+                node,
+                f"Do not truncate command output ({name}[:...]); CommandTelemetry.result_summary "
+                "must stay whole (it drives result_tokens). Store the full output and trim for "
+                "display in the renderers/reports instead.",
+            )
+        self.generic_visit(node)

{coder_eval-0.11.0 → coder_eval-0.11.1}/tests/lint/runner.py

@@ -24,6 +24,7 @@ from tests.lint.rules.ce032_criteria_path_seam import CriteriaPathSeam
 from tests.lint.rules.ce037_no_dead_private_helper import NoDeadPrivateHelper
 from tests.lint.rules.ce038_acquire_inside_try import AcquireInsideTry
 from tests.lint.rules.ce039_config_error_escalates import ConfigErrorEscalates
+from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation
 from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess
 from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync
 from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore
@@ -71,6 +72,7 @@ ALL_RULES: list[RuleClass] = [
     NoDeadPrivateHelper,
     AcquireInsideTry,
     ConfigErrorEscalates,
+    NoCommandOutputTruncation,
 ]
 
 # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule

{coder_eval-0.11.0 → coder_eval-0.11.1}/tests/test_codex_agent_unit.py

@@ -104,3 +104,28 @@ class TestCodexTurnState:
         # A tool_use block was recorded into the open buffer (cut at the next
         # tokenUsage flush, not here), joinable to the command by tool_id.
         assert any(b.block_type == "tool_use" and b.tool_use_id == "c1" for b in state.open_blocks)
+
+    def test_command_output_recorded_whole_not_truncated(self):
+        # Regression for the Codex `output[:100]` bug (CE043): result_summary must
+        # carry the FULL command output so result_tokens reflects real tool-output
+        # size instead of being pinned at a ~31-token, 100-char cap.
+        agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5-codex"))
+        state, commands, _messages = self._state(agent)
+
+        big_output = "X" * 4000  # far beyond the old 100-char clip
+        cmd_root = SimpleNamespace(
+            type="commandExecution",
+            id="c2",
+            command="cat big.txt",
+            exit_code=0,
+            aggregated_output=big_output,
+            duration_ms=5,
+        )
+        state.on_item_started(_item_notification("item/started", cmd_root))
+        state.on_item_completed(_item_notification("item/completed", cmd_root))
+
+        cmd = commands[0]
+        assert big_output in (cmd.result_summary or ""), "full output must be recorded, not truncated"
+        # result_tokens (ceil(len/4)) must scale with the real output, not ~31.
+        assert cmd.result_tokens >= len(big_output) // 4
+        assert cmd.result_tokens > 100

{coder_eval-0.11.0 → coder_eval-0.11.1}/tests/test_custom_lint.py

@@ -60,6 +60,48 @@ class TestCE016NoComputedTokenUsageKwargs:
         assert not self._run("ReconciliationMessage(input_tokens=-5)")
 
 
+@pytest.mark.lint
+class TestCE043NoCommandOutputTruncation:
+    """CE043 flags truncation of captured command output inside agents/, only."""
+
+    @staticmethod
+    def _run(src: str, *, in_agents: bool = True):
+        import ast
+
+        from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation
+
+        path = "src/coder_eval/agents/codex_agent.py" if in_agents else "src/coder_eval/reports_html.py"
+        return NoCommandOutputTruncation(path).check(ast.parse(src))
+
+    @pytest.mark.parametrize(
+        "expr",
+        [
+            "output[:100]",
+            "aggregated_output[:512]",
+            "command_item.aggregated_output[:100]",
+            "proc_stdout[:80]",
+            "result.stderr[:200]",
+            'f"Output: {output[:100]}"',
+        ],
+    )
+    def test_flags_output_truncation_in_agents(self, expr: str):
+        assert self._run(f"x = {expr}"), f"expected CE043 to flag {expr!r}"
+
+    def test_allows_untruncated_output(self):
+        assert not self._run('summary = f"Output: {output}"')
+        assert not self._run("summary = aggregated_output")
+
+    def test_ignores_non_output_slices(self):
+        # Legit error/orchestration summaries that are NOT captured command output.
+        assert not self._run("detail = summary.result[:200]")
+        assert not self._run("msg = content_str[:200]")
+        assert not self._run("s = '; '.join(messages)[:200]")
+
+    def test_scoped_to_agents_only(self):
+        # The same pattern outside agents/ is not this rule's concern (display code).
+        assert not self._run("preview = output[:100]", in_agents=False)
+
+
 @pytest.mark.lint
 class TestCE017ModelsLazyAgentImports:
     """CE017 flags only module-level agents/plugins imports inside models/."""

{coder_eval-0.11.0 → coder_eval-0.11.1}/tests/test_judge_anthropic.py

@@ -10,6 +10,7 @@ from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
+from anthropic.resources.messages import AsyncMessages
 
 from coder_eval.evaluation.judge_anthropic import invoke_anthropic_judge_async
 from coder_eval.evaluation.verdict_tool import SUBMIT_VERDICT_ANTHROPIC_TOOL
@@ -29,12 +30,25 @@ def _make_response(*, score: float = 0.5, rationale: str = "ok") -> MagicMock:
 
 def _make_client(response: MagicMock | None = None) -> MagicMock:
     client = MagicMock()
-    client.messages.create = AsyncMock(return_value=response if response is not None else _make_response())
+    # spec-bound to the real ``AsyncMessages.create`` signature so a kwarg the
+    # installed SDK no longer accepts (e.g. a removed ``temperature``) fails
+    # here instead of silently passing against an unconstrained MagicMock.
+    client.messages = MagicMock(spec=AsyncMessages)
+    client.messages.create = AsyncMock(
+        wraps=lambda **kwargs: _bind_and_return(response if response is not None else _make_response(), **kwargs)
+    )
     client.__aenter__ = AsyncMock(return_value=client)
     client.__aexit__ = AsyncMock(return_value=None)
     return client
 
 
+def _bind_and_return(response: MagicMock, **kwargs: Any) -> MagicMock:
+    import inspect
+
+    inspect.signature(AsyncMessages.create).bind(MagicMock(), **kwargs)
+    return response
+
+
 async def _invoke(**overrides):
     defaults = {
         "model": "anthropic.claude-sonnet-4-6",
@@ -79,24 +93,43 @@ async def test_invoke_anthropic_judge_raises_on_empty_model() -> None:
 
 
 async def test_invoke_anthropic_judge_passes_temperature_and_max_tokens() -> None:
+    """``temperature`` travels via ``extra_body`` — anthropic 1.0.0 dropped it as a
+    top-level ``messages.create`` kwarg, but the Messages API still accepts it in
+    the raw JSON body (the same body shape the Bedrock path sends it in)."""
     client = _make_client()
     with patch("coder_eval.evaluation.judge_anthropic.AsyncAnthropic", return_value=client):
         await _invoke(temperature=0.7, max_tokens=321, system="sys", user="usr")
     kwargs: dict[str, Any] = client.messages.create.call_args.kwargs
-    assert kwargs["temperature"] == 0.7
+    assert "temperature" not in kwargs
+    assert kwargs["extra_body"] == {"temperature": 0.7}
     assert kwargs["max_tokens"] == 321
     assert kwargs["system"] == "sys"
     assert kwargs["messages"] == [{"role": "user", "content": "usr"}]
 
 
+async def test_invoke_anthropic_judge_escalates_on_signature_break() -> None:
+    """A kwarg the installed SDK no longer accepts must escalate as infra, not
+    silently score the row 0.0 (see judge_bedrock.py's parallel retry/escalation
+    contract and CLAUDE.md's CE039 rationale)."""
+    from coder_eval.errors import JudgeInfrastructureError
+
+    client = _make_client()
+    client.messages.create.side_effect = TypeError("create() got an unexpected keyword argument 'temperature'")
+    with (
+        patch("coder_eval.evaluation.judge_anthropic.AsyncAnthropic", return_value=client),
+        pytest.raises(JudgeInfrastructureError, match="Anthropic judge call failed"),
+    ):
+        await _invoke()
+
+
 async def test_invoke_anthropic_judge_wraps_api_error() -> None:
-    import httpx
+    import httpx2
     from anthropic import APIConnectionError
 
     from coder_eval.errors import JudgeInfrastructureError
 
     client = _make_client()
-    sdk_error = APIConnectionError(request=httpx.Request("POST", "https://api.anthropic.com"))
+    sdk_error = APIConnectionError(request=httpx2.Request("POST", "https://api.anthropic.com"))
     client.messages.create.side_effect = sdk_error
     with (
         patch("coder_eval.evaluation.judge_anthropic.AsyncAnthropic", return_value=client),