coder-eval 0.11.0__tar.gz → 0.11.2__tar.gz

{coder_eval-0.11.0 → coder_eval-0.11.2}/.github/workflows/pr-checks.yml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -297,7 +297,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project without [uipath] extra
@@ -381,7 +381,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -527,7 +527,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -697,7 +697,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
@@ -840,7 +840,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (with codex extra)
@@ -919,7 +919,7 @@ jobs:
 
       - name: Install uv
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade "pip>=26.2"  # PYSEC-2026-3721 fix floor
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)

{coder_eval-0.11.0 → coder_eval-0.11.2}/CHANGELOG.md

@@ -2,6 +2,54 @@
 
 <!-- version list -->
 
+## v0.11.2 (2026-08-24)
+
+### Bug Fixes
+
+- **docker**: Expand ~ and $VAR in extra_mounts destinations
+  ([#128](https://github.com/UiPath/coder_eval/pull/128),
+  [`02dbf69`](https://github.com/UiPath/coder_eval/commit/02dbf694705fd3f7de4ce164230a70945f08a1ce))
+
+- **docker**: Reject expansions that inject a ':' into a mount path
+  ([#128](https://github.com/UiPath/coder_eval/pull/128),
+  [`02dbf69`](https://github.com/UiPath/coder_eval/commit/02dbf694705fd3f7de4ce164230a70945f08a1ce))
+
+### Documentation
+
+- **docker**: Trim the extra_mounts destination comments to scope
+  ([#128](https://github.com/UiPath/coder_eval/pull/128),
+  [`02dbf69`](https://github.com/UiPath/coder_eval/commit/02dbf694705fd3f7de4ce164230a70945f08a1ce))
+
+
+## v0.11.1 (2026-08-21)
+
+### Bug Fixes
+
+- **ci**: Pin pip floor to 26.2 to close PYSEC-2026-3721
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **codex**: Store command output whole in result_summary (+ code-review fixes)
+  ([#127](https://github.com/UiPath/coder_eval/pull/127),
+  [`3d09f0f`](https://github.com/UiPath/coder_eval/commit/3d09f0f4d966f619405058aca44aa982b1ba265a))
+
+- **deps**: Address PR #133 review — anthropic 1.0.0 dropped temperature kwarg
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **deps**: Bump anthropic to 1.0.0 and migrate Bedrock judge path to httpx2
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **deps**: Bump anthropic to 1.0.0, migrate Bedrock judge path to httpx2
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+- **deps**: Re-lock pip to 26.2.1 to actually close PYSEC-2026-3721
+  ([#133](https://github.com/UiPath/coder_eval/pull/133),
+  [`beecedd`](https://github.com/UiPath/coder_eval/commit/beeceddc521957d3cb53f944b7877dada1155548))
+
+
 ## v0.11.0 (2026-08-19)
 
 ### Bug Fixes

{coder_eval-0.11.0 → coder_eval-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coder-eval
-Version: 0.11.0
+Version: 0.11.2
 Summary: Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites.
 Project-URL: Homepage, https://coder-eval.com
 Project-URL: Repository, https://github.com/UiPath/coder_eval
@@ -23,11 +23,12 @@ Classifier: Topic :: Software Development :: Quality Assurance
 Classifier: Topic :: Software Development :: Testing
 Classifier: Typing :: Typed
 Requires-Python: >=3.13
-Requires-Dist: anthropic>=0.86.0
+Requires-Dist: anthropic<2.0.0,>=1.0.0
 Requires-Dist: anyio>=4.13.0
 Requires-Dist: azure-monitor-opentelemetry-exporter<1.1.0,>=1.0.0b30
 Requires-Dist: claude-agent-sdk>=0.2.124
 Requires-Dist: click>=8.3.3
+Requires-Dist: httpx2<3.0.0,>=2.12.0
 Requires-Dist: jmespath>=1.1.0
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: opentelemetry-sdk<2.0.0,>=1.30.0

{coder_eval-0.11.0 → coder_eval-0.11.2}/action.yml

@@ -41,7 +41,7 @@ inputs:
   version:
     description: coder-eval version to install from PyPI, or "local" to install from the action checkout
     required: false
-    default: "0.11.0"   # <-- kept in sync with releases by release.yml
+    default: "0.11.2"   # <-- kept in sync with releases by release.yml
   run-dir:
     description: Run directory (--run-dir)
     required: false

{coder_eval-0.11.0 → coder_eval-0.11.2}/experiments/default.yaml

@@ -18,9 +18,11 @@ defaults:
   # Per-row dataset multiplication: a 100-row task with max_usd: 0.10 permits up to
   # $10 cumulative spend.
   run_limits:
-    # Maximum agent inner-loop turns per iteration (null = SDK default).
-    # Typical range in tasks: 3-18 depending on complexity.
-    max_turns: 20
+    # Hard safety ceiling on agent inner-loop turns per iteration (null = SDK
+    # default). Set well above the typical 3-18 turns a task needs so it only
+    # trips on runaway loops; task_timeout / turn_timeout below are the practical
+    # guards. Override per-task when a task legitimately needs more.
+    max_turns: 100
     # Soft target: when cumulative SDK turns across all iterations of a task
     # exceed this, the orchestrator logs a one-shot warning and the report
     # surfaces a badge. Does NOT abort the run (max_turns is the hard cap).

{coder_eval-0.11.0 → coder_eval-0.11.2}/plugins/coder-eval/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "coder-eval",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.",
   "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" },
   "homepage": "https://coder-eval.com",

{coder_eval-0.11.0 → coder_eval-0.11.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "coder-eval"
-version = "0.11.0"
+version = "0.11.2"
 description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites."
 readme = "README.md"
 license = "Apache-2.0"
@@ -34,7 +34,14 @@ dependencies = [
     "click>=8.3.3",
     "rich>=14.3.3",
     "python-dotenv>=1.2.2",
-    "anthropic>=0.86.0",
+    # Cap the major: 1.0.0 already proved a major can drop call kwargs
+    # (temperature/top_p/top_k off messages.create — see judge_anthropic.py).
+    "anthropic>=1.0.0,<2.0.0",
+    # anthropic>=1.0.0 migrated its HTTP layer from httpx to httpx2 (its own
+    # exceptions, e.g. APIConnectionError, now carry an httpx2.Request). Declared
+    # explicitly since our code/tests construct httpx2 types directly. Capped to
+    # mirror anthropic's own bound on it (`httpx2<3,>=2.0.0`).
+    "httpx2>=2.12.0,<3.0.0",
     "claude-agent-sdk>=0.2.124",
     "anyio>=4.13.0",
     "radon>=6.0.1",

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/__init__.py

@@ -1,3 +1,3 @@
 """coder_eval - A framework for evaluating AI coding agents."""
 
-__version__ = "0.11.0"
+__version__ = "0.11.2"

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/agents/codex_agent.py

@@ -2178,10 +2178,15 @@ class CodexAgent(Agent[CodexAgentConfig]):
             # Determine result status from exit code
             result_status = "success" if exit_code == 0 else "error" if exit_code is not None else "unknown"
 
-            # Build result summary with output if available
+            # Build result summary with output if available. Store the output WHOLE:
+            # CommandTelemetry.result_summary is the untruncated tool-result body (its
+            # length drives CommandTelemetry.result_tokens), so truncating here would
+            # under-report tool-output size for every command (see CE043). The output is
+            # already bounded by the Codex harness's own exec-output truncation; any
+            # further trimming for display belongs in the renderers/reports, not capture.
             summary_parts = [f"Exit code: {exit_code}" if exit_code is not None else "Command executed"]
             if output and len(output.strip()) > 0:
-                summary_parts.append(f"Output: {output[:100]}")
+                summary_parts.append(f"Output: {output}")
             result_summary = " | ".join(summary_parts)
 
             # Try to parse output as JSON

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/evaluation/judge_anthropic.py

@@ -40,7 +40,14 @@ async def invoke_anthropic_judge_async(
     Returns the SDK response converted to a dict via ``model_dump`` so the
     caller can reuse ``extract_verdict_from_anthropic_response`` — Anthropic's
     native shape (content blocks with ``type: tool_use``) is identical
-    between this SDK call and the Bedrock httpx-direct call.
+    between this SDK call and the Bedrock httpx2-direct call.
+
+    ``temperature`` is forwarded via ``extra_body`` rather than as a top-level
+    kwarg: anthropic 1.0.0 dropped ``temperature``/``top_p``/``top_k`` from
+    ``AsyncMessages.create``'s typed signature, but the Messages API itself
+    still accepts ``temperature`` in the raw JSON body — the same body shape
+    the Bedrock path already sends it in — so ``extra_body`` keeps both judge
+    backends honoring ``LLMJudgeCriterion.temperature`` identically.
     """
     alias = to_anthropic_alias(model)
     client = AsyncAnthropic(timeout=timeout_seconds)
@@ -51,12 +58,17 @@ async def invoke_anthropic_judge_async(
                 system=system,
                 messages=[{"role": "user", "content": user}],
                 max_tokens=max_tokens,
-                temperature=temperature,
-                tools=[tool_spec],  # type: ignore[arg-type]
+                tools=[tool_spec],  # pyright: ignore[reportArgumentType]
                 tool_choice={"type": "tool", "name": tool_spec["name"]},
+                extra_body={"temperature": temperature},
             )
         except APIError as e:
             # The SDK already retries transient failures internally (2 attempts
             # by default) — do not add another retry loop here.
             raise JudgeInfrastructureError(f"Anthropic judge API error: {e}") from e
+        except Exception as e:
+            # A signature/contract break (e.g. a removed or renamed kwarg after an
+            # SDK bump) must not be scored as an agent failure — see CLAUDE.md's
+            # CE039 rationale: an eval-infra fault is not the agent's fault.
+            raise JudgeInfrastructureError(f"Anthropic judge call failed: {e}") from e
     return response.model_dump()

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/evaluation/judge_bedrock.py

@@ -14,7 +14,7 @@ agent_judge uses the Claude Code SDK subprocess instead — the two paths
 intentionally do not share an HTTP client.
 
 Async on purpose: this is llm_judge's only implementation of the network
-call (there is no sync twin) — ``httpx.AsyncClient`` lets the call yield the
+call (there is no sync twin) — ``httpx2.AsyncClient`` lets the call yield the
 event loop instead of blocking a thread-pool thread for the wait, so
 ``SuccessChecker.check_all_async`` awaits it directly without pinning a
 thread. (``check_all_async`` currently runs criteria sequentially; running
@@ -27,7 +27,7 @@ import asyncio
 import logging
 from typing import Any
 
-import httpx
+import httpx2
 
 from coder_eval.errors import JudgeInfrastructureError
 from coder_eval.errors.categories import RetryConfig
@@ -90,13 +90,13 @@ async def invoke_bedrock_judge_async(
     attempts = _JUDGE_RETRY.max_retries + 1
     last_failure = ""
     last_exc: Exception | None = None
-    async with httpx.AsyncClient() as client:
+    async with httpx2.AsyncClient() as client:
         for attempt in range(attempts):
             if attempt:
                 await asyncio.sleep(compute_backoff(_JUDGE_RETRY, attempt - 1))
             try:
                 response = await client.post(url, headers=headers, json=body, timeout=timeout_seconds)
-            except httpx.HTTPError as e:
+            except httpx2.HTTPError as e:
                 last_failure = f"Bedrock invoke transport error: {e}"
                 last_exc = e
                 logger.warning("Bedrock judge attempt %d/%d failed: %s", attempt + 1, attempts, last_failure)

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/evaluation/verdict_tool.py

@@ -40,7 +40,7 @@ SUBMIT_VERDICT_ANTHROPIC_TOOL: dict[str, Any] = {
     "description": _SUBMIT_VERDICT_DESCRIPTION,
     "input_schema": JudgeVerdict.model_json_schema(),
 }
-"""Anthropic-native tool spec for the Bedrock httpx-direct path and Anthropic SDK calls.
+"""Anthropic-native tool spec for the Bedrock httpx2-direct path and Anthropic SDK calls.
 
 Note: Anthropic uses ``input_schema``, not OpenAI's ``parameters``.
 """
@@ -132,7 +132,7 @@ def extract_verdict_from_anthropic_response(
     ``{"type": "tool_use", "name": "submit_verdict"}`` and validates the last
     one's ``input`` against ``JudgeVerdict``. Used by:
 
-    * The Bedrock httpx path (``invoke_bedrock_judge_async``) — raw JSON dict.
+    * The Bedrock httpx2 path (``invoke_bedrock_judge_async``) — raw JSON dict.
     * The Anthropic SDK Direct path (``invoke_anthropic_judge_async``) —
       response converted via ``Message.model_dump()``.
 

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/isolation/docker_runner.py

@@ -290,11 +290,14 @@ def _validate_extra_mount(spec: str) -> str:
 
     Defends against typos that would silently expose the host fs to the
     container, and against mount specs that shadow framework-owned mounts.
-    Normalizes the source side by expanding ``~`` and ``$VAR`` so authors
-    can write portable specs. Returns the (possibly rewritten) spec to
-    feed back into argv.
+    Normalizes BOTH sides by expanding ``~`` and ``$VAR`` so authors can
+    write portable specs. Returns the (possibly rewritten) spec to feed
+    back into argv.
 
     Notes:
+      - Destinations are expanded too. A container path that has to match a
+        host-valued var (``$SKILLS_REPO_PATH``) would otherwise have to be
+        hardcoded per machine.
       - Mode is REQUIRED. Forgetting ``:ro`` is the single most common way
         to accidentally hand the container RW access to a host directory,
         so we make the author write it explicitly.
@@ -312,26 +315,35 @@ def _validate_extra_mount(spec: str) -> str:
     parts = body.split(":")
     if len(parts) < 2 or len(parts) > 3:
         raise ValueError(f"Invalid extra_mounts entry {spec!r}: expected `src:dst[:ro|rw]`.")
-    src, dst = head + parts[0], parts[1]
+    src, raw_dst = head + parts[0], parts[1]
     # Default to read-only when mode is omitted. Mounting host paths RW
     # by default is the wrong sandbox stance: the few RW use-cases are
     # better stated explicitly than implied by silence.
     mode = parts[2] if len(parts) == 3 else "ro"
     if not src:
         raise ValueError(f"Invalid extra_mounts entry {spec!r}: empty source path.")
-    if not dst:
+    if not raw_dst:
         raise ValueError(f"Invalid extra_mounts entry {spec!r}: empty destination path.")
+    # Expanded before the absolute-path check: that is the point.
+    expanded_src = os.path.expandvars(os.path.expanduser(src))
+    dst = os.path.expandvars(os.path.expanduser(raw_dst))
+    # A variable whose value carries a ':' would add fields to the spec rebuilt
+    # at the bottom, silently moving the destination or widening the mode.
+    # The drive prefix is excluded: its colon is legitimate and already split off.
+    if ":" in dst or ":" in expanded_src[len(head) :]:
+        raise ValueError(f"Invalid extra_mounts entry {spec!r}: expansion introduced a ':' into a path.")
     if not dst.startswith("/"):
-        raise ValueError(f"Invalid extra_mounts entry {spec!r}: destination must be an absolute path.")
+        # expandvars leaves an unset var verbatim, so typos land here.
+        detail = f"{raw_dst!r}" if dst == raw_dst else f"{raw_dst!r} (expanded to {dst!r})"
+        raise ValueError(f"Invalid extra_mounts entry {spec!r}: destination must be an absolute path, got {detail}.")
     if mode not in ("ro", "rw"):
         raise ValueError(f"Invalid extra_mounts entry {spec!r}: mode must be 'ro' or 'rw'.")
-    # Expand ~ and $VAR in the source so authors can write portable specs.
-    expanded_src = os.path.expandvars(os.path.expanduser(src))
     if not Path(expanded_src).exists():
         raise ValueError(f"Invalid extra_mounts entry {spec!r}: source path does not exist on host.")
     # Reject destinations that shadow framework-owned mounts inside the
     # container. ``/work`` substrings are caught too -- /work/foo would
     # land underneath our staging dir and shadow the input/output tree.
+    # Expanded form: a var could itself expand to a reserved path.
     dst_norm = dst.rstrip("/") or "/"
     if dst_norm in _RESERVED_MOUNT_DESTS or dst_norm.startswith(CONTAINER_WORK_DIR + "/"):
         raise ValueError(

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/models/telemetry.py

@@ -410,6 +410,16 @@ class CommandTelemetry(BaseModel):
         result size from prompt-cache growth (which is unavailable when caching is
         disabled). Approximate, not the API's exact tokenizer count, but
         deterministic and always present. 0 when the tool returned no content.
+
+        This measure is only meaningful while ``result_summary`` stays whole: an
+        agent that truncates a command's output before recording it (as the Codex
+        agent once did with ``output[:100]``) silently under-reports that command's
+        result. Lint rule CE043 forbids truncating captured command output
+        (stdout/stderr) in the agents, so the "untruncated" contract holds for
+        command results across agents. (One-line summaries of non-command tool
+        items — e.g. collab/MCP status lines built by ``_summarize_tool_item`` —
+        are intentionally brief and out of scope.) Trim for DISPLAY in the
+        renderers/reports instead.
         """
         if not self.result_summary:
             return 0

{coder_eval-0.11.0 → coder_eval-0.11.2}/src/coder_eval/streaming/renderers.py

@@ -104,14 +104,18 @@ class RichStreamRenderer:
             return f"[cyan]>>> TOOL: {escape(event.tool.tool_name)}[/cyan] | {params_str}"
 
         if isinstance(event, ToolEndEvent):
-            preview = escape(event.tool.result_summary or "")
+            # result_summary is stored WHOLE (untruncated) at capture; cap it here
+            # for the terse live feed so a large command output doesn't flood the
+            # console. The reported char count reflects the true (full) length.
+            full = event.tool.result_summary or ""
+            preview = escape(_truncate(full, _MAX_RESULT_LEN))
             if event.status == ToolEndStatus.OK:
                 tag = "[green]<<< OK[/green]"
             elif event.status == ToolEndStatus.UNRESOLVED:
                 tag = "[yellow]<<< UNRESOLVED:[/yellow]"
             else:
                 tag = f"[red]<<< {escape(event.status.value.upper())}:[/red]"
-            return f"{tag} ({len(preview)} chars) {preview}"
+            return f"{tag} ({len(full)} chars) {preview}"
 
         if isinstance(event, TextChunkEvent):
             return f"[dim]{escape(event.text)}[/dim]"

coder_eval-0.11.2/tests/lint/rules/ce043_no_command_output_truncation.py

@@ -0,0 +1,75 @@
+"""CE043: Agents must not truncate a command's output when recording it.
+
+``CommandTelemetry.result_summary`` is contractually the *untruncated* tool-result
+body, and its length drives the ``result_tokens`` computed field (the cost
+simulator's cache-independent measure of tool-output size). An agent that clips a
+command's captured output before storing it silently under-reports every tool
+result — exactly the bug the Codex agent shipped with (``f"Output: {output[:100]}"``),
+which pinned ~77% of its Bash results at ~31 tokens and skewed the cost model.
+
+This rule flags, inside ``src/coder_eval/agents/``, a constant-upper-bound slice
+(``x[:N]``) applied to a value that denotes captured command output:
+
+  * a name whose id is/ends with ``output`` / ``stdout`` / ``stderr``
+    (e.g. ``output[:100]``, ``aggregated_output[:512]``, ``proc_stdout[:80]``)
+  * an attribute access ``.aggregated_output`` / ``.output`` / ``.stdout`` / ``.stderr``
+    (e.g. ``command_item.aggregated_output[:100]``)
+
+Store the output whole (it is already bounded by the harness's own exec-output
+truncation) and trim for DISPLAY in the renderers/reports instead.
+
+Add ``# noqa: CE043`` on the offending line for a genuinely non-recorded use
+(e.g. slicing stdout only to build a short crash/log message that never becomes a
+``result_summary``), with a comment explaining why.
+"""
+
+import ast
+
+from tests.lint.rules.base import BaseRule
+
+
+_OUTPUT_NAMES = {"output", "stdout", "stderr", "aggregated_output"}
+
+
+def _denotes_output(value: ast.AST) -> str | None:
+    """Return the output-ish identifier being sliced, or None."""
+    if isinstance(value, ast.Name):
+        low = value.id.lower()
+        if low in _OUTPUT_NAMES or low.endswith("_output") or low.endswith("_stdout") or low.endswith("_stderr"):
+            return value.id
+    elif isinstance(value, ast.Attribute):
+        if value.attr.lower() in _OUTPUT_NAMES:
+            return value.attr
+    return None
+
+
+def _is_const_upper_slice(sl: ast.AST) -> bool:
+    """True for ``[:N]`` / ``[:N:...]`` with a constant int upper bound."""
+    return (
+        isinstance(sl, ast.Slice)
+        and sl.lower is None
+        and isinstance(sl.upper, ast.Constant)
+        and isinstance(sl.upper.value, int)
+    )
+
+
+class NoCommandOutputTruncation(BaseRule):
+    id = "CE043"
+
+    def check(self, tree: ast.AST) -> list:
+        # Scope to the agent implementations — only they record CommandTelemetry.
+        if "/agents/" not in self.filepath.replace("\\", "/"):
+            return []
+        self.visit(tree)
+        return self.violations
+
+    def visit_Subscript(self, node: ast.Subscript) -> None:
+        name = _denotes_output(node.value)
+        if name is not None and _is_const_upper_slice(node.slice):
+            self.violation(
+                node,
+                f"Do not truncate command output ({name}[:...]); CommandTelemetry.result_summary "
+                "must stay whole (it drives result_tokens). Store the full output and trim for "
+                "display in the renderers/reports instead.",
+            )
+        self.generic_visit(node)

{coder_eval-0.11.0 → coder_eval-0.11.2}/tests/lint/runner.py

@@ -24,6 +24,7 @@ from tests.lint.rules.ce032_criteria_path_seam import CriteriaPathSeam
 from tests.lint.rules.ce037_no_dead_private_helper import NoDeadPrivateHelper
 from tests.lint.rules.ce038_acquire_inside_try import AcquireInsideTry
 from tests.lint.rules.ce039_config_error_escalates import ConfigErrorEscalates
+from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation
 from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess
 from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync
 from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore
@@ -71,6 +72,7 @@ ALL_RULES: list[RuleClass] = [
     NoDeadPrivateHelper,
     AcquireInsideTry,
     ConfigErrorEscalates,
+    NoCommandOutputTruncation,
 ]
 
 # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule

{coder_eval-0.11.0 → coder_eval-0.11.2}/tests/test_codex_agent_unit.py

@@ -104,3 +104,28 @@ class TestCodexTurnState:
         # A tool_use block was recorded into the open buffer (cut at the next
         # tokenUsage flush, not here), joinable to the command by tool_id.
         assert any(b.block_type == "tool_use" and b.tool_use_id == "c1" for b in state.open_blocks)
+
+    def test_command_output_recorded_whole_not_truncated(self):
+        # Regression for the Codex `output[:100]` bug (CE043): result_summary must
+        # carry the FULL command output so result_tokens reflects real tool-output
+        # size instead of being pinned at a ~31-token, 100-char cap.
+        agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5-codex"))
+        state, commands, _messages = self._state(agent)
+
+        big_output = "X" * 4000  # far beyond the old 100-char clip
+        cmd_root = SimpleNamespace(
+            type="commandExecution",
+            id="c2",
+            command="cat big.txt",
+            exit_code=0,
+            aggregated_output=big_output,
+            duration_ms=5,
+        )
+        state.on_item_started(_item_notification("item/started", cmd_root))
+        state.on_item_completed(_item_notification("item/completed", cmd_root))
+
+        cmd = commands[0]
+        assert big_output in (cmd.result_summary or ""), "full output must be recorded, not truncated"
+        # result_tokens (ceil(len/4)) must scale with the real output, not ~31.
+        assert cmd.result_tokens >= len(big_output) // 4
+        assert cmd.result_tokens > 100

{coder_eval-0.11.0 → coder_eval-0.11.2}/tests/test_custom_lint.py

@@ -60,6 +60,48 @@ class TestCE016NoComputedTokenUsageKwargs:
         assert not self._run("ReconciliationMessage(input_tokens=-5)")
 
 
+@pytest.mark.lint
+class TestCE043NoCommandOutputTruncation:
+    """CE043 flags truncation of captured command output inside agents/, only."""
+
+    @staticmethod
+    def _run(src: str, *, in_agents: bool = True):
+        import ast
+
+        from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation
+
+        path = "src/coder_eval/agents/codex_agent.py" if in_agents else "src/coder_eval/reports_html.py"
+        return NoCommandOutputTruncation(path).check(ast.parse(src))
+
+    @pytest.mark.parametrize(
+        "expr",
+        [
+            "output[:100]",
+            "aggregated_output[:512]",
+            "command_item.aggregated_output[:100]",
+            "proc_stdout[:80]",
+            "result.stderr[:200]",
+            'f"Output: {output[:100]}"',
+        ],
+    )
+    def test_flags_output_truncation_in_agents(self, expr: str):
+        assert self._run(f"x = {expr}"), f"expected CE043 to flag {expr!r}"
+
+    def test_allows_untruncated_output(self):
+        assert not self._run('summary = f"Output: {output}"')
+        assert not self._run("summary = aggregated_output")
+
+    def test_ignores_non_output_slices(self):
+        # Legit error/orchestration summaries that are NOT captured command output.
+        assert not self._run("detail = summary.result[:200]")
+        assert not self._run("msg = content_str[:200]")
+        assert not self._run("s = '; '.join(messages)[:200]")
+
+    def test_scoped_to_agents_only(self):
+        # The same pattern outside agents/ is not this rule's concern (display code).
+        assert not self._run("preview = output[:100]", in_agents=False)
+
+
 @pytest.mark.lint
 class TestCE017ModelsLazyAgentImports:
     """CE017 flags only module-level agents/plugins imports inside models/."""

{coder_eval-0.11.0 → coder_eval-0.11.2}/tests/test_docker_runner_mounts.py

@@ -108,6 +108,41 @@ class TestValidateExtraMount:
         result = _validate_extra_mount("$MYDIR:/mnt/x")
         assert result.startswith(real_dir + ":")
 
+    def test_var_expansion_in_destination(self, real_dir, monkeypatch):
+        """``$VAR`` in the destination expands too."""
+        monkeypatch.setenv("HOME", "/home/someuser")
+        result = _validate_extra_mount(f"{real_dir}:$HOME/.uipath:rw")
+        assert result == f"{real_dir}:/home/someuser/.uipath:rw"
+
+    def test_home_expansion_in_destination(self, real_dir, monkeypatch):
+        """``~`` in the destination expands the same way the source does."""
+        monkeypatch.setenv("HOME", "/home/someuser")
+        result = _validate_extra_mount(f"{real_dir}:~/.uipath:ro")
+        assert result == f"{real_dir}:/home/someuser/.uipath:ro"
+
+    def test_unset_var_destination_rejected_with_both_forms(self, real_dir):
+        """An unset var is left verbatim, so it must fail loudly."""
+        with pytest.raises(ValueError, match="destination must be an absolute path"):
+            _validate_extra_mount(f"{real_dir}:$NO_SUCH_VAR_HERE/x:ro")
+
+    def test_destination_var_expanding_to_reserved_is_rejected(self, real_dir, monkeypatch):
+        """The shadow check runs on the expanded destination."""
+        monkeypatch.setenv("SNEAKY", "/work")
+        with pytest.raises(ValueError, match="shadows a framework-owned mount"):
+            _validate_extra_mount(f"{real_dir}:$SNEAKY:ro")
+
+    def test_destination_var_carrying_a_colon_is_rejected(self, real_dir, monkeypatch):
+        """A ':' in an expanded value would add fields to the rebuilt spec."""
+        monkeypatch.setenv("SNEAKY", "/mnt/x:rw")
+        with pytest.raises(ValueError, match="introduced a ':'"):
+            _validate_extra_mount(f"{real_dir}:$SNEAKY:ro")
+
+    def test_source_var_carrying_a_colon_is_rejected(self, monkeypatch):
+        """Same guard on the source side, which is rebuilt the same way."""
+        monkeypatch.setenv("SNEAKY", "/mnt/x:rw")
+        with pytest.raises(ValueError, match="introduced a ':'"):
+            _validate_extra_mount("$SNEAKY:/mnt/y:ro")
+
     def test_malformed_no_colon(self):
         with pytest.raises(ValueError, match="expected `src:dst"):
             _validate_extra_mount("just-one-token")