coder-eval 0.8.2__py3-none-any.whl

coder_eval/evaluation/judge_persistence.py

@@ -0,0 +1,291 @@
+"""Sibling-file persistence for ``JudgeCriterionResult.transcript``.
+
+The full judge transcript (tool calls, raw verdict, rendered prompt and
+system prompt) can run 10-100 KB. Inlining it into every ``task.json``
+inflates the row record for consumers (suite rollups, report renderers)
+that don't need it. Spilling each transcript to a sibling
+``judge-<idx>.yaml`` next to ``task.json`` keeps the row record lean and
+lets reviewers grep transcripts independently.
+
+YAML (over JSON) for the sibling: the transcript carries multi-line text
+(``judge_prompt``, ``judge_system_prompt``, ``raw_verdict``) which YAML's
+literal block scalar (``|``) renders as readable paragraphs instead of
+single-line strings with ``\\n`` escapes. Sibling consumers are humans;
+YAML wins.
+
+Two functions:
+
+- ``spill_judge_transcripts``: called by the orchestrator just before it
+  writes ``task.json``. For each judge result with an inline ``transcript``,
+  writes a sibling file and sets ``transcript_path`` on the result. The
+  inline ``transcript`` is left in place so HTML rendering against the
+  in-memory ``EvaluationResult`` still sees it; ``model_dump_json``
+  callers strip it via ``exclude={...}``.
+
+- ``load_judge_transcripts``: called by re-render paths
+  (``coder-eval report``, stats loaders) after
+  ``EvaluationResult.model_validate_json``. For each result whose
+  ``transcript_path`` points at an existing sibling file, reads it back
+  and attaches it as a dict on ``transcript`` so renderers see the same
+  shape they get during the original run. Accepts both ``.yaml`` (the
+  current format) and ``.json`` (the previous format) so previously-spilled
+  runs keep rendering.
+
+Backward compatibility: old ``task.json`` files with inline ``transcript``
+keep working — the loader treats ``transcript_path is None`` as a no-op.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import PurePosixPath, PureWindowsPath
+from typing import TYPE_CHECKING, Any
+
+import yaml
+from pydantic import ValidationError
+
+from coder_eval.models import JudgeCriterionResult, JudgeTranscript
+
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from coder_eval.models import EvaluationResult
+
+
+logger = logging.getLogger(__name__)
+
+
+# Windows reserved device basenames. The Win32 API maps these to character
+# devices regardless of the directory they sit in — opening ``CON`` or ``NUL.yaml``
+# inside ``task_dir`` resolves to the console or the null device, not a file.
+# Case-insensitive; the trailing extension (if any) is ignored by Win32 too,
+# so ``con``, ``CON``, ``CON.yaml``, ``nul.txt`` all map to devices.
+# Only COM1-9 / LPT1-9 are device names; COM10 and beyond are regular files.
+_WINDOWS_RESERVED_BASENAMES = frozenset(
+    {"CON", "PRN", "AUX", "NUL"} | {f"COM{i}" for i in range(1, 10)} | {f"LPT{i}" for i in range(1, 10)}
+)
+
+
+# Field order in the YAML output: lead with the human-readable summary fields
+# (durations, token counts, prompts), then the long body fields. Verdict-shape
+# (raw_verdict) goes last because it's the bulkiest.
+_TRANSCRIPT_FIELD_ORDER = (
+    "duration_seconds",
+    "truncated",
+    "token_usage",
+    "tool_calls",
+    "judge_system_prompt",
+    "judge_prompt",
+    "raw_verdict",
+)
+
+
+class _BlockLiteralDumper(yaml.SafeDumper):
+    """SafeDumper that renders multi-line strings as literal block scalars.
+
+    Without the override, pyyaml dumps multi-line strings as quoted single-line
+    strings with ``\\n`` escapes — unreadable for the rendered prompts and raw
+    verdicts that are the whole point of this file. The override flips strings
+    containing newlines to use the ``|`` style (literal block scalar) so they
+    render as native indented paragraphs.
+    """
+
+
+def _str_presenter(dumper: _BlockLiteralDumper, data: str) -> Any:
+    if "\n" in data:
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+_BlockLiteralDumper.add_representer(str, _str_presenter)
+
+
+def _ordered_transcript_dict(transcript_dump: dict[str, Any]) -> dict[str, Any]:
+    """Return ``transcript_dump`` with keys in the human-friendly order above.
+
+    Unknown keys (forward-compat) are appended after the known ones in their
+    original insertion order.
+    """
+    ordered: dict[str, Any] = {}
+    for key in _TRANSCRIPT_FIELD_ORDER:
+        if key in transcript_dump:
+            ordered[key] = transcript_dump[key]
+    for key, value in transcript_dump.items():
+        if key not in ordered:
+            ordered[key] = value
+    return ordered
+
+
+def spill_judge_transcripts(result: EvaluationResult, output_dir: Path) -> int:
+    """Write each judge result's inline transcript to a sibling YAML file.
+
+    For each ``JudgeCriterionResult`` in ``result.success_criteria_results``
+    that carries a non-None ``transcript``, writes ``judge-<idx>.yaml`` in
+    ``output_dir`` (creating the directory if needed) and sets
+    ``transcript_path`` on the result to the sibling filename.
+
+    The inline ``transcript`` is **left in place** so in-memory consumers
+    (HTML rendering at the end of the orchestrator run) still see it.
+    Callers writing ``task.json`` should pass
+    ``exclude={"success_criteria_results": {"__all__": {"transcript"}}}``
+    to ``model_dump_json`` so the on-disk record carries only the path.
+
+    Returns the count of transcripts spilled (informational; no-op when 0).
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+    spilled = 0
+    # ORDER IS LOAD-BEARING. ``judge-{idx}.yaml`` is keyed off the criterion's
+    # position in ``success_criteria_results``; ``load_judge_transcripts`` reads
+    # ``transcript_path`` (which we set below) to find each sibling, so the
+    # filename naming scheme itself can change freely. What MUST stay stable is
+    # the index→file mapping for the lifetime of any task.json that references
+    # these siblings: writers that reorder ``success_criteria_results`` between
+    # spill and read would break the binding. Today's only writer is the
+    # orchestrator and the order is preserved through model_dump_json/
+    # model_validate_json, so this is safe — keep it that way.
+    for idx, cr in enumerate(result.success_criteria_results):
+        if not isinstance(cr, JudgeCriterionResult):
+            continue
+        if cr.transcript is None:
+            continue
+        sibling_name = f"judge-{idx}.yaml"
+        sibling_path = output_dir / sibling_name
+        ordered = _ordered_transcript_dict(cr.transcript.model_dump())
+        sibling_path.write_text(
+            yaml.dump(
+                ordered,
+                Dumper=_BlockLiteralDumper,
+                sort_keys=False,
+                allow_unicode=True,
+                width=100,
+            ),
+            encoding="utf-8",
+        )
+        cr.transcript_path = sibling_name
+        spilled += 1
+    if spilled:
+        logger.debug("spilled %d judge transcript(s) to %s", spilled, output_dir)
+    return spilled
+
+
+def load_judge_transcripts(result: EvaluationResult, task_dir: Path) -> int:
+    """Read sibling judge transcript files and attach them to each result.
+
+    For each criterion result in ``result.success_criteria_results`` that has
+    a ``transcript_path`` set (and no inline ``transcript`` — already-loaded
+    results are left alone), reads the sibling file relative to ``task_dir``
+    and attaches the parsed dict on ``transcript`` so HTML / markdown
+    renderers see the same shape they get during the original run.
+
+    Missing sibling files are skipped silently and logged at debug level —
+    runs predating this feature have no sibling files and render fine via
+    their inline transcript (preserved through ``CriterionResult``'s
+    ``extra="allow"`` round-trip).
+
+    Returns the count of transcripts loaded.
+    """
+    loaded = 0
+    for cr in result.success_criteria_results:
+        path = getattr(cr, "transcript_path", None)
+        if not path:
+            continue
+        # Skip results that already have an inline transcript — typical for
+        # the orchestrator's own first HTML render, which runs against the
+        # in-memory result before it gets dumped + reloaded.
+        if getattr(cr, "transcript", None):
+            continue
+        # SECURITY: transcript_path comes from task.json, which may travel across
+        # trust boundaries (CI artifacts, shared eval bundles). spill_judge_transcripts
+        # only ever writes the literal ``f"judge-{idx}.yaml"`` — a basename, no
+        # separators, no ``..``. Allowlist the basename shape directly so a tampered
+        # ``transcript_path: '/etc/passwd'`` or ``../../secrets`` is refused at the
+        # door rather than relying on ``is_relative_to`` to catch it after a join.
+        # Check BOTH PurePosixPath (forward-slash separator) AND PureWindowsPath
+        # (forward and back-slash separators, drive-letter prefixes): a path like
+        # ``subdir\judge-0.yaml`` passes the POSIX check on Linux (backslash is a
+        # regular char) but resolves to a nested file on Windows. Rejecting under
+        # either interpretation enforces the basename-only policy regardless of
+        # which platform the task.json travels to next.
+        if PurePosixPath(path).name != path or PureWindowsPath(path).name != path:
+            logger.warning("Refusing to load judge transcript with non-basename path: %s", path)
+            continue
+        # Reject Windows reserved device basenames. On Windows, ``CON.yaml`` /
+        # ``NUL`` / ``COM1`` open the console / null device / serial port
+        # regardless of where they sit in the directory tree. The check is
+        # platform-independent so a task.json minted on Linux that ships such a
+        # transcript_path is rejected before it travels to Windows.
+        stem_upper = path.split(".", 1)[0].upper()
+        if stem_upper in _WINDOWS_RESERVED_BASENAMES:
+            logger.warning("Refusing to load judge transcript with reserved Windows device name: %s", path)
+            continue
+        sibling = task_dir / path
+        # Defense-in-depth: even with the basename guard above, resolve and verify
+        # containment before reading — symlinks inside ``task_dir`` could redirect
+        # outside it (a malicious bundle could ship one).
+        try:
+            resolved_sibling = sibling.resolve()
+            resolved_root = task_dir.resolve()
+        except OSError as e:
+            logger.warning("Failed to resolve judge transcript path %s: %s", sibling, e)
+            continue
+        if not resolved_sibling.is_relative_to(resolved_root):
+            logger.warning(
+                "Refusing to read judge transcript outside task dir: path=%s resolved=%s task_dir=%s",
+                path,
+                resolved_sibling,
+                resolved_root,
+            )
+            continue
+        if not resolved_sibling.is_file():
+            logger.debug("judge transcript path %s set but %s missing — skipping", path, resolved_sibling)
+            continue
+        sibling = resolved_sibling
+        try:
+            text = sibling.read_text(encoding="utf-8")
+            # Parse as JSON for legacy ``.json`` siblings; anything else (today's
+            # ``.yaml`` and any future format yaml.safe_load handles) goes through
+            # PyYAML, which also accepts JSON as a subset.
+            data = json.loads(text) if path.endswith(".json") else yaml.safe_load(text)
+        except Exception as e:
+            logger.warning("Failed to read judge transcript %s: %s", sibling, e)
+            continue
+        if not isinstance(data, dict):
+            # A scalar / list / None payload would silently land on the result and
+            # crash the HTML renderer (which assumes dict-or-typed) with an
+            # AttributeError on the first ``.get()``. Reject early.
+            logger.warning(
+                "Judge transcript %s is %s, expected mapping — skipping",
+                sibling,
+                type(data).__name__,
+            )
+            continue
+        # Prefer typed JudgeTranscript so renderer / aggregator code that does
+        # isinstance checks sees the same shape it gets during the original run.
+        # Fall back to the raw dict on ValidationError — older spilled siblings
+        # (pre-schema-change) or forward-compat keys shouldn't break re-render.
+        attached: JudgeTranscript | dict[str, Any]
+        try:
+            attached = JudgeTranscript.model_validate(data)
+        except ValidationError as e:
+            logger.debug("Judge transcript %s did not match JudgeTranscript schema, attaching as dict: %s", sibling, e)
+            attached = data
+        # Use object.__setattr__ so we don't go through pydantic's setter,
+        # which (depending on model_config of the loaded subclass) might
+        # validate or reject. The HTML renderer accepts both typed
+        # JudgeTranscript and dict-shape so either shape works downstream.
+        # NOTE: With the ``CriterionResultUnion`` discriminator on
+        # ``EvaluationResult.success_criteria_results``, ``cr`` is now a
+        # properly-typed ``JudgeCriterionResult`` after reload (not a base
+        # ``CriterionResult`` with the field in ``__pydantic_extra__``), so
+        # the assignment lands on the declared field directly.
+        try:
+            object.__setattr__(cr, "transcript", attached)
+        except Exception as e:
+            logger.warning("Failed to attach judge transcript onto %s: %s", type(cr).__name__, e)
+            continue
+        loaded += 1
+    if loaded:
+        logger.debug("loaded %d judge transcript(s) from %s", loaded, task_dir)
+    return loaded

coder_eval/evaluation/judge_usage.py

@@ -0,0 +1,50 @@
+"""Usage extraction from judge LLM responses.
+
+Kept separate from ``judge_models.py`` (whose docstring scopes it to pure
+string utilities for model-name translation) — usage parsing is a distinct
+concern. This helper turns a judge response into a ``TokenUsage`` so
+``llm_judge`` can populate ``JudgeCriterionResult.token_usage`` uniformly
+across the Anthropic / Bedrock-invoke (dict shape) backends.
+
+Returns ``None`` (not a zero ``TokenUsage``) when usage is absent or empty,
+so "unknown" stays distinguishable from "zero".
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from coder_eval.models import TokenUsage
+
+
+def _coerce_int(value: Any) -> int:
+    """Best-effort non-negative int coercion for a usage counter.
+
+    A malformed provider payload (non-numeric token value) must degrade to 0,
+    not fail the judge criterion — these extractors promise "usage or None,
+    never raise". ``int(value or 0)`` already handles falsy/missing; this also
+    absorbs genuinely non-coercible values.
+    """
+    try:
+        return int(value or 0)
+    except (TypeError, ValueError):
+        return 0
+
+
+def token_usage_from_anthropic_dict(resp: dict[str, Any]) -> TokenUsage | None:
+    """Extract usage from an Anthropic / Bedrock-invoke Messages response dict.
+
+    Both ``invoke_anthropic_judge`` (``response.model_dump()``) and
+    ``invoke_bedrock_judge`` (parsed ``/invoke`` JSON) carry an Anthropic-shaped
+    ``usage`` block. Returns ``None`` when usage is missing or carries no tokens.
+    """
+    u = resp.get("usage")
+    if not isinstance(u, dict):
+        return None
+    tu = TokenUsage(
+        uncached_input_tokens=_coerce_int(u.get("input_tokens")),
+        output_tokens=_coerce_int(u.get("output_tokens")),
+        cache_creation_input_tokens=_coerce_int(u.get("cache_creation_input_tokens")),
+        cache_read_input_tokens=_coerce_int(u.get("cache_read_input_tokens")),
+    )
+    return None if tu.is_empty() else tu

coder_eval/evaluation/sub_agent.py

@@ -0,0 +1,231 @@
+"""Isolated sandbox-copy + Claude Code SDK subprocess lifecycle for judge-style criteria.
+
+SECURITY: owns the hardening knobs in one place — symlink-stripping copy,
+``setting_sources=[]`` enforcement, ignore patterns for ``.claude``/``.mcp.json``.
+Any future sub-agent criterion inherits the same posture by construction.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import shutil
+import tempfile
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from coder_eval.agents.claude_code_agent import ClaudeCodeAgent
+from coder_eval.evaluation.verdict_tool import VerdictCapture
+from coder_eval.models import ClaudeCodeAgentConfig
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.models.routing import ApiRoute
+    from coder_eval.sandbox import Sandbox
+
+
+logger = logging.getLogger(__name__)
+
+
+def _ignore_patterns_and_symlinks(patterns: list[str]):
+    """``copytree`` ``ignore`` callable that drops pattern matches AND every symlink.
+
+    Symlinks in the sandbox — whether malicious or accidental — are rejected
+    rather than dereferenced into the judge workspace, which would leak host
+    files (e.g. a ``creds -> /root/.aws/credentials`` plant) to a Bash-enabled
+    judge.
+    """
+    pattern_ignore = shutil.ignore_patterns(*patterns)
+
+    def _ignore(src: str, names: list[str]) -> set[str]:
+        ignored = set(pattern_ignore(src, names))
+        src_path = Path(src)
+        for name in names:
+            if name in ignored:
+                continue
+            if (src_path / name).is_symlink():
+                ignored.add(name)
+        return ignored
+
+    return _ignore
+
+
+class SubAgentRunner:
+    """Spawn a Claude Code SDK agent in an isolated sandbox copy and return its turn.
+
+    Owns:
+    - ``mkdtemp`` + ``copytree`` with symlink filtering + pattern ignores.
+    - ``ClaudeCodeAgent`` construction, ``start``/``communicate``/``stop``/``kill`` lifecycle.
+    - Temp-dir cleanup on every exit path.
+
+    Does NOT own:
+    - Verdict parsing — caller does that on the returned ``TurnRecord``.
+    - Error-to-CriterionResult mapping — caller maps exceptions to their own result type.
+    """
+
+    def __init__(
+        self,
+        *,
+        sandbox: Sandbox,
+        agent_config: ClaudeCodeAgentConfig,
+        ignore_patterns: list[str],
+        route: ApiRoute,
+        reference_dir: Path | None = None,
+        reference_ignore_patterns: list[str] | None = None,
+        extra_mcp_servers: dict[str, Any] | None = None,
+        capture: VerdictCapture | None = None,
+    ) -> None:
+        # SECURITY: setting_sources=[] is enforced by the caller — it's the caller's
+        # responsibility to build the AgentConfig correctly because the field is part of
+        # the type contract. We raise rather than mutate so a misconfigured caller fails
+        # loudly instead of silently having its config changed. Not `assert` because the
+        # check must survive `python -O`.
+        if agent_config.setting_sources != []:
+            raise ValueError(
+                "SubAgentRunner requires agent_config.setting_sources=[] so the SDK does not "
+                + "load .claude/settings.json or .mcp.json from the sub-agent's working directory."
+            )
+        assert sandbox.sandbox_dir is not None, "sandbox not initialized"
+        self._sandbox = sandbox
+        self._agent_config = agent_config
+        self._ignore_patterns = ignore_patterns
+        self._route = route
+        # When set, copied into ``judge_dir / "_reference"`` after the main sandbox
+        # copy. The judge can browse it via Read/Glob. ``None`` skips the copy —
+        # callers MUST set this to None when the criterion has ``include_reference=False``
+        # so the judge can't see grading material it was opted out of.
+        self._reference_dir = reference_dir
+        # Separate ignore set for the reference-side copytree. Defaults to ``[]``
+        # so we DON'T silently strip a nested ``_reference/`` inside the user's
+        # reference dir — the sandbox-side ignore set legitimately contains
+        # ``_reference`` (defense-in-depth against agent-planted collisions at
+        # the mount point), but reusing it here would silently drop a customer
+        # subdir of the same name. Symlinks are stripped unconditionally by
+        # ``_ignore_patterns_and_symlinks([])``.
+        self._reference_ignore_patterns = reference_ignore_patterns or []
+        # Runtime-only in-process MCP server injection (e.g. the judge
+        # submit_verdict tool). NOT routed through ``sdk_options`` —
+        # ``mcp_servers`` is in ``_FRAMEWORK_OWNED_SDK_FIELDS``.
+        self._extra_mcp_servers = extra_mcp_servers or {}
+        # Public attribute so the criterion can read it after ``run()`` returns.
+        # When the caller passes ``capture=None`` the runner doesn't expose one,
+        # matching the opt-in-per-construction contract for the verdict channel.
+        self.capture = capture
+
+    def run(self, user_msg: str, *, max_turns: int | None, turn_timeout: float) -> TurnRecord:
+        """Copy sandbox → start agent → communicate → stop. Kill on any exception.
+
+        Raises ``TurnTimeoutError`` when the agent exceeds ``turn_timeout``.
+        """
+        # Narrow via local var — checked in __init__ but pyright doesn't track that.
+        src_dir = self._sandbox.sandbox_dir
+        assert src_dir is not None, "sandbox not initialized"
+
+        judge_dir = Path(tempfile.mkdtemp(prefix="sub_agent_"))
+        try:
+            # Copy the sandbox into an isolated temp dir. The sub-agent never touches
+            # the original sandbox, so later criteria are unaffected by whatever it
+            # does. Symlinks are skipped (vs preserved) so a malicious
+            # `creds -> /root/.aws/credentials` plant can't leak host files to a
+            # Bash-enabled sub-agent.
+            shutil.copytree(
+                src_dir,
+                judge_dir,
+                symlinks=True,
+                ignore=_ignore_patterns_and_symlinks(self._ignore_patterns),
+                dirs_exist_ok=True,  # mkdtemp already created the target; allow merging in
+            )
+
+            # Mount the reference solution at ``_reference/`` for the judge to browse.
+            # Symlinks are stripped unconditionally by the helper. Pattern-based
+            # ignores use a SEPARATE list (``_reference_ignore_patterns``, default
+            # ``[]``) so we don't reuse the sandbox-side ``ignore_patterns`` which
+            # includes ``_reference`` as defense-in-depth — applying that here would
+            # silently strip a nested ``_reference/`` subdir inside the user's
+            # reference, dropping grading material the user explicitly chose to
+            # include.
+            #
+            # Defense-in-depth on the sandbox side: ``AgentJudgeCriterion.agent.ignore_patterns``
+            # includes ``_reference`` so the first copytree above strips any
+            # sandbox-side ``_reference/`` (agent-planted or template-staged). If a
+            # caller overrides ignore_patterns and removes ``_reference``, the second
+            # copytree would FileExistsError; rmtree is the safety net AND ensures the
+            # judge sees grading material exclusively from ``task.reference``.
+            if self._reference_dir is not None:
+                ref_dest = judge_dir / "_reference"
+                if ref_dest.exists():
+                    shutil.rmtree(ref_dest, ignore_errors=True)
+                # Deliberately NO ``dirs_exist_ok=True`` here. The rmtree above
+                # is the canonical clear; if any file survives (read-only flag,
+                # ENOTEMPTY race, hostile permission bits), we want copytree to
+                # FileExistsError loudly rather than silently merge the reference
+                # into agent-planted content under the same path. Loud failure on
+                # a partial-rmtree edge case is preferred over silently grading
+                # against a tampered ``_reference/``.
+                shutil.copytree(
+                    self._reference_dir,
+                    ref_dest,
+                    symlinks=True,
+                    ignore=_ignore_patterns_and_symlinks(self._reference_ignore_patterns),
+                )
+
+            agent = ClaudeCodeAgent(
+                self._agent_config,
+                route=self._route,
+                extra_mcp_servers=self._extra_mcp_servers,
+            )
+            logger.info(
+                "sub_agent: starting (model=%s, max_turns=%s, allowed_tools=%s)",
+                self._agent_config.model,
+                max_turns,
+                self._agent_config.allowed_tools,
+            )
+            # Safe because callers run check_all via asyncio.to_thread, so this
+            # invocation is on a worker thread with no active event loop. A direct
+            # async caller would get RuntimeError — acceptable for the architecture.
+            turn = asyncio.run(
+                self._run_agent(
+                    agent,
+                    judge_dir,
+                    user_msg,
+                    max_turns,
+                    turn_timeout,
+                    plugin_tools_dir=self._sandbox.plugin_tools_dir,
+                )
+            )
+            logger.info(
+                "sub_agent: finished (duration=%.1fs, tokens=%s)",
+                turn.duration_seconds,
+                turn.token_usage,
+            )
+            return turn
+        finally:
+            shutil.rmtree(judge_dir, ignore_errors=True)
+
+    @staticmethod
+    async def _run_agent(
+        agent: ClaudeCodeAgent,
+        judge_dir: Path,
+        user_msg: str,
+        max_turns: int | None,
+        turn_timeout: float,
+        *,
+        plugin_tools_dir: str | None = None,
+    ) -> TurnRecord:
+        """Run the sub-agent. Hard-kill on any exit path.
+
+        ``stop()`` is cooperative; the SDK's anyio task groups can swallow
+        cancellation, so ``kill()`` is required to guarantee the subprocess dies.
+        """
+        try:
+            await agent.start(str(judge_dir), plugin_tools_dir=plugin_tools_dir)
+            return await agent.communicate(user_msg, timeout=turn_timeout, max_turns=max_turns)
+        except BaseException:
+            with contextlib.suppress(Exception):
+                await agent.kill()
+            raise
+        finally:
+            with contextlib.suppress(Exception):
+                await agent.stop()

coder_eval/evaluation/summaries.py

@@ -0,0 +1,48 @@
+"""Shared formatters for evaluation-related summaries."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from coder_eval.models import CommandTelemetry
+
+
+def summarize_commands(commands: list[CommandTelemetry]) -> str | None:
+    """Build a concise one-line-per-command summary of agent tool calls.
+
+    Used by the llm_judge criterion checker.
+
+    Args:
+        commands: Ordered list of ``CommandTelemetry`` entries (typically
+            ``turn_record.commands``).
+
+    Returns:
+        Formatted multi-line string, or ``None`` when ``commands`` is empty.
+    """
+    if not commands:
+        return None
+
+    lines = []
+    for i, cmd in enumerate(commands, 1):
+        status = cmd.result_status or "unknown"
+        # Extract the most useful parameter for each tool type
+        detail = ""
+        params = cmd.parameters
+        if cmd.tool_name == "Bash" and "command" in params:
+            detail = f" `{params['command'][:120]}`"
+        elif cmd.tool_name in ("Read", "Write", "Edit", "Glob") and "file_path" in params:
+            detail = f" {params['file_path']}"
+        elif cmd.tool_name == "Grep" and "pattern" in params:
+            detail = f" pattern={params['pattern'][:60]}"
+        elif cmd.tool_name in ("Task", "Agent"):
+            detail = f" ({params.get('description', '')[:60]})"
+
+        result_preview = ""
+        if cmd.result_summary:
+            result_preview = f" → {cmd.result_summary[:80]}"
+
+        lines.append(f"  {i}. [{status}] {cmd.tool_name}{detail}{result_preview}")
+
+    return "\n".join(lines)