coder-eval 0.8.2__py3-none-any.whl

coder_eval/evaluation/verdict_tool.py

@@ -0,0 +1,213 @@
+"""Typed verdict tool channel — replaces text-with-JSON parsing for judge criteria.
+
+Defines a single ``submit_verdict`` tool the judge calls as its terminal action,
+exposed in two native shapes (SDK MCP, Anthropic) so each judge backend can use
+its own tool-calling protocol. The extractors then collapse those native
+responses into a single ``tuple[JudgeVerdict | None, str | None]`` contract so
+the rest of the criterion code doesn't branch on backend.
+
+The shared anchor is ``JudgeVerdict.model_validate(args_dict)``: every backend
+ultimately delivers a ``dict``-shaped tool input, and that one call decides
+validity. Error vocabulary is preserved via ``_format_validation_error`` so
+existing tests / on-disk records keep matching.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from claude_agent_sdk import SdkMcpTool, create_sdk_mcp_server, tool
+from claude_agent_sdk.types import McpSdkServerConfig
+from pydantic import ValidationError
+
+from coder_eval.models import JudgeVerdict
+
+
+SUBMIT_VERDICT_TOOL_NAME = "submit_verdict"
+"""Single source of truth for the tool name across all three backends."""
+
+SUBMIT_VERDICT_MCP_TOOL_NAME = "mcp__coder_eval_judge__submit_verdict"
+"""SDK ``allowed_tools`` entry — must match the SDK's ``mcp__<server>__<tool>`` convention."""
+
+_SUBMIT_VERDICT_DESCRIPTION = (
+    "Submit your final score, rationale, and findings. Call exactly once as your terminal action."
+)
+
+
+SUBMIT_VERDICT_ANTHROPIC_TOOL: dict[str, Any] = {
+    "name": SUBMIT_VERDICT_TOOL_NAME,
+    "description": _SUBMIT_VERDICT_DESCRIPTION,
+    "input_schema": JudgeVerdict.model_json_schema(),
+}
+"""Anthropic-native tool spec for the Bedrock httpx-direct path and Anthropic SDK calls.
+
+Note: Anthropic uses ``input_schema``, not OpenAI's ``parameters``.
+"""
+
+
+@dataclass
+class VerdictCapture:
+    """Closure-side state shared between the SDK ``@tool`` and the criterion caller.
+
+    The SDK tool runs inside the agent's MCP server but writes through to this
+    plain object so the criterion can read the verdict after the agent loop
+    finishes. Each judge invocation owns a fresh ``VerdictCapture`` — no global
+    state, safe under concurrent judge runs.
+    """
+
+    verdict: JudgeVerdict | None = None
+    error: str | None = None
+    called_count: int = 0
+    # LAST observed call wins: a successful retry overwrites the previous verdict,
+    # and an invalid retry clears the prior valid verdict (and sets ``error``).
+    # The criterion reads the final state via ``extract_verdict_from_capture``.
+
+
+def _build_submit_verdict_tool(capture: VerdictCapture) -> SdkMcpTool[Any]:
+    """Construct the ``@tool``-decorated submit_verdict closure bound to ``capture``.
+
+    Kept as a separate helper so tests can drive the underlying handler directly
+    (``SdkMcpTool.handler``) without going through the MCP Server's JSONRPC layer.
+    """
+
+    @tool(
+        SUBMIT_VERDICT_TOOL_NAME,
+        _SUBMIT_VERDICT_DESCRIPTION,
+        JudgeVerdict.model_json_schema(),
+    )
+    async def submit_verdict(args: dict[str, Any]) -> dict[str, Any]:
+        capture.called_count += 1
+        try:
+            verdict = JudgeVerdict.model_validate(args)
+        except ValidationError as e:
+            # LAST-call discipline: overwrite verdict too — a successful first
+            # call followed by an invalid retry should surface the latest state.
+            capture.verdict = None
+            capture.error = _format_validation_error(e)
+            return {"content": [{"type": "text", "text": f"Invalid: {capture.error}. Retry."}]}
+        capture.verdict = verdict
+        capture.error = None
+        return {"content": [{"type": "text", "text": "Verdict received. Stop here."}]}
+
+    return submit_verdict
+
+
+def build_submit_verdict_mcp_server(capture: VerdictCapture) -> tuple[str, McpSdkServerConfig]:
+    """Build the ``submit_verdict`` SDK MCP server bound to ``capture``.
+
+    Returns ``(server_name, McpSdkServerConfig)`` ready to plug into
+    ``ClaudeAgentOptions.mcp_servers``. The server name is
+    ``"coder_eval_judge"`` and the resulting allowed-tool entry is
+    ``SUBMIT_VERDICT_MCP_TOOL_NAME`` (the SDK exposes MCP tools as
+    ``mcp__<server>__<tool>``).
+    """
+    sdk_tool = _build_submit_verdict_tool(capture)
+    server = create_sdk_mcp_server("coder_eval_judge", "1.0.0", [sdk_tool])
+    return "coder_eval_judge", server
+
+
+def extract_verdict_from_capture(
+    capture: VerdictCapture,
+) -> tuple[JudgeVerdict | None, str | None]:
+    """Read the LAST successful verdict from ``capture``.
+
+    Returns ``(verdict, None)`` on success, ``(None, error)`` when the tool
+    was called with invalid args, and ``(None, "Judge did not call submit_verdict")``
+    when the tool was never called at all.
+    """
+    if capture.verdict is not None:
+        return capture.verdict, None
+    if capture.error is not None:
+        return None, capture.error
+    return None, "Judge did not call submit_verdict"
+
+
+def extract_verdict_from_anthropic_response(
+    response: dict[str, Any],
+) -> tuple[JudgeVerdict | None, str | None]:
+    """Read the LAST ``submit_verdict`` tool_use block from an Anthropic-shaped response.
+
+    Walks ``response["content"]`` for blocks matching
+    ``{"type": "tool_use", "name": "submit_verdict"}`` and validates the last
+    one's ``input`` against ``JudgeVerdict``. Used by:
+
+    * The Bedrock httpx path (``invoke_bedrock_judge``) — raw JSON dict.
+    * The Anthropic SDK Direct path (``invoke_anthropic_judge``) —
+      response converted via ``Message.model_dump()``.
+
+    Both backends share Anthropic's native message shape.
+
+    LAST-call discipline + non-dict guard: a final ``submit_verdict`` block
+    whose ``input`` is missing or not a dict is treated as an invalid-args
+    failure, not "did not call".
+    """
+    blocks = response.get("content") or []
+    saw_submit_verdict = False
+    last_input: dict[str, Any] | None = None
+    for block in blocks:
+        if not isinstance(block, dict):
+            continue
+        if block.get("type") != "tool_use" or block.get("name") != SUBMIT_VERDICT_TOOL_NAME:
+            continue
+        saw_submit_verdict = True
+        block_input = block.get("input")
+        last_input = block_input if isinstance(block_input, dict) else None
+    if not saw_submit_verdict:
+        return None, "Judge did not call submit_verdict"
+    if last_input is None:
+        return None, "submit_verdict input must be an object"
+    try:
+        return JudgeVerdict.model_validate(last_input), None
+    except ValidationError as e:
+        return None, _format_validation_error(e)
+
+
+def _format_validation_error(e: ValidationError) -> str:
+    """Normalize a Pydantic ``ValidationError`` into the legacy vocabulary.
+
+    Preserves the strings produced by the old ``_verdict_shaped_error`` so
+    persisted ``task.json`` records and existing test assertions stay stable:
+      * ``score field missing in judge verdict``
+      * ``score field is not a number: <repr>``
+      * ``score field is not a finite number: <repr>``
+      * ``rationale field must be a string, got <type>``
+      * ``rationale field is empty after whitespace collapse``
+
+    Multiple errors join with ``"; "``.
+    """
+    messages: list[str] = []
+    for err in e.errors():
+        loc = err.get("loc", ())
+        field = loc[0] if loc else ""
+        etype = err.get("type", "")
+        msg = err.get("msg", "")
+
+        if etype == "missing":
+            # Generic on ``field`` so any future required ``JudgeVerdict`` field gets
+            # the legacy "<name> field missing in judge verdict" vocabulary. Today
+            # ``score`` is the only required field; the test suite pins the string.
+            messages.append(f"{field or 'unknown'} field missing in judge verdict")
+            continue
+
+        # Pydantic wraps custom ValueErrors as "Value error, <body>".
+        body = msg[len("Value error, ") :] if msg.startswith("Value error, ") else msg
+
+        if field == "score" and body.startswith("score is not a number:"):
+            messages.append("score field is not a number:" + body[len("score is not a number:") :])
+            continue
+        if field == "score" and body.startswith("score is not finite:"):
+            messages.append("score field is not a finite number:" + body[len("score is not finite:") :])
+            continue
+        if field == "rationale" and body.startswith("rationale must be a string,"):
+            messages.append("rationale field must be a string," + body[len("rationale must be a string,") :])
+            continue
+        if field == "rationale" and body == "rationale is empty after whitespace collapse":
+            messages.append("rationale field is empty after whitespace collapse")
+            continue
+
+        # Fallback for any new error class we haven't mapped — surface the field qualifier
+        # so the diagnostic still names the offending key.
+        messages.append(f"{field} field: {body}" if field else body)
+
+    return "; ".join(messages)

coder_eval/formatting.py

@@ -0,0 +1,191 @@
+"""Pretty-print SDK payloads for display in logs and stream events."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from claude_agent_sdk import (
+    AssistantMessage,
+    Message,
+    ResultMessage,
+    SystemMessage,
+    UserMessage,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def format_messages(
+    messages: list[Message],
+    *,
+    warned_unknown_types: set[str],
+    log: logging.Logger | logging.LoggerAdapter[Any] = logger,
+) -> str:
+    """Render a list of SDK messages as a readable transcript string.
+
+    System and user messages are filtered out; assistant text, result status,
+    and ``tool_use`` stream events are tagged. Unknown message types emit only a
+    bare ``[TypeName]`` tag — never a truncated ``__repr__``, which could leak
+    unmatched braces into the transcript persisted to ``task.json``.
+
+    Uses ``isinstance`` (not type-name equality) so ``SystemMessage`` subclasses
+    (e.g. ``TaskStartedMessage``) hit the ``SystemMessage`` arm per their drop-in
+    contract.
+
+    Args:
+        messages: SDK message objects from a turn.
+        warned_unknown_types: Mutable set used to deduplicate the
+            "unhandled SDK message type" warning across calls; the caller owns
+            it (e.g. one per agent instance) so the warning fires once per type.
+        log: Logger (or adapter) for that warning; defaults to this module's logger.
+
+    Returns:
+        The formatted transcript, or ``"[No output]"`` when nothing was emitted.
+    """
+    formatted_parts = []
+
+    for msg in messages:
+        if isinstance(msg, SystemMessage):
+            continue
+
+        if isinstance(msg, UserMessage):
+            continue
+
+        if isinstance(msg, AssistantMessage):
+            content = getattr(msg, "content", "")
+            if isinstance(content, list):
+                for block in content:
+                    text = getattr(block, "text", None)
+                    if text:
+                        formatted_parts.append(f"[ASSISTANT] {text}")
+            elif content:
+                formatted_parts.append(f"[ASSISTANT] {content}")
+            continue
+
+        if isinstance(msg, ResultMessage):
+            result_text = getattr(msg, "result", "") or ""
+            is_error = getattr(msg, "is_error", False)
+            status = "ERROR" if is_error else "SUCCESS"
+            formatted_parts.append(f"[RESULT - {status}] {result_text}")
+            continue
+
+        # StreamEvent isn't exported by the SDK — duck-type on ``type``.
+        event_type = getattr(msg, "type", None)
+        if event_type == "tool_use":
+            tool_name = getattr(msg, "name", "unknown")
+            formatted_parts.append(f"[TOOL USE] {tool_name}")
+            continue
+
+        type_name = type(msg).__name__
+        # StreamEvent is a known SDK type used for token-delta capture
+        # elsewhere; don't surface it as an "unhandled" warning here.
+        if type_name == "StreamEvent":
+            continue
+        if type_name not in warned_unknown_types:
+            warned_unknown_types.add(type_name)
+            log.warning(
+                "Unhandled SDK message type %s in format_messages — "
+                + "extend the isinstance chain when the SDK adds new types.",
+                type_name,
+            )
+        formatted_parts.append(f"[{type_name}]")
+
+    return "\n".join(formatted_parts) if formatted_parts else "[No output]"
+
+
+def format_payload(value: Any, *, max_chars: int = 800) -> str:
+    """Render a tool parameter or result payload as a display string.
+
+    JSON-shaped payloads (dict, list, or a string/MCP-block whose first
+    non-whitespace character is ``{`` or ``[``) are pretty-printed with
+    ``indent=2``. Everything else — plain text, strings with prefix noise —
+    falls through to ``str()``. Output is capped at ``max_chars`` with a
+    visible ``…(N more chars)`` marker.
+
+    A companion parser, ``ClaudeCodeAgent._try_parse_json_value``, handles
+    the telemetry path (``result_data`` capture) with stricter heuristics
+    for prefix noise; this helper is intentionally leaner since the fallback
+    to ``str()`` is harmless for display.
+    """
+    parsed = _extract_json(value)
+    if parsed is not None:
+        return _truncate(json.dumps(parsed, indent=2, default=str), max_chars)
+    if value is None:
+        return ""
+    return _truncate(str(value), max_chars)
+
+
+def _extract_json(value: Any) -> dict[str, Any] | list[Any] | None:
+    """Return the non-empty JSON dict/list inside ``value``, else None."""
+    if isinstance(value, dict):
+        return value or None
+    if isinstance(value, list):
+        if value and all(
+            isinstance(v, dict) and v.get("type") == "text" and isinstance(v.get("text"), str) for v in value
+        ):
+            return _parse_json_string("".join(v["text"] for v in value))
+        return value or None
+    if isinstance(value, str):
+        return _parse_json_string(value)
+    return None
+
+
+def _parse_json_string(text: str) -> dict[str, Any] | list[Any] | None:
+    """Parse ``text`` as JSON when it starts (after ``lstrip``) with ``{`` or
+    ``[``. Tolerates trailing garbage via ``raw_decode``. Rejects empty
+    containers so accidental matches don't suppress the ``str()`` fallback.
+    Does NOT strip leading non-whitespace prefix lines — strings like
+    ``"Warning: foo\\n{...}"`` fall through to ``str()``.
+    """
+    stripped = text.lstrip()
+    if not stripped or stripped[0] not in "{[":
+        return None
+    try:
+        parsed, _ = json.JSONDecoder().raw_decode(stripped)
+    except json.JSONDecodeError:
+        return None
+    if isinstance(parsed, (dict, list)) and parsed:
+        return parsed
+    return None
+
+
+def _truncate(text: str, max_chars: int) -> str:
+    """Truncate ``text`` to ``max_chars``, appending a visible marker."""
+    if len(text) <= max_chars:
+        return text
+    dropped = len(text) - max_chars
+    return f"{text[:max_chars]}…({dropped} more chars)"
+
+
+def format_token_usage(usage: Any) -> str:
+    """Format TokenUsage into a short display string like 'in=1234, out=567'.
+
+    Returns empty string if usage is None or has no token counts.
+    Includes cache_read_input_tokens if non-zero.
+
+    ``in=`` shows ``uncached_input_tokens`` (the fresh, billed-at-input-rate
+    slice), NOT the derived ``input_tokens`` total — the total already folds in
+    ``cache_read``, so using it here while also appending ``cached=`` would
+    double-count the cached prefix. Falls back to ``input_tokens`` only for
+    objects that predate the split / don't expose the uncached field.
+    """
+    if usage is None:
+        return ""
+    try:
+        input_tokens = getattr(usage, "uncached_input_tokens", None)
+        if input_tokens is None:
+            input_tokens = getattr(usage, "input_tokens", 0)
+        input_tokens = input_tokens or 0
+        output_tokens = getattr(usage, "output_tokens", 0) or 0
+        if not input_tokens and not output_tokens:
+            return ""
+        parts = [f"in={input_tokens}", f"out={output_tokens}"]
+        cached = getattr(usage, "cache_read_input_tokens", 0) or 0
+        if cached:
+            parts.append(f"cached={cached}")
+        return ", ".join(parts)
+    except (AttributeError, TypeError):
+        return ""

coder_eval/isolation/__init__.py

@@ -0,0 +1,15 @@
+"""Per-task Docker isolation.
+
+The host dispatches each task as ``docker run --rm coder-eval-agent ...``.
+Inside the container, the same Orchestrator/Sandbox/Agent code runs in-process
+with ``driver: tempdir``, writes ``task.json``, and exits. The host reads
+``task.json`` from a bind-mounted output dir and feeds it to the normal
+aggregation pipeline.
+
+Aggregation (P/R/F1, suite thresholds, reports) always stays on the host.
+"""
+
+from .docker_runner import DockerRunner
+
+
+__all__ = ["DockerRunner"]