coder-eval 0.8.2__py3-none-any.whl

coder_eval/evaluation/checker.py

@@ -0,0 +1,260 @@
+"""Success criterion checking with pluggable criterion types.
+
+This module provides the SuccessChecker class which orchestrates
+criterion checking using the registered criterion checkers from
+the criteria registry.
+"""
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from ..criteria import BaseCriterion, CriterionRegistry, init_criteria
+from ..criteria.base import CheckContext
+from ..errors import JudgeInfrastructureError
+from ..models import CriteriaResults, CriterionResult, SuccessCriteria, SuccessCriterion, TurnRecords
+from ..sandbox import Sandbox
+
+
+if TYPE_CHECKING:
+    from ..models.routing import ApiRoute
+
+
+# Get module logger
+logger = logging.getLogger(__name__)
+
+
+def _short_failure_reason(result: CriterionResult, max_len: int = 200) -> str:
+    """Return a one-line, length-capped failure reason for a criterion result.
+
+    Prefers ``result.error``; then falls back to the first ``stderr:`` line in
+    ``result.details``; then the first non-empty details line. Multi-line text
+    is collapsed with ``" | "`` separators before truncation. Returns
+    ``"no details"`` when nothing usable is available.
+
+    Shared by ``SuccessChecker`` (console fail log) and
+    ``orchestrator._emit_criteria_event`` (streamed ``failure_reason`` field)
+    so both surfaces show identical text.
+    """
+
+    def _collapse(text: str) -> str:
+        lines = [line.strip() for line in text.splitlines()]
+        collapsed = " | ".join(line for line in lines if line)
+        if len(collapsed) > max_len:
+            return collapsed[: max_len - 1] + "…"
+        return collapsed
+
+    if result.error:
+        return _collapse(result.error) or "no details"
+    if result.details:
+        for line in result.details.splitlines():
+            stripped = line.strip()
+            if stripped.lower().startswith("stderr:"):
+                reason = stripped[len("stderr:") :].strip()
+                if reason:
+                    return _collapse(reason)
+        for line in result.details.splitlines():
+            stripped = line.strip()
+            if stripped:
+                return _collapse(stripped)
+    return "no details"
+
+
+class SuccessChecker:
+    """Orchestrates criterion checking using registered checkers."""
+
+    def __init__(
+        self,
+        sandbox: Sandbox,
+        init_registry: bool = True,
+        validate_registry: bool = True,
+        route: "ApiRoute | None" = None,
+    ):
+        """Initialize the success checker.
+
+        Args:
+            sandbox: Sandbox instance for running checks
+            init_registry: Whether to initialize the criteria registry
+            validate_registry: Whether to validate all expected types are registered
+            route: Resolved ``ApiRoute`` from the orchestrator. Forwarded to every
+                checker's ``check()`` so criteria that spawn sub-agents (e.g.
+                ``agent_judge``) can route through the same backend (Direct /
+                Bedrock) as the main coding agent. ``None`` is acceptable
+                for non-sub-agent criteria; ``agent_judge`` requires a route.
+        """
+        self.sandbox = sandbox
+        self._checker_instances: dict[str, BaseCriterion[Any]] = {}
+        # Cached reference code - automatically set by check()/check_all() when provided
+        # Used by subsequent check() calls that don't explicitly pass reference_code
+        self._reference_code: str | None = None
+        # Cached reference directory path (resolved). Set by check()/check_all() when provided.
+        # Mutually exclusive with self._reference_code at the task level — task.reference
+        # is exactly one of code/file/directory.
+        self._reference_dir: Path | None = None
+        # Cached turn records - set by check()/check_all() when provided
+        self._turn_records: TurnRecords | None = None
+        self.route = route
+
+        # V3: Lazy initialization - registry loaded here, not at import
+        if init_registry:
+            init_criteria(validate=validate_registry)
+
+    def check(
+        self,
+        criterion: SuccessCriterion,
+        reference_code: str | None = None,
+        turn_records: TurnRecords | None = None,
+        reference_dir: Path | None = None,
+    ) -> CriterionResult:
+        """Check a single criterion (backward compatibility wrapper).
+
+        Args:
+            criterion: Criterion definition
+            reference_code: Optional reference code (string form: from
+                ``task.reference.code`` or ``task.reference.file``).
+            turn_records: Optional turn records for command inspection
+            reference_dir: Optional resolved path to a reference directory
+                (from ``task.reference.directory``). Only consumed by
+                ``agent_judge``; non-judge criteria ignore it.
+
+        Returns:
+            CriterionResult with score
+        """
+        # Persist reference_code / reference_dir for subsequent calls (backward compat)
+        if reference_code is not None:
+            self._reference_code = reference_code
+        if reference_dir is not None:
+            self._reference_dir = reference_dir
+        if turn_records is not None:
+            self._turn_records = turn_records
+        ref_code = reference_code if reference_code is not None else self._reference_code
+        ref_dir = reference_dir if reference_dir is not None else self._reference_dir
+        records = turn_records if turn_records is not None else self._turn_records
+        return self._check_single(criterion, ref_code, records, ref_dir)
+
+    def check_all(
+        self,
+        criteria: SuccessCriteria,
+        reference_code: str | None = None,
+        turn_records: TurnRecords | None = None,
+        reference_dir: Path | None = None,
+    ) -> CriteriaResults:
+        """Check all success criteria.
+
+        Args:
+            criteria: List of criterion definitions
+            reference_code: Optional reference code (string form).
+            turn_records: Optional turn records for command inspection
+            reference_dir: Optional resolved path to a reference directory.
+                Only consumed by ``agent_judge``; non-judge criteria ignore it.
+
+        Returns:
+            List of criterion results with scores
+        """
+        # Persist for subsequent check() calls
+        if reference_code is not None:
+            self._reference_code = reference_code
+        if reference_dir is not None:
+            self._reference_dir = reference_dir
+        if turn_records is not None:
+            self._turn_records = turn_records
+        ref_code = reference_code if reference_code is not None else self._reference_code
+        ref_dir = reference_dir if reference_dir is not None else self._reference_dir
+        records = turn_records if turn_records is not None else self._turn_records
+        results = []
+        for criterion in criteria:
+            result = self._check_single(criterion, ref_code, records, ref_dir)
+            results.append(result)
+        return results
+
+    def _get_checker_instance(self, criterion_type: str) -> BaseCriterion[Any]:
+        """Get or create a checker instance (V3: cached).
+
+        Args:
+            criterion_type: The criterion type
+
+        Returns:
+            Checker instance (reused within this evaluation run)
+        """
+        if criterion_type not in self._checker_instances:
+            checker_class = CriterionRegistry.get_checker(criterion_type)
+            self._checker_instances[criterion_type] = checker_class()
+        return self._checker_instances[criterion_type]
+
+    def _check_single(
+        self,
+        criterion: SuccessCriterion,
+        reference_code: str | None,
+        turn_records: TurnRecords | None = None,
+        reference_dir: Path | None = None,
+    ) -> CriterionResult:
+        """Check a single criterion using registered checker.
+
+        Args:
+            criterion: Criterion definition (discriminated union)
+            reference_code: Optional reference code (string form)
+            turn_records: Optional turn records for command inspection
+            reference_dir: Optional resolved path to a reference directory.
+
+        Returns:
+            CriterionResult with score
+        """
+        criterion_type = criterion.type
+
+        # V3: Broader exception handling - catches checker constructor failures too
+        try:
+            # Get cached instance
+            checker = self._get_checker_instance(criterion_type)
+            context = CheckContext(route=self.route, reference_dir=reference_dir)
+            result = checker.check(
+                criterion,
+                self.sandbox,
+                reference_code,
+                turn_records=turn_records,
+                context=context,
+            )
+            result.pass_threshold = criterion.pass_threshold
+
+            logger.debug(f"Criterion '{criterion_type}' score: {result.score:.2f}")
+            if result.score < criterion.pass_threshold:
+                logger.info(
+                    "Criterion '%s' FAILED (score=%.2f, threshold=%.2f): %s",
+                    criterion_type,
+                    result.score,
+                    criterion.pass_threshold,
+                    _short_failure_reason(result),
+                )
+            return result
+
+        except KeyError:
+            # No checker registered for this type - return failed result for consistency.
+            # ERROR record carries enough context — skip the companion FAILED line.
+            logger.error(f"No checker found for criterion type '{criterion_type}'")
+            return CriterionResult(
+                criterion_type=criterion_type,
+                description=criterion.description,
+                score=0.0,
+                details=f"No checker registered for criterion type '{criterion_type}'",
+                error=f"Unsupported criterion type: '{criterion_type}'",
+                pass_threshold=criterion.pass_threshold,
+            )
+        except JudgeInfrastructureError:
+            raise  # judge infra failure escalates to FinalStatus.ERROR; do not score it
+        except Exception as e:
+            # V3: Catch ALL exceptions, including checker __init__ failures
+            logger.exception(f"Checker failure for criterion '{criterion_type}': {e}")
+            failed = CriterionResult(
+                criterion_type=criterion_type,
+                description=criterion.description,
+                score=0.0,
+                details="Error running checker",
+                error=str(e),
+                pass_threshold=criterion.pass_threshold,
+            )
+            logger.info(
+                "Criterion '%s' FAILED (score=0.00, threshold=%.2f): %s",
+                criterion_type,
+                criterion.pass_threshold,
+                _short_failure_reason(failed),
+            )
+            return failed

coder_eval/evaluation/judge_anthropic.py

@@ -0,0 +1,55 @@
+"""Single-completion Anthropic-SDK invoker for the Direct backend.
+
+Hits api.anthropic.com using ANTHROPIC_API_KEY from env.
+
+The judge issues a forced ``submit_verdict`` tool call; the SDK response is
+converted to a dict via ``model_dump`` so the caller can reuse
+``extract_verdict_from_anthropic_response`` — Anthropic SDK and Bedrock share
+Anthropic's native message shape (content blocks with ``type: tool_use``).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from anthropic import Anthropic, APIError
+
+from coder_eval.errors import JudgeInfrastructureError
+from coder_eval.evaluation.judge_models import to_anthropic_alias
+
+
+def invoke_anthropic_judge(
+    *,
+    model: str,
+    system: str,
+    user: str,
+    temperature: float,
+    max_tokens: int,
+    tool_spec: dict[str, Any],
+    timeout_seconds: float = 120.0,
+) -> dict[str, Any]:
+    """One Messages call with ``tools`` + forced ``tool_choice``.
+
+    Returns the SDK response converted to a dict via ``model_dump`` so the
+    caller can reuse ``extract_verdict_from_anthropic_response`` — Anthropic's
+    native shape (content blocks with ``type: tool_use``) is identical
+    between this SDK call and the Bedrock httpx-direct call.
+    """
+    alias = to_anthropic_alias(model)
+    client = Anthropic(timeout=timeout_seconds)
+    with client:
+        try:
+            response = client.messages.create(
+                model=alias,
+                system=system,
+                messages=[{"role": "user", "content": user}],
+                max_tokens=max_tokens,
+                temperature=temperature,
+                tools=[tool_spec],  # type: ignore[arg-type]
+                tool_choice={"type": "tool", "name": tool_spec["name"]},
+            )
+        except APIError as e:
+            # The SDK already retries transient failures internally (2 attempts
+            # by default) — do not add another retry loop here.
+            raise JudgeInfrastructureError(f"Anthropic judge API error: {e}") from e
+    return response.model_dump()

coder_eval/evaluation/judge_bedrock.py

@@ -0,0 +1,114 @@
+"""Single-completion Bedrock invoker for judge-style criteria.
+
+Narrow on purpose: one POST to bedrock-runtime, bearer-token auth, no streaming.
+The judge issues a forced ``submit_verdict`` tool call; we return the parsed
+response dict so the caller can walk ``content`` for ``tool_use`` blocks via
+``extract_verdict_from_anthropic_response``.
+
+Transient failures (transport errors, 429 throttles, 5xx) are retried with
+jittered exponential backoff; exhaustion and non-retryable failures raise
+``JudgeInfrastructureError`` so the row escalates to ``FinalStatus.ERROR``
+instead of being scored 0.0.
+
+agent_judge uses the Claude Code SDK subprocess instead — the two paths
+intentionally do not share an HTTP client.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+import httpx
+
+from coder_eval.errors import JudgeInfrastructureError
+from coder_eval.errors.categories import RetryConfig
+from coder_eval.errors.retry import compute_backoff
+from coder_eval.evaluation.judge_models import to_bedrock_model
+from coder_eval.models import BedrockRoute
+
+
+logger = logging.getLogger(__name__)
+
+_JUDGE_RETRY = RetryConfig(max_retries=3, initial_delay=2.0, backoff_multiplier=2.0)
+
+
+def _is_retryable_status(status_code: int) -> bool:
+    """Throttles and server-side failures are worth retrying; 4xx (auth, bad request) are not."""
+    return status_code == 429 or status_code >= 500
+
+
+def invoke_bedrock_judge(
+    *,
+    route: BedrockRoute,
+    model: str,
+    system: str,
+    user: str,
+    temperature: float,
+    max_tokens: int,
+    tool_spec: dict[str, Any],
+    timeout_seconds: float = 120.0,
+) -> dict[str, Any]:
+    """POST a Bedrock Messages request with ``tools`` + forced ``tool_choice``.
+
+    Returns the parsed response dict so the caller can walk ``content`` for
+    ``tool_use`` blocks via ``extract_verdict_from_anthropic_response``.
+
+    Transport errors, 429, and 5xx responses are retried up to
+    ``_JUDGE_RETRY.max_retries`` times with jittered exponential backoff.
+
+    Raises:
+        ValueError: ``model`` empty.
+        JudgeInfrastructureError: retries exhausted, non-retryable HTTP failure
+            (e.g. 400/401/403), or a non-dict JSON body.
+    """
+    qualified = to_bedrock_model(model, route.region)
+    url = f"https://bedrock-runtime.{route.region}.amazonaws.com/model/{qualified}/invoke"
+    body = {
+        "anthropic_version": "bedrock-2023-05-31",
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+        "system": system,
+        "messages": [{"role": "user", "content": user}],
+        "tools": [tool_spec],
+        "tool_choice": {"type": "tool", "name": tool_spec["name"]},
+    }
+    headers = {
+        "Authorization": f"Bearer {route.bearer_token}",
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+    # invoke_bedrock_judge is sync (called via asyncio.to_thread), so a plain
+    # time.sleep between attempts is correct here.
+    attempts = _JUDGE_RETRY.max_retries + 1
+    last_failure = ""
+    last_exc: Exception | None = None
+    for attempt in range(attempts):
+        if attempt:
+            time.sleep(compute_backoff(_JUDGE_RETRY, attempt - 1))
+        try:
+            response = httpx.post(url, headers=headers, json=body, timeout=timeout_seconds)
+        except httpx.HTTPError as e:
+            last_failure = f"Bedrock invoke transport error: {e}"
+            last_exc = e
+            logger.warning("Bedrock judge attempt %d/%d failed: %s", attempt + 1, attempts, last_failure)
+            continue
+        if _is_retryable_status(response.status_code):
+            last_failure = f"Bedrock invoke failed: {response.status_code} {response.text[:500]}"
+            last_exc = None
+            logger.warning("Bedrock judge attempt %d/%d failed: %s", attempt + 1, attempts, last_failure)
+            continue
+        if response.status_code >= 300:
+            raise JudgeInfrastructureError(f"Bedrock invoke failed: {response.status_code} {response.text[:500]}")
+        try:
+            data = response.json()
+        except ValueError as e:
+            # A malformed/truncated 2xx body (flaky proxy/gateway) is infra, not
+            # agent quality — escalate like the non-dict arm below.
+            raise JudgeInfrastructureError(f"Bedrock response is not valid JSON: {e}") from e
+        if not isinstance(data, dict):
+            raise JudgeInfrastructureError(f"Bedrock response is not a JSON object: {str(data)[:500]}")
+        return data
+    raise JudgeInfrastructureError(f"{last_failure} (after {attempts} attempts)") from last_exc