coder-eval 0.8.2__py3-none-any.whl

coder_eval/errors/categorization.py

@@ -0,0 +1,240 @@
+"""Error categorization logic using pattern matching.
+
+This module provides the categorize_error function that analyzes exceptions
+and maps them to appropriate ErrorCategory values for retry logic.
+"""
+
+import logging
+from typing import Any
+
+from .agent import AgentConfigError, AgentCrashError
+from .budget import BudgetExceededError
+from .categories import ErrorCategory
+from .timeout import EvaluationTimeoutError
+
+
+logger = logging.getLogger(__name__)
+
+
+def _categorize_by_exception_type(error: Exception, component: str) -> ErrorCategory | None:
+    """Typed-exception checks (most reliable), in precedence order.
+
+    Returns None when no typed check matches, so the caller falls through to
+    string-pattern matching. ``AgentCrashError`` is deliberately NOT checked here:
+    it is the last resort (see ``categorize_error``), after string patterns get a
+    chance to recognise auth / rate-limit / billing / content-filter signatures
+    stamped into the message. The component-gated branches also return ``None``
+    when their component doesn't match, so non-agent errors fall through to the
+    component group (e.g. a sandbox TimeoutError reaches the sandbox categories
+    with their designed retry behavior instead of dead-ending at ``UNKNOWN``).
+    """
+    # Check custom timeout exceptions first (before generic TimeoutError)
+    if isinstance(error, EvaluationTimeoutError):
+        return ErrorCategory.AGENT_TIMEOUT
+
+    # RunLimits budget breach — typed, distinct from time-based exceptions.
+    if isinstance(error, BudgetExceededError):
+        return ErrorCategory.BUDGET_EXCEEDED
+
+    # AgentConfigError subclasses RuntimeError, so this typed check must precede
+    # any string-pattern fallback that could re-categorise a missing-prerequisite
+    # message as the retryable AGENT_API_ERROR.
+    if isinstance(error, AgentConfigError):
+        return ErrorCategory.AGENT_CONFIG_ERROR
+
+    if isinstance(error, TimeoutError):
+        if component == "agent":
+            return ErrorCategory.AGENT_TIMEOUT
+        return None  # fall through: let the component group categorize it
+
+    if isinstance(error, FileNotFoundError):
+        if "task" in str(error).lower() or "yaml" in str(error).lower():
+            return ErrorCategory.TASK_NOT_FOUND
+        return None  # fall through: let the component group categorize it
+
+    if isinstance(error, MemoryError):
+        return ErrorCategory.OUT_OF_MEMORY
+
+    # Check for known SDK exceptions (if available)
+    try:
+        from anthropic import AuthenticationError, RateLimitError
+
+        if isinstance(error, RateLimitError):
+            return ErrorCategory.AGENT_RATE_LIMIT
+        if isinstance(error, AuthenticationError):
+            return ErrorCategory.AGENT_AUTH_ERROR
+    except ImportError:
+        pass  # anthropic package not installed; fall through to string matching
+
+    return None
+
+
+def _categorize_by_message(error_str: str, component: str) -> ErrorCategory | None:
+    """String-pattern matching on the already-lowercased error message.
+
+    ``error_str`` is the caller's ``str(error).lower()``. Patterns run in precedence
+    order; returns None when no pattern matches (fall through to component
+    categorization). The component-gated branches also return ``None`` for
+    non-agent components, so e.g. a sandbox pip failure mentioning "connection"
+    reaches ``PACKAGE_INSTALL_ERROR`` (retryable) instead of ``UNKNOWN``.
+    """
+    # Authentication errors
+    if any(pat in error_str for pat in ["authentication", "unauthorized", "invalid api key", "401"]):
+        return ErrorCategory.AGENT_AUTH_ERROR
+
+    # Billing/credit errors (NOT retryable - retrying wastes time)
+    # NOTE: Broad patterns like "insufficient" and "credit" are intentional. We prefer
+    # false positives (skipping retry on a non-billing error) over false negatives
+    # (wasting retries on a billing error that will never succeed).
+    if any(
+        pat in error_str
+        for pat in ["credit", "billing", "payment", "insufficient", "402", "quota exceeded", "spending limit"]
+    ):
+        return ErrorCategory.AGENT_BILLING_ERROR
+
+    # Rate limiting
+    if any(pat in error_str for pat in ["rate limit", "429", "ratelimit", "too many requests"]):
+        return ErrorCategory.AGENT_RATE_LIMIT
+
+    # Timeouts
+    if "timeout" in error_str:
+        if component == "agent":
+            return ErrorCategory.AGENT_TIMEOUT
+        return None  # fall through: let the component group categorize it
+
+    # Content filtering (Bedrock guardrails) — NOT retryable, same output will be blocked again
+    if any(pat in error_str for pat in ["content filtering policy", "content filter", "guardrail"]):
+        return ErrorCategory.AGENT_INVALID_OUTPUT
+
+    # API/Network errors
+    if any(pat in error_str for pat in ["api error", "connection", "network", "502", "503", "504"]):
+        if component == "agent":
+            return ErrorCategory.AGENT_API_ERROR
+        return None  # fall through: let the component group categorize it
+
+    # Disk errors
+    if any(pat in error_str for pat in ["disk", "no space", "enospc"]):
+        return ErrorCategory.DISK_FULL
+
+    # Memory errors
+    if any(pat in error_str for pat in ["memory", "oom", "out of memory"]):
+        return ErrorCategory.OUT_OF_MEMORY
+
+    return None
+
+
+def _categorize_by_component(error: Exception, error_str: str, component: str) -> ErrorCategory | None:
+    """Component-specific categorization for callers that supplied a component hint.
+
+    Returns None **only** when ``component`` is not one of sandbox/agent/evaluator/task,
+    so the main function reaches the final AgentCrashError / UNKNOWN fallback.
+    """
+    if component == "sandbox":
+        if "venv" in error_str or "virtualenv" in error_str:
+            return ErrorCategory.VENV_CREATION_ERROR
+        if "install" in error_str or "pip" in error_str or "package" in error_str:
+            return ErrorCategory.PACKAGE_INSTALL_ERROR
+        if "git" in error_str or "clone" in error_str:
+            return ErrorCategory.GIT_CLONE_ERROR
+        if "template" in error_str or "copy" in error_str:
+            return ErrorCategory.TEMPLATE_COPY_ERROR
+        return ErrorCategory.SANDBOX_SETUP_ERROR
+
+    if component == "agent":
+        # Substring heuristics for plain RuntimeErrors that mention a crash
+        # but aren't typed as AgentCrashError. The typed-AgentCrashError
+        # check below catches the agent's wrapped exceptions; this branch
+        # only fires for bare exceptions whose message happens to describe
+        # a crash.
+        if (
+            "crash" in error_str
+            or "killed" in error_str
+            or "cli process failed (exit code" in error_str
+            or "command failed with exit code" in error_str
+        ):
+            return ErrorCategory.AGENT_CRASH
+        if "invalid" in error_str or "malformed" in error_str:
+            return ErrorCategory.AGENT_INVALID_OUTPUT
+
+        # Typed agent-crash exception is the LAST agent-side resort: by this
+        # point pattern matching has had a chance to recognise auth / rate-
+        # limit / billing / content-filter / api-network signatures stamped
+        # into the message, so anything still typed as AgentCrashError here
+        # is a genuinely unexpected failure that the user wants retried.
+        if isinstance(error, AgentCrashError):
+            return ErrorCategory.AGENT_CRASH
+
+        # Default to API error for unknown agent failures (more likely to be retryable)
+        return ErrorCategory.AGENT_API_ERROR
+
+    if component == "evaluator":
+        return ErrorCategory.CRITERION_CHECK_ERROR
+
+    if component == "task":
+        if "invalid" in error_str or "malformed" in error_str or "validation" in error_str:
+            return ErrorCategory.TASK_INVALID
+        return ErrorCategory.TASK_NOT_FOUND
+
+    return None
+
+
+def categorize_error(
+    error: Exception,
+    context: dict[str, Any],
+    hint: ErrorCategory | None = None,
+) -> ErrorCategory:
+    """Categorize an exception into an ErrorCategory.
+
+    Uses pattern matching on error message and exception type,
+    plus context hints (component name) to determine category.
+
+    Prioritizes:
+    1. Explicit hint (if provided by caller who knows the context)
+    2. Exception type checking (most reliable)
+    3. Known SDK exceptions (if available)
+    4. String pattern matching (fallback)
+
+    Args:
+        error: The exception to categorize
+        context: Additional context dict with keys:
+            - component: "agent", "sandbox", "evaluator", etc.
+            - task_id: Task identifier (for logging)
+        hint: Optional explicit category (when caller knows the type)
+
+    Returns:
+        ErrorCategory enum value
+
+    Example:
+        >>> categorize_error(TimeoutError(), {"component": "agent"})
+        <ErrorCategory.AGENT_TIMEOUT: 'agent_timeout'>
+        >>> categorize_error(Exception("Rate limit"), {"component": "agent"})
+        <ErrorCategory.AGENT_RATE_LIMIT: 'agent_rate_limit'>
+    """
+    # Use hint if provided (caller knows best)
+    if hint:
+        return hint
+
+    component = context.get("component", "")
+
+    # Precedence: typed-exception group → message-string group → component group.
+    # A helper returning None means "no match in my group, fall through" — including
+    # the component-gated timeout/network arms for non-agent components, so a
+    # sandbox failure reaches the sandbox categories (and their retries) below.
+    if (category := _categorize_by_exception_type(error, component)) is not None:
+        return category
+
+    error_str = str(error).lower()
+    if (category := _categorize_by_message(error_str, component)) is not None:
+        return category
+
+    if (category := _categorize_by_component(error, error_str, component)) is not None:
+        return category
+
+    # Final typed-AgentCrashError fallback for callers without a component hint:
+    # nothing more specific matched, treat as a genuinely unexpected crash.
+    if isinstance(error, AgentCrashError):
+        return ErrorCategory.AGENT_CRASH
+
+    # Default to unknown
+    logger.warning(f"Could not categorize error: {error} (component={component})")
+    return ErrorCategory.UNKNOWN

coder_eval/errors/executor.py

@@ -0,0 +1,124 @@
+"""Retry executor - main retry orchestration logic."""
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from .categories import RETRY_CONFIG, RetryConfig
+from .categorization import categorize_error
+from .retry import get_error_tip, get_retry_delay, should_retry
+
+
+logger = logging.getLogger(__name__)
+
+
+async def execute_with_retry(
+    operation: Callable[[], Awaitable[Any]],
+    operation_name: str,
+    context: dict[str, Any],
+    max_attempts: int | None = None,
+    on_attempt_error: Callable[[Exception, int], Awaitable[None]] | None = None,
+) -> Any:
+    """Execute an operation with automatic retry on transient errors.
+
+    This is the core retry mechanism. It wraps any async operation and:
+    1. Executes the operation
+    2. Catches exceptions
+    3. Categorizes the error
+    4. Retries if eligible (with exponential backoff + jitter)
+    5. Re-raises after exhausting retries
+
+    Args:
+        operation: Async callable to execute (no arguments, use closures/lambdas)
+        operation_name: Human-readable operation name for logging
+        context: Context dict with:
+            - task_id: Task identifier (required)
+            - component: Component name (optional but recommended)
+            - agent_name: Agent name (optional)
+        max_attempts: Override max attempts (defaults to 10 as safety limit)
+        on_attempt_error: Async ``(exception, zero_indexed_attempt) -> None`` callback
+            invoked after every failed attempt (including the final non-retryable one),
+            for draining ``agent.pending_turn`` and calling ``agent.discard_pending_turn()``.
+            Callback exceptions are logged and swallowed so they cannot mask the original.
+
+    Returns:
+        Result from operation
+
+    Raises:
+        Last exception if all retries exhausted
+
+    Example:
+        >>> async def flaky_api_call():
+        ...     return await agent.communicate(prompt)
+        >>>
+        >>> result = await execute_with_retry(
+        ...     operation=flaky_api_call,
+        ...     operation_name="Agent communication",
+        ...     context={"task_id": "task-001", "component": "agent"},
+        ... )
+    """
+    task_id = context.get("task_id", "unknown")
+    last_error = None
+
+    # Determine max attempts (safety limit)
+    if max_attempts is None:
+        max_attempts = 10
+
+    for attempt in range(max_attempts):
+        try:
+            # Execute operation
+            return await operation()
+
+        except asyncio.CancelledError:
+            # Re-raise cancellation immediately - not an error to retry
+            raise
+
+        except Exception as e:
+            last_error = e
+
+            # Fire callback before the retry decision so partial telemetry
+            # is captured even on the final non-retryable attempt.
+            if on_attempt_error is not None:
+                try:
+                    await on_attempt_error(e, attempt)
+                except Exception:
+                    logger.exception(
+                        "[%s] on_attempt_error callback raised for %s (attempt %d); ignoring",
+                        task_id,
+                        operation_name,
+                        attempt + 1,
+                    )
+
+            # Categorize error
+            category = categorize_error(e, context)
+            config = RETRY_CONFIG.get(category, RetryConfig())
+
+            # Check if we should retry (handles both non-retryable categories and exhausted attempts)
+            if not should_retry(category, attempt):
+                if config.max_retries == 0:
+                    logger.error(f"[{task_id}] {operation_name} failed (non-retryable): {category.value} - {e}")
+                else:
+                    logger.error(
+                        f"[{task_id}] {operation_name} failed after {attempt + 1} attempts: {category.value} - {e}"
+                    )
+                raise
+
+            # Calculate backoff with jitter
+            delay = get_retry_delay(category, attempt)
+
+            # Enhanced warning with actionable tip
+            tip = get_error_tip(category)
+            message = (
+                f"[{task_id}] {operation_name} failed (attempt {attempt + 1}/{config.max_retries + 1}): "
+                f"{category.value} - {e}. Retrying in {delay:.1f}s...\n  💡 Tip: {tip}"
+            )
+            logger.warning(message)
+
+            # Async sleep (non-blocking)
+            await asyncio.sleep(delay)
+
+    # Should never reach here, but just in case
+    if last_error:
+        raise last_error
+    raise RuntimeError(f"Unexpected retry loop exit for {operation_name}")

coder_eval/errors/judge.py

@@ -0,0 +1,12 @@
+"""Typed error for judge-side infrastructure failures."""
+
+
+class JudgeInfrastructureError(RuntimeError):
+    """Judge-side infrastructure failure (transport error, throttle, 5xx, auth)
+    after bounded retries.
+
+    Deliberately NOT converted to a scored-0.0 ``CriterionResult``:
+    ``handle_criterion_errors`` and ``SuccessChecker._check_single`` re-raise it
+    so the orchestrator lands the row as ``FinalStatus.ERROR`` — a judge outage
+    must be distinguishable from genuine agent failure in pass-rate metrics.
+    """

coder_eval/errors/retry.py

@@ -0,0 +1,211 @@
+"""Retry logic with exponential backoff and error context management.
+
+This module provides retry functionality, error context creation, and helper
+utilities for production-grade error handling.
+"""
+
+import logging
+import random
+import traceback
+from datetime import datetime
+from typing import Any
+
+from .categories import ERROR_TIPS, RETRY_CONFIG, ErrorCategory, ErrorContext, RetryConfig
+from .categorization import categorize_error
+
+
+logger = logging.getLogger(__name__)
+
+
+def should_retry(category: ErrorCategory, attempt: int) -> bool:
+    """Determine if an error should be retried.
+
+    Args:
+        category: Error category
+        attempt: Current attempt number (0-indexed)
+
+    Returns:
+        True if retry should be attempted
+    """
+    config = RETRY_CONFIG.get(category, RetryConfig())
+    return attempt < config.max_retries
+
+
+def compute_backoff(config: RetryConfig, attempt: int) -> float:
+    """Calculate delay with exponential backoff and jitter.
+
+    Formula: (initial_delay * backoff_multiplier^attempt) + jitter
+    Jitter: Random value between 0 and 25% of base delay
+
+    Args:
+        config: Retry configuration (initial delay, multiplier)
+        attempt: Current attempt number (0-indexed)
+
+    Returns:
+        Delay in seconds
+    """
+    base_delay = config.initial_delay * (config.backoff_multiplier**attempt)
+
+    # Add jitter: up to 25% of base delay to prevent thundering herd
+    jitter = random.uniform(0, base_delay * 0.25)
+
+    return base_delay + jitter
+
+
+def get_retry_delay(category: ErrorCategory, attempt: int) -> float:
+    """Calculate delay before retry with exponential backoff and jitter.
+
+    Convenience wrapper around ``compute_backoff`` that looks up the
+    ``RetryConfig`` for the given error category.
+
+    Args:
+        category: Error category
+        attempt: Current attempt number (0-indexed)
+
+    Returns:
+        Delay in seconds
+
+    Example:
+        >>> get_retry_delay(ErrorCategory.AGENT_API_ERROR, 0)  # doctest: +SKIP
+        5.2  # 5.0 base + 0.2 jitter
+        >>> get_retry_delay(ErrorCategory.AGENT_API_ERROR, 1)  # doctest: +SKIP
+        10.8  # 10.0 base + 0.8 jitter
+    """
+    config = RETRY_CONFIG.get(category, RetryConfig())
+    return compute_backoff(config, attempt)
+
+
+def get_error_tip(category: ErrorCategory) -> str:
+    """Get actionable tip for error category.
+
+    Args:
+        category: Error category
+
+    Returns:
+        Actionable tip string
+    """
+    return ERROR_TIPS.get(category, "Check logs for details or run with --verbose for more information.")
+
+
+def truncate_log(log: str, max_chars: int = 1000) -> str:
+    """Truncate log preserving both start and end.
+
+    Keeps first 40% and last 60% of the log instead of just the end.
+    This preserves important startup information while still showing
+    the final error state.
+
+    Args:
+        log: Log string to truncate
+        max_chars: Maximum characters to keep (default: 1000)
+
+    Returns:
+        Truncated log string with separator if truncated
+
+    Example:
+        >>> truncate_log("x" * 100, max_chars=100)
+        'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+        >>> len(truncate_log("x" * 2000, max_chars=1000))
+        1000
+    """
+    if len(log) <= max_chars:
+        return log
+
+    # Keep first 40% and last 60% with separator
+    separator = "\n\n... [truncated] ...\n\n"
+    head_size = int(max_chars * 0.4)
+    tail_size = max(0, max_chars - head_size - len(separator))
+
+    return log[:head_size] + separator + log[-tail_size:]
+
+
+def error_context_to_dict(ctx: ErrorContext) -> dict[str, Any]:
+    """Convert ErrorContext to dict for EvaluationResult.error_details.
+
+    Args:
+        ctx: ErrorContext instance
+
+    Returns:
+        Dict representation of error context with error_tip added
+    """
+    result = ctx.model_dump()
+    # Add error tip for this error category
+    category = ErrorCategory(ctx.error_category)
+    result["error_tip"] = get_error_tip(category)
+    return result
+
+
+def create_error_context(
+    error: Exception,
+    task_id: str,
+    attempt: int,
+    component: str | None = None,
+    agent_name: str | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """Create comprehensive error context for reporting.
+
+    Internally creates a type-safe ErrorContext model, then converts to dict
+    for backward compatibility with EvaluationResult.error_details.
+
+    Args:
+        error: The exception that occurred
+        task_id: Task identifier
+        attempt: Attempt number (1-indexed)
+        component: Component that failed (agent/sandbox/evaluator)
+        agent_name: Agent name (optional)
+        **kwargs: Additional context fields:
+            - disk_usage_gb: Disk usage in GB
+            - memory_usage_gb: Memory usage in GB
+            - agent_stdout: Agent stdout (will be truncated)
+            - agent_stderr: Agent stderr (will be truncated)
+
+    Returns:
+        Dict with error context fields (from ErrorContext.model_dump())
+
+    Example:
+        >>> ctx = create_error_context(
+        ...     error=ValueError("test"),
+        ...     task_id="task-001",
+        ...     attempt=1,
+        ...     component="agent"
+        ... )
+        >>> ctx["task_id"]
+        'task-001'
+        >>> ctx["attempt_number"]
+        1
+    """
+    # Categorize error
+    category = categorize_error(error, {"component": component, "task_id": task_id})
+
+    # Truncate logs (improved: keeps start + end)
+    agent_stdout = kwargs.get("agent_stdout")
+    if agent_stdout:
+        agent_stdout = truncate_log(agent_stdout)
+
+    agent_stderr = kwargs.get("agent_stderr")
+    if agent_stderr:
+        agent_stderr = truncate_log(agent_stderr)
+
+    # Convert to 0-indexed attempt (defensive: prevent negative indexing)
+    attempt_index = max(attempt - 1, 0)
+
+    # Create type-safe ErrorContext model
+    ctx = ErrorContext(
+        error_category=category.value,
+        error_message=str(error),
+        stack_trace=traceback.format_exc(),
+        task_id=task_id,
+        component=component,
+        agent_name=agent_name,
+        attempt_number=attempt,
+        is_retryable=should_retry(category, attempt_index),
+        retry_delay_seconds=get_retry_delay(category, attempt_index) if should_retry(category, attempt_index) else 0.0,
+        disk_usage_gb=kwargs.get("disk_usage_gb"),
+        memory_usage_gb=kwargs.get("memory_usage_gb"),
+        agent_stdout=agent_stdout,
+        agent_stderr=agent_stderr,
+        timestamp=datetime.now().isoformat(),
+    )
+
+    # Return dict for backward compatibility
+    return error_context_to_dict(ctx)

coder_eval/errors/timeout.py

@@ -0,0 +1,63 @@
+"""Timeout exceptions for evaluation lifecycle."""
+
+from .agent import format_timeout_reason
+
+
+class EvaluationTimeoutError(Exception):
+    """Base timeout error for evaluation lifecycle.
+
+    Wraps asyncio.TimeoutError with structured context about
+    what timed out and where.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        timeout_seconds: float,
+        layer: str,  # "turn" | "task"
+        task_id: str | None = None,
+        iteration: int | None = None,
+        elapsed_seconds: float | None = None,
+    ):
+        super().__init__(message)
+        self.timeout_seconds = timeout_seconds
+        self.layer = layer
+        self.task_id = task_id
+        self.iteration = iteration
+        self.elapsed_seconds = elapsed_seconds
+
+
+class TurnTimeoutError(EvaluationTimeoutError):
+    """Agent turn (communicate) exceeded its time limit."""
+
+    def __init__(
+        self,
+        timeout_seconds: float,
+        *,
+        task_id: str | None = None,
+        iteration: int | None = None,
+    ):
+        message = format_timeout_reason(timeout_seconds)
+        if iteration is not None:
+            message = f"{message} (iteration {iteration})"
+        super().__init__(
+            message,
+            timeout_seconds=timeout_seconds,
+            layer="turn",
+            task_id=task_id,
+            iteration=iteration,
+        )
+
+
+class TaskTimeoutError(EvaluationTimeoutError):
+    """Overall task evaluation loop exceeded its time limit."""
+
+    def __init__(self, timeout_seconds: float, *, task_id: str | None = None, elapsed_seconds: float | None = None):
+        super().__init__(
+            f"Task timed out after {timeout_seconds}s",
+            timeout_seconds=timeout_seconds,
+            layer="task",
+            task_id=task_id,
+            elapsed_seconds=elapsed_seconds,
+        )

coder_eval/evaluation/__init__.py

@@ -0,0 +1,10 @@
+"""Evaluation components for checking success criteria and providing LLM feedback.
+
+This package contains:
+- checker: Success criterion validation with continuous scoring (0.0-1.0)
+- summaries: summarize_commands helper shared by orchestrator + llm_judge
+
+NO re-exports - use explicit imports:
+    from coder_eval.evaluation.checker import SuccessChecker
+    from coder_eval.evaluation.summaries import summarize_commands
+"""