coder-eval 0.8.2__py3-none-any.whl

coder_eval/criteria/base.py

@@ -0,0 +1,248 @@
+"""Base criterion checker interface with error handling."""
+
+import logging
+import os
+import statistics
+import traceback
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass
+from functools import wraps
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from coder_eval.errors import JudgeInfrastructureError
+from coder_eval.models import BaseSuccessCriterion, CriterionAggregate, CriterionResult
+
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.models.routing import ApiRoute
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class CheckContext:
+    """Live run context forwarded to every criterion checker's ``_check_impl``.
+
+    Bundles the pieces of orchestrator state that judge-style criteria
+    (``llm_judge`` / ``agent_judge``) need to route their own LLM calls. Carries
+    a live object (the resolved ``route``), so it is NOT a ``coder_eval.models``
+    Pydantic model — it never gets serialized into a result record.
+
+    Non-judge checkers receive it too (uniform ``_check_impl`` signature) and
+    ignore it.
+    """
+
+    route: "ApiRoute | None" = None
+    reference_dir: "Path | None" = None
+
+
+def handle_criterion_errors(func: Callable[..., CriterionResult]) -> Callable[..., CriterionResult]:
+    """Decorator to handle errors in criterion checkers.
+
+    Wraps checker methods to catch exceptions and return a failed
+    CriterionResult with error details instead of raising.
+
+    This is the CENTRALIZED error handling that was in evaluator.py.
+    """
+
+    @wraps(func)
+    def wrapper(
+        self: Any,
+        criterion: BaseSuccessCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        turn_records: list["TurnRecord"] | None = None,
+        context: "CheckContext | None" = None,
+    ) -> CriterionResult:
+        try:
+            return func(
+                self,
+                criterion,
+                sandbox,
+                reference_code,
+                turn_records=turn_records,
+                context=context,
+            )
+        except JudgeInfrastructureError:
+            # Judge infra failure is NOT an agent failure — do not score it 0.0.
+            # Propagates to Orchestrator.run()'s broad except → FinalStatus.ERROR.
+            raise
+        except Exception as e:
+            exc_info = f"{e.__class__.__name__}: {e}"
+            tb = ""
+            if os.getenv("CODER_EVAL_DEBUG") == "1":
+                tb = "\n" + "".join(traceback.format_exc(limit=5))
+
+            criterion_type = criterion.type
+            logger.error(
+                f"Error in {self.__class__.__name__}.check() for criterion type '{criterion_type}': {exc_info}",
+                exc_info=True,  # Adds full stack trace to logs
+            )
+            return CriterionResult(
+                criterion_type=criterion_type,
+                description=criterion.description,
+                score=0.0,
+                details=f"Error during check: {exc_info}{tb}",
+                error=exc_info,  # Include exception type and message
+            )
+
+    return wrapper
+
+
+class BaseCriterion[C: BaseSuccessCriterion](ABC):
+    """Abstract base class for all criterion checkers.
+
+    Each criterion checker must:
+    1. Define `criterion_type` as a ClassVar[str] matching the discriminator
+    2. Implement `_check_impl()` with the actual checking logic
+
+    The `check()` method is final and applies centralized error handling.
+
+    Type parameter C binds the checker to its specific criterion model for
+    better IDE support and static type checking.
+
+    Example:
+        @register_criterion
+        class FileExistsChecker(BaseCriterion[FileExistsCriterion]):
+            criterion_type = "file_exists"
+
+            def _check_impl(
+                self,
+                criterion: FileExistsCriterion,
+                sandbox: Sandbox,
+                reference_code: str | None = None,
+            ) -> CriterionResult:
+                # Implementation here
+                pass
+    """
+
+    # Subclasses MUST define this as a class variable
+    criterion_type: ClassVar[str]
+
+    @handle_criterion_errors
+    def check(
+        self,
+        criterion: C,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        turn_records: list["TurnRecord"] | None = None,
+        context: "CheckContext | None" = None,
+    ) -> CriterionResult:
+        """Execute the criterion check with centralized error handling.
+
+        This method is FINAL - subclasses must NOT override it.
+        Implement _check_impl() instead.
+
+        Args:
+            criterion: The specific criterion definition (Pydantic model)
+            sandbox: Sandbox instance for file access and command execution
+            reference_code: Optional reference code string for comparison
+            turn_records: Optional list of turn records for command inspection
+            context: Optional :class:`CheckContext` carrying the live run state
+                (``route`` / ``reference_dir``) that judge criteria
+                (``agent_judge``, ``llm_judge``) consume. Non-judge checkers
+                accept the uniform signature and ignore it.
+
+        Returns:
+            CriterionResult with score (0.0-1.0), details, and error info
+        """
+        return self._check_impl(
+            criterion,
+            sandbox,
+            reference_code,
+            turn_records=turn_records,
+            context=context,
+        )
+
+    @abstractmethod
+    def _check_impl(
+        self,
+        criterion: C,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: "CheckContext | None" = None,
+    ) -> CriterionResult:
+        """Implement the actual criterion checking logic.
+
+        Subclasses override this method, NOT check(). Every checker shares this
+        uniform signature; non-judge checkers accept ``context`` and ignore it.
+
+        Args:
+            criterion: The specific criterion definition (Pydantic model)
+            sandbox: Sandbox instance for file access and command execution
+            reference_code: Optional reference code string for comparison
+            turn_records: Optional list of turn records for command inspection
+            context: Optional :class:`CheckContext` (route / reference_dir).
+                Consumed by ``llm_judge`` / ``agent_judge``; ignored by
+                the rest.
+
+        Returns:
+            CriterionResult with score (0.0-1.0), details, and error info
+
+        Raises:
+            Any exception - will be caught by @handle_criterion_errors decorator
+        """
+        pass
+
+    def aggregate(
+        self,
+        criterion: C,
+        per_row_results: list[CriterionResult],
+    ) -> CriterionAggregate | None:
+        """Across-row aggregate for dataset-backed tasks.
+
+        Default implementation emits summary statistics over the per-row scores:
+        ``count``, ``mean``, ``median``, ``std``, ``min``, ``max``. This gives
+        every criterion a thresholdable baseline for free (e.g. a ``file_exists``
+        task can gate on ``suite_thresholds: {mean: 0.9}``).
+
+        Returns ``None`` only when there are no per-row results (empty dataset).
+
+        Subclasses with richer signals (classification, per-label metrics, etc.)
+        should override and call ``super().aggregate(...)`` to inherit these
+        baseline stats, then merge their own metrics and details on top.
+        """
+        if not per_row_results:
+            return None
+
+        scores = [r.score for r in per_row_results]
+        std = statistics.pstdev(scores) if len(scores) > 1 else 0.0
+        metrics: dict[str, float] = {
+            "count": float(len(scores)),
+            "mean": statistics.fmean(scores),
+            "median": statistics.median(scores),
+            "std": std,
+            "min": min(scores),
+            "max": max(scores),
+        }
+        return CriterionAggregate(
+            criterion_type=criterion.type,
+            metrics=metrics,
+            threshold_checks=[],
+            passed=True,
+            details={},
+        )
+
+
+# Decorator for registration (defined here to avoid circular imports)
+def register_criterion(cls: type[BaseCriterion[Any]]) -> type[BaseCriterion[Any]]:
+    """Decorator to register a criterion checker.
+
+    Moved here from __init__.py to prevent circular import issues.
+
+    Usage:
+        @register_criterion
+        class MyChecker(BaseCriterion[MyCriterion]):
+            criterion_type = "my_type"
+            ...
+    """
+    from coder_eval.criteria import CriterionRegistry
+
+    return CriterionRegistry.register(cls)

coder_eval/criteria/classification_match.py

@@ -0,0 +1,107 @@
+"""Classification-match criterion checker."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from coder_eval.criteria._classification_aggregate import overlay_classification_metrics
+from coder_eval.criteria.base import BaseCriterion, register_criterion
+from coder_eval.models import (
+    ClassificationCriterionResult,
+    ClassificationMatchCriterion,
+    CriterionAggregate,
+    CriterionResult,
+)
+
+
+if TYPE_CHECKING:
+    from coder_eval.criteria.base import CheckContext
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+# Sentinels for the observed label — surfaced as their own classes in the
+# suite rollup's confusion matrix so "didn't write anything" and "wrote
+# something unrecognisable" are visible failure modes rather than vanishing.
+_SENTINEL_NONE = "(none)"
+_SENTINEL_OTHER = "(other)"
+
+
+@register_criterion
+class ClassificationMatchChecker(BaseCriterion[ClassificationMatchCriterion]):
+    """Checker for ClassificationMatchCriterion.
+
+    Reads a file written by the agent, extracts its single-line label, and
+    compares it to the ground-truth label. Populates ``observed_label`` and
+    ``expected_label`` on the result so the suite rollup can compute per-class
+    precision / recall / F1 and a confusion matrix.
+    """
+
+    criterion_type = "classification_match"
+
+    def _check_impl(
+        self,
+        criterion: ClassificationMatchCriterion,
+        sandbox: Sandbox,
+        reference_code: str | None = None,
+        *,
+        turn_records: list[TurnRecord] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        expected = self._canonicalize(criterion.expected_label, criterion)
+
+        try:
+            raw = sandbox.get_file_content(criterion.path)
+        except FileNotFoundError:
+            observed = _SENTINEL_NONE
+        else:
+            content = raw.strip()
+            observed = self._canonicalize(content, criterion) if content else _SENTINEL_NONE
+            if observed not in {*criterion.allowed_labels, _SENTINEL_NONE}:
+                # Content didn't canonicalize to a known label.
+                observed = _SENTINEL_OTHER
+
+        score = 1.0 if observed == expected else 0.0
+        details = f"observed={observed!r}, expected={expected!r}"
+
+        return ClassificationCriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details=details,
+            observed_label=observed,
+            expected_label=expected,
+        )
+
+    @staticmethod
+    def _canonicalize(value: str, criterion: ClassificationMatchCriterion) -> str:
+        """Map a raw string to its canonical form in ``allowed_labels``.
+
+        When ``case_sensitive=False`` and the value matches an allowed label
+        up to case, return the canonical casing from ``allowed_labels``.
+        Otherwise return the value as-is; callers decide whether to replace
+        it with a sentinel.
+        """
+        value = value.strip()
+        if criterion.case_sensitive:
+            return value
+        lower_map = {label.lower(): label for label in criterion.allowed_labels}
+        return lower_map.get(value.lower(), value)
+
+    def aggregate(
+        self,
+        criterion: ClassificationMatchCriterion,
+        per_row_results: list[CriterionResult],
+    ) -> CriterionAggregate | None:
+        """Baseline stats (from super) + sklearn-style classification metrics.
+
+        Delegates the classification math to
+        ``overlay_classification_metrics`` so skill_triggered and any future
+        label-emitting criterion share one implementation.
+        """
+        base = super().aggregate(criterion, per_row_results)
+        if base is None:
+            return None
+        return overlay_classification_metrics(base, per_row_results)

coder_eval/criteria/command_executed.py

@@ -0,0 +1,181 @@
+"""Command executed criterion checker."""
+
+import json
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CommandExecutedCriterion, CriterionResult
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+# Limit regex search input length to mitigate ReDoS on large command strings
+_MAX_PATTERN_SEARCH_LEN = 2000
+
+
+@register_criterion
+class CommandExecutedChecker(BaseCriterion[CommandExecutedCriterion]):
+    """Checker for CommandExecutedCriterion.
+
+    Inspects agent CommandTelemetry records to verify specific
+    tools/commands were used during evaluation.
+    """
+
+    criterion_type = "command_executed"
+
+    def _check_impl(
+        self,
+        criterion: CommandExecutedCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        """Check if the agent executed commands matching the criterion filters.
+
+        Scoring is fractional: min(1.0, matching_count / min_count).
+
+        Args:
+            criterion: Command executed criterion with filters
+            sandbox: Sandbox instance (not used for this criterion)
+            reference_code: Not used for this criterion
+            turn_records: List of turn records containing command telemetry
+
+        Returns:
+            CriterionResult with fractional score based on matching commands
+        """
+        if turn_records is None:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                details="No turn records available",
+                error="turn_records not provided to checker",
+            )
+
+        all_commands = [cmd for turn in turn_records for cmd in turn.commands]
+
+        # Note: do NOT short-circuit when ``all_commands`` is empty. A
+        # negative-assertion criterion (``min_count: 0`` + ``max_count: 0``)
+        # SHOULD pass here — zero commands trivially satisfies "must not
+        # call X". Falling through into the matching loop sets
+        # ``match_count = 0`` and the scoring branch handles both shapes.
+
+        # Compile regex patterns if provided.
+        # re.DOTALL so `.` matches newlines — agents commonly write multi-line bash
+        # commands using backslash line-continuation (e.g. `uip ... \\\n  --body ...`),
+        # and patterns like `foo.*--body` must span those line breaks.
+        pattern: re.Pattern[str] | None = None
+        if criterion.command_pattern is not None:
+            try:
+                pattern = re.compile(criterion.command_pattern, re.DOTALL)
+            except re.error as e:
+                return CriterionResult(
+                    criterion_type=criterion.type,
+                    description=criterion.description,
+                    score=0.0,
+                    details=f"Invalid command_pattern regex: {e}",
+                    error=f"Invalid regex: {e}",
+                )
+
+        exclude_re: re.Pattern[str] | None = None
+        if criterion.exclude_pattern is not None:
+            try:
+                exclude_re = re.compile(criterion.exclude_pattern, re.DOTALL)
+            except re.error as e:
+                return CriterionResult(
+                    criterion_type=criterion.type,
+                    description=criterion.description,
+                    score=0.0,
+                    details=f"Invalid exclude_pattern regex: {e}",
+                    error=f"Invalid regex: {e}",
+                )
+
+        # Filter and count matching commands
+        matching_commands: list[str] = []
+        for cmd in all_commands:
+            # Filter by tool name
+            if criterion.tool_name is not None and cmd.tool_name != criterion.tool_name:
+                continue
+
+            # Filter by success status
+            if criterion.require_success and cmd.result_status != "success":
+                continue
+
+            # Extract text for pattern matching (Bash: command param; others: JSON-serialized params)
+            if cmd.tool_name == "Bash" and cmd.parameters.get("command"):
+                cmd_text = cmd.parameters["command"]
+            else:
+                cmd_text = json.dumps(cmd.parameters)
+
+            # Truncate to mitigate ReDoS on large command strings
+            if len(cmd_text) > _MAX_PATTERN_SEARCH_LEN:
+                cmd_text = cmd_text[:_MAX_PATTERN_SEARCH_LEN]
+
+            # Filter by command pattern
+            if pattern is not None and not pattern.search(cmd_text):
+                continue
+
+            # Apply exclusion pattern (skip commands matching the exclusion)
+            if exclude_re is not None and exclude_re.search(cmd_text):
+                continue
+
+            # Build a display label for the matched command
+            if cmd.tool_name == "Bash" and cmd.parameters.get("command"):
+                label = cmd.parameters["command"]
+            else:
+                label = f"{cmd.tool_name}({json.dumps(cmd.parameters)[:80]})"
+            matching_commands.append(label)
+
+        match_count = len(matching_commands)
+
+        # Score model:
+        #   max_count is None  →  fractional towards min_count (legacy behavior).
+        #                         When min_count == 0, the criterion is trivially
+        #                         satisfied (no minimum to hit) and scores 1.0.
+        #   max_count is set   →  binary in-range. Pass iff
+        #                         min_count <= match_count <= max_count.
+        # The "negative assertion" pattern (min_count: 0, max_count: 0) drops
+        # naturally out of the binary branch — it now expresses "must NOT match"
+        # exactly. The model validator already rejects max_count < min_count.
+        if criterion.max_count is None:
+            score = 1.0 if criterion.min_count == 0 else min(1.0, match_count / criterion.min_count)
+        else:
+            score = 1.0 if criterion.min_count <= match_count <= criterion.max_count else 0.0
+
+        # Build details
+        filters = []
+        if criterion.tool_name is not None:
+            filters.append(f"tool_name={criterion.tool_name}")
+        if criterion.command_pattern is not None:
+            filters.append(f"pattern=/{criterion.command_pattern}/")
+        if criterion.require_success:
+            filters.append("require_success=True")
+        if criterion.exclude_pattern is not None:
+            filters.append(f"exclude=/{criterion.exclude_pattern}/")
+        filter_text = ", ".join(filters) if filters else "none"
+
+        if criterion.max_count is None:
+            range_text = f"{match_count}/{criterion.min_count} required"
+        else:
+            range_text = f"{match_count} matches (allowed range {criterion.min_count}..{criterion.max_count})"
+        details = f"Matched {range_text} commands (filters: {filter_text})"
+        if matching_commands:
+            # Show up to 3 example matches
+            examples = matching_commands[:3]
+            truncated = [ex[:120] for ex in examples]
+            details += f"\nExamples: {truncated}"
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details=details,
+        )

coder_eval/criteria/commands_efficiency.py

@@ -0,0 +1,73 @@
+"""Commands efficiency criterion checker."""
+
+import logging
+from typing import TYPE_CHECKING, ClassVar
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CommandsEfficiencyCriterion, CriterionResult
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+def compute_commands_efficiency(actual: int, expected: int) -> float:
+    """Compute commands efficiency score.
+
+    Returns expected / max(actual, expected), or 0.0 if actual is 0.
+    Raises ValueError if expected < 1 (callers must validate).
+    """
+    if expected < 1:
+        raise ValueError(f"expected must be >= 1, got {expected}")
+    if actual == 0:
+        return 0.0
+    return expected / max(actual, expected)
+
+
+@register_criterion
+class CommandsEfficiencyChecker(BaseCriterion[CommandsEfficiencyCriterion]):
+    """Checker for CommandsEfficiencyCriterion."""
+
+    criterion_type: ClassVar[str] = "commands_efficiency"
+
+    def _check_impl(
+        self,
+        criterion: CommandsEfficiencyCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        if turn_records is None:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error="turn_records not provided to checker",
+                details="No turn records available",
+            )
+
+        actual = sum(len(t.commands) for t in turn_records)
+        if actual == 0:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                details="No commands recorded (agent may not have run)",
+            )
+
+        score = compute_commands_efficiency(actual, criterion.expected_commands)
+        over = actual - criterion.expected_commands
+        budget_note = " (at or under budget)" if over <= 0 else f" (over budget by {over} commands)"
+        details = f"Commands: {actual}, expected: {criterion.expected_commands}, efficiency: {score:.3f}{budget_note}"
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details=details,
+        )