coder-eval 0.8.2__py3-none-any.whl

coder_eval/criteria/file_check.py

@@ -0,0 +1,119 @@
+"""Unified file check criterion checker (existence + contents + regex)."""
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CriterionResult, FileCheckCriterion
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+@register_criterion
+class FileCheckChecker(BaseCriterion[FileCheckCriterion]):
+    """Checker for FileCheckCriterion."""
+
+    criterion_type = "file_check"
+
+    def _check_impl(
+        self,
+        criterion: FileCheckCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        """Unified file check: existence, string includes/excludes, regex patterns.
+
+        Returns:
+            CriterionResult with fractional score based on active sub-checks
+        """
+        # 1. File existence (implicit)
+        if not sandbox.file_exists(criterion.path):
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error=f"File '{criterion.path}' does not exist",
+            )
+
+        has_includes = len(criterion.includes) > 0
+        has_excludes = len(criterion.excludes) > 0
+        has_patterns = len(criterion.patterns) > 0
+
+        # 2. Pure existence check (no sub-checks specified)
+        if not has_includes and not has_excludes and not has_patterns:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=1.0,
+                details=f"File '{criterion.path}' exists",
+            )
+
+        # 3. Read file content
+        content = sandbox.get_file_content(criterion.path)
+
+        scores: list[float] = []
+        details_parts: list[str] = []
+
+        # 4a. Includes score
+        if has_includes:
+            found = sum(1 for inc in criterion.includes if inc in content)
+            total = len(criterion.includes)
+            includes_score = found / total
+            scores.append(includes_score)
+            details_parts.append(f"Includes: {found}/{total} found")
+
+        # 4b. Excludes score
+        if has_excludes:
+            present = sum(1 for exc in criterion.excludes if exc in content)
+            total = len(criterion.excludes)
+            excludes_score = 1.0 - (present / total)
+            scores.append(excludes_score)
+            absent = total - present
+            details_parts.append(f"Excludes: {absent}/{total} absent")
+
+        # 4c. Pattern scores
+        if has_patterns:
+            pattern_scores: list[float] = []
+            pattern_details: list[str] = []
+            for p in criterion.patterns:
+                try:
+                    regex = re.compile(p.pattern, p.flags)
+                    match = regex.search(content)
+                except re.error as e:
+                    logger.warning("Invalid regex pattern '%s': %s", p.pattern, e)
+                    pattern_scores.append(0.0)
+                    pattern_details.append(f"Invalid regex '{p.pattern}': {e}")
+                    continue
+
+                if p.must_match:
+                    pattern_scores.append(1.0 if match else 0.0)
+                else:
+                    pattern_scores.append(1.0 if match is None else 0.0)
+
+            matched = sum(1 for s in pattern_scores if s == 1.0)
+            patterns_score = sum(pattern_scores) / len(pattern_scores)
+            scores.append(patterns_score)
+            detail = f"Patterns: {matched}/{len(pattern_scores)} matched"
+            if pattern_details:
+                detail += f" ({'; '.join(pattern_details)})"
+            details_parts.append(detail)
+
+        # 5. Combine active category scores
+        score = sum(scores) / len(scores)
+        details_parts.append(f"Score: {score:.2f}")
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details="; ".join(details_parts),
+        )

coder_eval/criteria/file_contains.py

@@ -0,0 +1,85 @@
+"""File contains criterion checker."""
+
+import logging
+from typing import TYPE_CHECKING
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CriterionResult, FileContainsCriterion
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+@register_criterion
+class FileContainsChecker(BaseCriterion[FileContainsCriterion]):
+    """Checker for FileContainsCriterion."""
+
+    criterion_type = "file_contains"
+
+    def _check_impl(
+        self,
+        criterion: FileContainsCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        """Check if file contains required strings and excludes forbidden ones.
+
+        Returns:
+            CriterionResult with fractional score based on includes/excludes matched
+        """
+        if not sandbox.file_exists(criterion.path):
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error=f"File '{criterion.path}' does not exist",
+            )
+
+        content = sandbox.get_file_content(criterion.path)
+
+        # Calculate includes score (fraction of includes found)
+        includes_found = sum(1 for inc in criterion.includes if inc in content)
+        includes_total = len(criterion.includes)
+        includes_score = includes_found / includes_total if includes_total > 0 else 1.0
+
+        # Calculate excludes score (fraction of excludes absent)
+        if criterion.excludes:
+            excludes_found = sum(1 for exc in criterion.excludes if exc in content)
+            excludes_total = len(criterion.excludes)
+            excludes_score = 1.0 - (excludes_found / excludes_total)
+        else:
+            excludes_score = 1.0
+
+        # Combined score: only average when both categories are active
+        has_includes = includes_total > 0
+        has_excludes = bool(criterion.excludes)
+        if has_includes and has_excludes:
+            score = (includes_score + excludes_score) / 2.0
+        elif has_includes:
+            score = includes_score
+        elif has_excludes:
+            score = excludes_score
+        else:
+            score = 1.0
+
+        # Build details
+        details_parts = []
+        details_parts.append(f"Includes: {includes_found}/{includes_total} found")
+        if criterion.excludes:
+            excludes_absent = len(criterion.excludes) - sum(1 for exc in criterion.excludes if exc in content)
+            details_parts.append(f"Excludes: {excludes_absent}/{len(criterion.excludes)} absent")
+        details_parts.append(f"Score: {score:.2f}")
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details="; ".join(details_parts),
+        )

coder_eval/criteria/file_exists.py

@@ -0,0 +1,45 @@
+"""File existence criterion checker."""
+
+import logging
+from typing import TYPE_CHECKING
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CriterionResult, FileExistsCriterion
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+@register_criterion
+class FileExistsChecker(BaseCriterion[FileExistsCriterion]):
+    """Checker for FileExistsCriterion."""
+
+    criterion_type = "file_exists"
+
+    def _check_impl(
+        self,
+        criterion: FileExistsCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        """Check if a file exists in the sandbox.
+
+        Returns:
+            CriterionResult with score 1.0 if exists, 0.0 if not
+        """
+        exists = sandbox.file_exists(criterion.path)
+        score = 1.0 if exists else 0.0
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details=f"File '{criterion.path}' {'exists' if exists else 'does not exist'}",
+        )

coder_eval/criteria/file_matches_regex.py

@@ -0,0 +1,86 @@
+"""File matches regex criterion checker."""
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CriterionResult, FileMatchesRegexCriterion
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+@register_criterion
+class FileMatchesRegexChecker(BaseCriterion[FileMatchesRegexCriterion]):
+    """Checker for FileMatchesRegexCriterion."""
+
+    criterion_type = "file_matches_regex"
+
+    def _check_impl(
+        self,
+        criterion: FileMatchesRegexCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        """Check if file content matches the regex pattern.
+
+        Args:
+            criterion: File matches regex criterion
+            sandbox: Sandbox instance for file access
+            reference_code: Not used for this criterion
+
+        Returns:
+            Result with binary score (1.0 if pattern matches as expected, 0.0 otherwise)
+        """
+        if not sandbox.file_exists(criterion.path):
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error=f"File '{criterion.path}' does not exist",
+            )
+
+        content = sandbox.get_file_content(criterion.path)
+
+        # Compile and search for pattern
+        try:
+            regex = re.compile(criterion.pattern, criterion.flags)
+            match = regex.search(content)
+        except re.error as e:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error=f"Invalid regex pattern: {e}",
+            )
+
+        # Check based on must_match flag
+        if criterion.must_match:
+            score = 1.0 if match is not None else 0.0
+            if match:
+                matched_text = match.group()[:100]
+                details = f"Pattern '{criterion.pattern}' found in file (matched: '{matched_text}')"
+            else:
+                details = f"Pattern '{criterion.pattern}' not found in file"
+        else:
+            score = 1.0 if match is None else 0.0
+            if match is None:
+                details = f"Pattern '{criterion.pattern}' correctly absent from file"
+            else:
+                matched_text = match.group()[:100]
+                details = f"Pattern '{criterion.pattern}' found but should not be present (matched: '{matched_text}')"
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details=details,
+        )

coder_eval/criteria/json_check.py

@@ -0,0 +1,184 @@
+"""JSON check criterion checker (existence + parse + schema + JMESPath assertions)."""
+
+import json
+import logging
+import re
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+import jmespath
+import jsonschema
+
+from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
+from coder_eval.models import CriterionResult, JMESPathAssertion, JsonCheckCriterion
+
+
+if TYPE_CHECKING:
+    from coder_eval.models.results import TurnRecord
+    from coder_eval.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+# JSON type name -> Python type(s) mapping for the "type" operator.
+# Task authors use JSON names (string, number, array, object, boolean, null).
+_JSON_TYPE_MAP: dict[str, type | tuple[type, ...]] = {
+    "string": str,
+    "number": (int, float),
+    "boolean": bool,
+    "array": list,
+    "object": dict,
+    "null": type(None),
+}
+
+# Operator dispatch table. Each operator takes (actual, expected) and returns bool.
+# All exceptions are caught per-assertion by the caller.
+_OPERATORS: dict[str, Callable[[Any, Any], bool]] = {
+    "equals": lambda actual, expected: actual == expected,
+    "not_equals": lambda actual, expected: actual != expected,
+    "contains": lambda actual, expected: expected in actual,
+    "gt": lambda actual, expected: actual > expected,
+    "gte": lambda actual, expected: actual >= expected,
+    "lt": lambda actual, expected: actual < expected,
+    "lte": lambda actual, expected: actual <= expected,
+    "type": lambda actual, expected: (
+        expected in _JSON_TYPE_MAP
+        and (
+            (isinstance(actual, bool) and expected == "boolean")
+            or (not isinstance(actual, bool) and isinstance(actual, _JSON_TYPE_MAP[expected]))
+        )
+    ),
+    "regex": lambda actual, expected: bool(re.search(expected, actual)),
+    "exists": lambda actual, _: actual is not None,
+}
+
+
+@register_criterion
+class JsonCheckChecker(BaseCriterion[JsonCheckCriterion]):
+    """Checker for JsonCheckCriterion."""
+
+    criterion_type = "json_check"
+
+    def _check_impl(
+        self,
+        criterion: JsonCheckCriterion,
+        sandbox: "Sandbox",
+        reference_code: str | None = None,
+        *,
+        turn_records: list["TurnRecord"] | None = None,
+        context: CheckContext | None = None,
+    ) -> CriterionResult:
+        """JSON check: existence, parse, schema validation, JMESPath assertions."""
+        # 1. File existence
+        if not sandbox.file_exists(criterion.path):
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error=f"File '{criterion.path}' does not exist",
+            )
+
+        # 2. Parse JSON
+        content = sandbox.get_file_content(criterion.path)
+        try:
+            data = json.loads(content)
+        except (json.JSONDecodeError, ValueError) as e:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=0.0,
+                error=f"Invalid JSON in '{criterion.path}': {e}",
+            )
+
+        has_schema = criterion.json_schema is not None
+        has_assertions = len(criterion.assertions) > 0
+
+        # 3. Pure validity check
+        if not has_schema and not has_assertions:
+            return CriterionResult(
+                criterion_type=criterion.type,
+                description=criterion.description,
+                score=1.0,
+                details=f"'{criterion.path}' is valid JSON",
+            )
+
+        scores: list[float] = []
+        details_parts: list[str] = []
+
+        # 4. Schema validation (gates assertions — if schema fails, skip assertions)
+        if has_schema:
+            assert criterion.json_schema is not None  # narrowing for pyright
+            schema_score, schema_detail = self._check_schema(criterion.json_schema, data, sandbox)
+            scores.append(schema_score)
+            details_parts.append(schema_detail)
+
+            if schema_score == 0.0 and has_assertions:
+                details_parts.append("Assertions: skipped (schema failed)")
+                score = 0.0
+                details_parts.append(f"Score: {score:.2f}")
+                return CriterionResult(
+                    criterion_type=criterion.type,
+                    description=criterion.description,
+                    score=score,
+                    details="; ".join(details_parts),
+                )
+
+        # 5. JMESPath assertions
+        if has_assertions:
+            assertions_score, assertions_detail = self._check_assertions(criterion.assertions, data)
+            scores.append(assertions_score)
+            details_parts.append(assertions_detail)
+
+        # 6. Average active categories
+        score = sum(scores) / len(scores)
+        details_parts.append(f"Score: {score:.2f}")
+
+        return CriterionResult(
+            criterion_type=criterion.type,
+            description=criterion.description,
+            score=score,
+            details="; ".join(details_parts),
+        )
+
+    def _check_schema(self, schema_path: str, data: Any, sandbox: "Sandbox") -> tuple[float, str]:
+        """Validate data against a JSON Schema file."""
+        if not sandbox.file_exists(schema_path):
+            return 0.0, f"Schema file '{schema_path}' not found"
+
+        schema_content = sandbox.get_file_content(schema_path)
+        try:
+            schema = json.loads(schema_content)
+        except (json.JSONDecodeError, ValueError) as e:
+            return 0.0, f"Invalid JSON in schema file '{schema_path}': {e}"
+
+        try:
+            jsonschema.validate(instance=data, schema=schema)
+            return 1.0, "Schema: valid"
+        except jsonschema.ValidationError as e:
+            return 0.0, f"Schema: invalid — {e.message}"
+        except jsonschema.SchemaError as e:
+            return 0.0, f"Schema: malformed schema — {e.message}"
+
+    def _check_assertions(self, assertions: list[JMESPathAssertion], data: Any) -> tuple[float, str]:
+        """Evaluate JMESPath assertions against parsed JSON data."""
+        passed = 0
+        total = len(assertions)
+        failed_details: list[str] = []
+
+        for assertion in assertions:
+            try:
+                actual = jmespath.search(assertion.expression, data)
+                op_func = _OPERATORS[assertion.operator]
+                if op_func(actual, assertion.expected):
+                    passed += 1
+                else:
+                    expr = assertion.expression
+                    op = assertion.operator
+                    failed_details.append(f"'{expr}' {op}: expected {assertion.expected!r}, got {actual!r}")
+            except Exception as e:
+                failed_details.append(f"'{assertion.expression}': error — {e}")
+
+        score = passed / total
+        detail = f"Assertions: {passed}/{total} passed"
+        if failed_details:
+            detail += f" ({'; '.join(failed_details)})"
+        return score, detail