aieval-py 0.1.0__py3-none-any.whl

aieval/__init__.py

@@ -0,0 +1,86 @@
+"""aieval: Deterministic-first runtime validation and reliability for LLM outputs and agent actions."""
+
+from aieval.checks import (
+    AllowedFieldsCheck,
+    AllowedValuesCheck,
+    AsyncCheck,
+    Check,
+    CheckResult,
+    LengthBoundsCheck,
+    MultiToolCallCheck,
+    NumericRangeCheck,
+    RegexMatchCheck,
+    RequiredFieldsCheck,
+    SchemaCheck,
+    ToolCallCheck,
+    ValidJsonCheck,
+    allowed_fields,
+    allowed_values,
+    length_bounds,
+    numeric_range,
+    regex_match,
+    required_fields,
+    schema_check,
+    tool_call_check,
+    tool_call_one_of,
+    valid_json,
+)
+from aieval.domain import (
+    EvaluationContext,
+    EvaluationError,
+    EvaluationResult,
+    EvaluationSummary,
+    Failure,
+    FailureCode,
+    Severity,
+)
+from aieval.runner import (
+    EvaluationConfigError,
+    aevaluate,
+    evaluate,
+)
+from aieval.semantic import (
+    AsyncSemanticJudge,
+    SemanticJudge,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AllowedFieldsCheck",
+    "AllowedValuesCheck",
+    "AsyncCheck",
+    "AsyncSemanticJudge",
+    "Check",
+    "CheckResult",
+    "EvaluationConfigError",
+    "EvaluationContext",
+    "EvaluationError",
+    "EvaluationResult",
+    "EvaluationSummary",
+    "Failure",
+    "FailureCode",
+    "LengthBoundsCheck",
+    "MultiToolCallCheck",
+    "NumericRangeCheck",
+    "RegexMatchCheck",
+    "RequiredFieldsCheck",
+    "SchemaCheck",
+    "SemanticJudge",
+    "Severity",
+    "ToolCallCheck",
+    "ValidJsonCheck",
+    "__version__",
+    "aevaluate",
+    "allowed_fields",
+    "allowed_values",
+    "evaluate",
+    "length_bounds",
+    "numeric_range",
+    "regex_match",
+    "required_fields",
+    "schema_check",
+    "tool_call_check",
+    "tool_call_one_of",
+    "valid_json",
+]

aieval/checks/__init__.py

@@ -0,0 +1,51 @@
+from aieval.checks.base import AsyncCheck, Check, CheckResult
+from aieval.checks.constraints import (
+    AllowedValuesCheck,
+    LengthBoundsCheck,
+    NumericRangeCheck,
+    RegexMatchCheck,
+    allowed_values,
+    length_bounds,
+    numeric_range,
+    regex_match,
+)
+from aieval.checks.fields import (
+    AllowedFieldsCheck,
+    RequiredFieldsCheck,
+    allowed_fields,
+    required_fields,
+)
+from aieval.checks.json_check import ValidJsonCheck, valid_json
+from aieval.checks.schema import SchemaCheck, schema_check
+from aieval.checks.tool_call import (
+    MultiToolCallCheck,
+    ToolCallCheck,
+    tool_call_check,
+    tool_call_one_of,
+)
+
+__all__ = [
+    "AllowedFieldsCheck",
+    "AllowedValuesCheck",
+    "AsyncCheck",
+    "Check",
+    "CheckResult",
+    "LengthBoundsCheck",
+    "MultiToolCallCheck",
+    "NumericRangeCheck",
+    "RegexMatchCheck",
+    "RequiredFieldsCheck",
+    "SchemaCheck",
+    "ToolCallCheck",
+    "ValidJsonCheck",
+    "allowed_fields",
+    "allowed_values",
+    "length_bounds",
+    "numeric_range",
+    "regex_match",
+    "required_fields",
+    "schema_check",
+    "tool_call_check",
+    "tool_call_one_of",
+    "valid_json",
+]

aieval/checks/base.py

@@ -0,0 +1,55 @@
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+from aieval.domain.context import EvaluationContext
+from aieval.domain.failure import Failure
+
+
+@dataclass(frozen=True)
+class CheckResult:
+    """The outcome of an individual check."""
+
+    passed: bool
+    failures: tuple[Failure, ...] = field(default_factory=tuple)
+    name: str = "check"
+
+    @classmethod
+    def ok(cls, name: str = "check") -> "CheckResult":
+        return cls(passed=True, failures=(), name=name)
+
+    @classmethod
+    def failed(
+        cls,
+        failures: Sequence[Failure] | Failure,
+        name: str = "check",
+    ) -> "CheckResult":
+        if isinstance(failures, Failure):
+            return cls(passed=False, failures=(failures,), name=name)
+        return cls(passed=False, failures=tuple(failures), name=name)
+
+
+@runtime_checkable
+class Check(Protocol):
+    """Protocol for synchronous validation checks."""
+
+    name: str
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult | Sequence[Failure] | None: ...
+
+
+@runtime_checkable
+class AsyncCheck(Protocol):
+    """Protocol for asynchronous validation checks."""
+
+    name: str
+
+    async def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult | Sequence[Failure] | None: ...

aieval/checks/constraints.py

@@ -0,0 +1,243 @@
+import re
+from collections.abc import Collection
+from typing import Any
+
+from aieval.checks.base import CheckResult
+from aieval.domain.context import EvaluationContext
+from aieval.domain.failure import Failure, FailureCode, Severity
+
+
+class RegexMatchCheck:
+    """Checks that a string matches a given regular expression."""
+
+    def __init__(
+        self,
+        pattern: str | re.Pattern[str],
+        *,
+        severity: Severity = Severity.ERROR,
+        name: str | None = None,
+    ) -> None:
+        if isinstance(pattern, str):
+            self.pattern = re.compile(pattern)
+            self.pattern_str = pattern
+        else:
+            self.pattern = pattern
+            self.pattern_str = pattern.pattern
+        self.severity = severity
+        self.name = name or f"regex_match({self.pattern_str})"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        if not isinstance(output, str):
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.REGEX_MISMATCH,
+                    message=f"Expected string output for regex match, got {type(output).__name__}",
+                    severity=self.severity,
+                ),
+                name=self.name,
+            )
+
+        if not self.pattern.search(output):
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.REGEX_MISMATCH,
+                    message=f"Output does not match required pattern: '{self.pattern_str}'",
+                    severity=self.severity,
+                    metadata={"pattern": self.pattern_str, "sample": output[:100]},
+                ),
+                name=self.name,
+            )
+        return CheckResult.ok(name=self.name)
+
+
+class AllowedValuesCheck:
+    """Checks that the output value belongs to a predefined set of allowed values."""
+
+    def __init__(
+        self,
+        allowed: Collection[Any],
+        *,
+        severity: Severity = Severity.ERROR,
+        name: str | None = None,
+    ) -> None:
+        self.allowed = set(allowed)
+        self.severity = severity
+        self.name = name or f"allowed_values({list(allowed)})"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        try:
+            is_member = output in self.allowed
+        except TypeError:
+            is_member = False
+
+        if not is_member:
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.ENUM_MISMATCH,
+                    message=f"Value '{output}' is not one of the allowed values: {sorted(str(x) for x in self.allowed)}",
+                    severity=self.severity,
+                    metadata={"value": str(output), "allowed": list(self.allowed)},
+                ),
+                name=self.name,
+            )
+        return CheckResult.ok(name=self.name)
+
+
+class NumericRangeCheck:
+    """Checks that a numeric value falls within [min_val, max_val]."""
+
+    def __init__(
+        self,
+        *,
+        min_val: float | None = None,
+        max_val: float | None = None,
+        severity: Severity = Severity.ERROR,
+        name: str | None = None,
+    ) -> None:
+        self.min_val = min_val
+        self.max_val = max_val
+        self.severity = severity
+        self.name = name or f"numeric_range(min={min_val}, max={max_val})"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        if not isinstance(output, (int, float)) or isinstance(output, bool):
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.VALUE_CONSTRAINT_VIOLATION,
+                    message=f"Expected numeric output, got {type(output).__name__}",
+                    severity=self.severity,
+                ),
+                name=self.name,
+            )
+
+        if self.min_val is not None and output < self.min_val:
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.VALUE_CONSTRAINT_VIOLATION,
+                    message=f"Value {output} is less than minimum allowed {self.min_val}",
+                    severity=self.severity,
+                    metadata={"value": output, "min": self.min_val},
+                ),
+                name=self.name,
+            )
+
+        if self.max_val is not None and output > self.max_val:
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.VALUE_CONSTRAINT_VIOLATION,
+                    message=f"Value {output} is greater than maximum allowed {self.max_val}",
+                    severity=self.severity,
+                    metadata={"value": output, "max": self.max_val},
+                ),
+                name=self.name,
+            )
+
+        return CheckResult.ok(name=self.name)
+
+
+class LengthBoundsCheck:
+    """Checks that the length of a string, list, or sequence falls within [min_len, max_len]."""
+
+    def __init__(
+        self,
+        *,
+        min_len: int | None = None,
+        max_len: int | None = None,
+        severity: Severity = Severity.ERROR,
+        name: str | None = None,
+    ) -> None:
+        self.min_len = min_len
+        self.max_len = max_len
+        self.severity = severity
+        self.name = name or f"length_bounds(min={min_len}, max={max_len})"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        try:
+            length = len(output)
+        except TypeError:
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.LENGTH_OUT_OF_BOUNDS,
+                    message=f"Output of type {type(output).__name__} does not have a length",
+                    severity=self.severity,
+                ),
+                name=self.name,
+            )
+
+        if self.min_len is not None and length < self.min_len:
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.LENGTH_OUT_OF_BOUNDS,
+                    message=f"Length {length} is less than minimum required {self.min_len}",
+                    severity=self.severity,
+                    metadata={"length": length, "min_len": self.min_len},
+                ),
+                name=self.name,
+            )
+
+        if self.max_len is not None and length > self.max_len:
+            return CheckResult.failed(
+                Failure(
+                    code=FailureCode.LENGTH_OUT_OF_BOUNDS,
+                    message=f"Length {length} is greater than maximum allowed {self.max_len}",
+                    severity=self.severity,
+                    metadata={"length": length, "max_len": self.max_len},
+                ),
+                name=self.name,
+            )
+
+        return CheckResult.ok(name=self.name)
+
+
+def regex_match(
+    pattern: str | re.Pattern[str],
+    *,
+    severity: Severity = Severity.ERROR,
+    name: str | None = None,
+) -> RegexMatchCheck:
+    return RegexMatchCheck(pattern, severity=severity, name=name)
+
+
+def allowed_values(
+    allowed: Collection[Any],
+    *,
+    severity: Severity = Severity.ERROR,
+    name: str | None = None,
+) -> AllowedValuesCheck:
+    return AllowedValuesCheck(allowed, severity=severity, name=name)
+
+
+def numeric_range(
+    *,
+    min_val: float | None = None,
+    max_val: float | None = None,
+    severity: Severity = Severity.ERROR,
+    name: str | None = None,
+) -> NumericRangeCheck:
+    return NumericRangeCheck(min_val=min_val, max_val=max_val, severity=severity, name=name)
+
+
+def length_bounds(
+    *,
+    min_len: int | None = None,
+    max_len: int | None = None,
+    severity: Severity = Severity.ERROR,
+    name: str | None = None,
+) -> LengthBoundsCheck:
+    return LengthBoundsCheck(min_len=min_len, max_len=max_len, severity=severity, name=name)

aieval/checks/fields.py

@@ -0,0 +1,127 @@
+import json
+from collections.abc import Sequence
+from typing import Any
+
+from aieval.checks.base import CheckResult
+from aieval.domain.context import EvaluationContext
+from aieval.domain.failure import Failure, FailureCode, Severity
+
+
+def _normalize_dict(output: Any) -> tuple[dict[str, Any] | None, Failure | None]:
+    if isinstance(output, dict):
+        return output, None
+    if isinstance(output, str):
+        try:
+            parsed = json.loads(output)
+            if isinstance(parsed, dict):
+                return parsed, None
+            return None, Failure(
+                code=FailureCode.INVALID_JSON,
+                message=f"Expected JSON object, parsed {type(parsed).__name__}",
+                severity=Severity.ERROR,
+            )
+        except json.JSONDecodeError as err:
+            return None, Failure(
+                code=FailureCode.INVALID_JSON,
+                message=f"Output is not valid JSON: {err.msg}",
+                severity=Severity.ERROR,
+            )
+    return None, Failure(
+        code=FailureCode.INVALID_JSON,
+        message=f"Output of type {type(output).__name__} is not a dictionary or JSON object",
+        severity=Severity.ERROR,
+    )
+
+
+class RequiredFieldsCheck:
+    """Checks that all specified fields are present in the output dictionary."""
+
+    def __init__(
+        self,
+        fields: Sequence[str],
+        *,
+        severity: Severity = Severity.ERROR,
+    ) -> None:
+        self.fields = tuple(fields)
+        self.severity = severity
+        self.name = f"required_fields({list(self.fields)})"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        data, err = _normalize_dict(output)
+        if err is not None:
+            return CheckResult.failed(err, name=self.name)
+        assert data is not None
+
+        missing = [f for f in self.fields if f not in data]
+        if missing:
+            failures = [
+                Failure(
+                    code=FailureCode.MISSING_REQUIRED_FIELD,
+                    message=f"Missing required field: '{field}'",
+                    severity=self.severity,
+                    path=field,
+                )
+                for field in missing
+            ]
+            return CheckResult.failed(failures, name=self.name)
+        return CheckResult.ok(name=self.name)
+
+
+class AllowedFieldsCheck:
+    """Checks that no unexpected fields are present beyond the allowed list."""
+
+    def __init__(
+        self,
+        fields: Sequence[str],
+        *,
+        severity: Severity = Severity.ERROR,
+    ) -> None:
+        self.allowed = set(fields)
+        self.severity = severity
+        self.name = f"allowed_fields({list(fields)})"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        data, err = _normalize_dict(output)
+        if err is not None:
+            return CheckResult.failed(err, name=self.name)
+        assert data is not None
+
+        unexpected = [f for f in data if f not in self.allowed]
+        if unexpected:
+            failures = [
+                Failure(
+                    code=FailureCode.UNEXPECTED_FIELD,
+                    message=f"Unexpected field present: '{field}'",
+                    severity=self.severity,
+                    path=field,
+                )
+                for field in unexpected
+            ]
+            return CheckResult.failed(failures, name=self.name)
+        return CheckResult.ok(name=self.name)
+
+
+def required_fields(
+    fields: Sequence[str],
+    *,
+    severity: Severity = Severity.ERROR,
+) -> RequiredFieldsCheck:
+    """Returns a check verifying the presence of specified fields."""
+    return RequiredFieldsCheck(fields, severity=severity)
+
+
+def allowed_fields(
+    fields: Sequence[str],
+    *,
+    severity: Severity = Severity.ERROR,
+) -> AllowedFieldsCheck:
+    """Returns a check verifying that only allowed fields are present."""
+    return AllowedFieldsCheck(fields, severity=severity)

aieval/checks/json_check.py

@@ -0,0 +1,59 @@
+import json
+from typing import Any
+
+from aieval.checks.base import CheckResult
+from aieval.domain.context import EvaluationContext
+from aieval.domain.failure import Failure, FailureCode, Severity
+
+
+class ValidJsonCheck:
+    """Checks that output is valid JSON (if string) or a valid JSON-serializable structure."""
+
+    name: str = "valid_json"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        if isinstance(output, (dict, list)):
+            return CheckResult.ok(name=self.name)
+
+        if isinstance(output, str):
+            trimmed = output.strip()
+            if not trimmed:
+                return CheckResult.failed(
+                    Failure(
+                        code=FailureCode.INVALID_JSON,
+                        message="Output is an empty string, expected valid JSON.",
+                        severity=Severity.ERROR,
+                    ),
+                    name=self.name,
+                )
+            try:
+                json.loads(trimmed)
+                return CheckResult.ok(name=self.name)
+            except json.JSONDecodeError as err:
+                return CheckResult.failed(
+                    Failure(
+                        code=FailureCode.INVALID_JSON,
+                        message=f"Output is not valid JSON: {err.msg} at line {err.lineno} col {err.colno}",
+                        severity=Severity.ERROR,
+                        metadata={"error": str(err), "pos": err.pos},
+                    ),
+                    name=self.name,
+                )
+
+        return CheckResult.failed(
+            Failure(
+                code=FailureCode.INVALID_JSON,
+                message=f"Output of type {type(output).__name__} is not a valid JSON string or object.",
+                severity=Severity.ERROR,
+            ),
+            name=self.name,
+        )
+
+
+def valid_json() -> ValidJsonCheck:
+    """Returns a check ensuring the output is valid JSON."""
+    return ValidJsonCheck()

aieval/checks/schema.py

@@ -0,0 +1,86 @@
+import json
+from typing import Any
+
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+from aieval.checks.base import CheckResult
+from aieval.domain.context import EvaluationContext
+from aieval.domain.failure import Failure, FailureCode, Severity
+
+
+class SchemaCheck:
+    """Validates output against a Pydantic v2 BaseModel or TypeAdapter."""
+
+    def __init__(
+        self,
+        schema: type[BaseModel] | TypeAdapter[Any],
+        *,
+        severity: Severity = Severity.ERROR,
+        name: str | None = None,
+    ) -> None:
+        self.schema = schema
+        self.severity = severity
+        if name is not None:
+            self.name = name
+        elif isinstance(schema, type) and issubclass(schema, BaseModel):
+            self.name = f"schema_check({schema.__name__})"
+        else:
+            self.name = "schema_check"
+
+    def __call__(
+        self,
+        output: Any,
+        context: EvaluationContext | None = None,
+    ) -> CheckResult:
+        data = output
+
+        # Auto-parse JSON string if input is string
+        if isinstance(output, str):
+            trimmed = output.strip()
+            try:
+                data = json.loads(trimmed)
+            except json.JSONDecodeError as err:
+                return CheckResult.failed(
+                    Failure(
+                        code=FailureCode.INVALID_JSON,
+                        message=f"Failed to parse JSON string for schema validation: {err.msg}",
+                        severity=self.severity,
+                        metadata={"raw": output},
+                    ),
+                    name=self.name,
+                )
+
+        try:
+            if isinstance(self.schema, type) and issubclass(self.schema, BaseModel):
+                self.schema.model_validate(data)
+            elif isinstance(self.schema, TypeAdapter):
+                self.schema.validate_python(data)
+            return CheckResult.ok(name=self.name)
+        except ValidationError as err:
+            failures: list[Failure] = []
+            for e in err.errors():
+                loc = e.get("loc", ())
+                path_str = ".".join(str(item) for item in loc) if loc else None
+                failures.append(
+                    Failure(
+                        code=FailureCode.SCHEMA_VALIDATION_ERROR,
+                        message=e.get("msg", "Schema validation error"),
+                        severity=self.severity,
+                        path=path_str,
+                        metadata={
+                            "type": e.get("type"),
+                            "input": e.get("input"),
+                        },
+                    )
+                )
+            return CheckResult.failed(failures, name=self.name)
+
+
+def schema_check(
+    schema: type[BaseModel] | TypeAdapter[Any],
+    *,
+    severity: Severity = Severity.ERROR,
+    name: str | None = None,
+) -> SchemaCheck:
+    """Returns a check validating output against a Pydantic v2 BaseModel or TypeAdapter."""
+    return SchemaCheck(schema, severity=severity, name=name)