agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/metrics/structured/structured_output_score.py

@@ -0,0 +1,298 @@
+"""
+Structured Output Score - Composite Metric.
+
+Combines multiple structured validation aspects into a single score.
+"""
+
+from typing import Any, Dict, Optional
+
+from ..base_metric import BaseMetric
+from .types import StructuredInput, JSONInput, ValidationMode
+from .validators import JSONValidator, YAMLValidator
+
+
+class StructuredOutputScore(BaseMetric[StructuredInput]):
+    """
+    Comprehensive structured output evaluation.
+
+    Combines multiple aspects:
+    - Syntax validity (parseability)
+    - Schema compliance (matches expected structure)
+    - Field completeness (required fields present)
+    - Type correctness (values have correct types)
+    - Value accuracy (optional, if expected provided)
+
+    Score: 0.0 to 1.0 weighted combination of all aspects.
+
+    Example:
+        >>> metric = StructuredOutputScore()
+        >>> result = metric.evaluate([{
+        ...     "response": '{"name": "Alice", "age": 25}',
+        ...     "format": "json",
+        ...     "schema": {
+        ...         "type": "object",
+        ...         "required": ["name", "age"],
+        ...         "properties": {
+        ...             "name": {"type": "string"},
+        ...             "age": {"type": "integer"}
+        ...         }
+        ...     }
+        ... }])
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "structured_output_score"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.json_validator = JSONValidator()
+        self._yaml_validator = None
+
+        # Weights for different aspects
+        self.syntax_weight = self.config.get("syntax_weight", 0.2)
+        self.schema_weight = self.config.get("schema_weight", 0.3)
+        self.completeness_weight = self.config.get("completeness_weight", 0.25)
+        self.type_weight = self.config.get("type_weight", 0.15)
+        self.value_weight = self.config.get("value_weight", 0.1)
+
+    @property
+    def yaml_validator(self):
+        if self._yaml_validator is None:
+            try:
+                self._yaml_validator = YAMLValidator()
+            except ImportError:
+                return None
+        return self._yaml_validator
+
+    def _get_validator(self, format_name: str):
+        """Get validator for format."""
+        if format_name.lower() == "yaml":
+            if self.yaml_validator is None:
+                raise ImportError("PyYAML is required for YAML validation")
+            return self.yaml_validator
+        return self.json_validator
+
+    def compute_one(self, inputs: StructuredInput) -> Dict[str, Any]:
+        response = inputs.response
+        format_name = inputs.format
+        schema = inputs.schema
+        expected = inputs.expected
+        mode_str = inputs.mode
+
+        # Convert mode string to enum
+        try:
+            mode = ValidationMode(mode_str)
+        except ValueError:
+            mode = ValidationMode.COERCE
+
+        if not response or not response.strip():
+            return {
+                "output": 0.0,
+                "reason": "Empty response",
+                "breakdown": {
+                    "syntax": 0.0,
+                    "schema": 0.0,
+                    "completeness": 0.0,
+                    "types": 0.0,
+                    "values": 0.0,
+                },
+            }
+
+        # Get validator
+        try:
+            validator = self._get_validator(format_name)
+        except ImportError as e:
+            return {
+                "output": 0.0,
+                "reason": str(e),
+            }
+
+        # Initialize scores
+        scores = {
+            "syntax": 0.0,
+            "schema": 0.0,
+            "completeness": 0.0,
+            "types": 0.0,
+            "values": 0.0,
+        }
+
+        # 1. Syntax validation
+        syntax_result = validator.validate_syntax(response)
+        if not syntax_result.syntax_valid:
+            return {
+                "output": 0.0,
+                "reason": f"Syntax error: {syntax_result.errors[0].message if syntax_result.errors else 'Unknown'}",
+                "breakdown": scores,
+                "errors": [e.dict() for e in syntax_result.errors],
+            }
+
+        scores["syntax"] = 1.0
+        parsed = syntax_result.parsed
+
+        # Track which dimensions are evaluable
+        evaluable = {"syntax": self.syntax_weight}
+
+        # 2. Schema validation (if schema provided)
+        if schema:
+            evaluable["schema"] = self.schema_weight
+            evaluable["completeness"] = self.completeness_weight
+            evaluable["types"] = self.type_weight
+
+            schema_result = validator.validate_schema(response, schema, mode)
+            scores["completeness"] = schema_result.completeness
+
+            if schema_result.schema_valid:
+                scores["schema"] = 1.0
+                scores["types"] = 1.0
+            else:
+                # Analyze errors for partial scores
+                type_errors = sum(1 for e in schema_result.errors if e.error_type == "type")
+                schema_errors = len(schema_result.errors) - type_errors
+
+                # Estimate total expected validations
+                total_fields = self._count_schema_fields(schema)
+
+                if total_fields > 0:
+                    scores["schema"] = max(0.0, 1.0 - schema_errors / total_fields)
+                    scores["types"] = max(0.0, 1.0 - type_errors / total_fields)
+
+        # 3. Value accuracy (if expected provided)
+        if expected is not None:
+            evaluable["values"] = self.value_weight
+
+            compare_result = validator.compare(response, expected, mode)
+            if compare_result.valid:
+                scores["values"] = 1.0
+            else:
+                # Calculate partial value match
+                value_errors = len(compare_result.errors)
+                total_values = self._count_values(expected)
+                scores["values"] = max(0.0, 1.0 - value_errors / max(total_values, 1))
+
+        # Calculate weighted overall score — only evaluable dimensions contribute
+        # Unevaluable dimensions score 0, not 1 (no data ≠ perfect)
+        overall = sum(evaluable[k] * scores[k] for k in evaluable)
+
+        return {
+            "output": round(overall, 4),
+            "reason": self._generate_reason(scores, evaluable),
+            "breakdown": {k: round(v, 4) for k, v in scores.items()},
+            "parsed": parsed,
+        }
+
+    def _count_schema_fields(self, schema: Dict[str, Any], depth: int = 0) -> int:
+        """Count fields in schema."""
+        if depth > 10:
+            return 1
+
+        count = 0
+        if schema.get("type") == "object":
+            properties = schema.get("properties", {})
+            count = len(properties)
+            for prop_schema in properties.values():
+                count += self._count_schema_fields(prop_schema, depth + 1)
+        elif schema.get("type") == "array":
+            items = schema.get("items", {})
+            count = 1 + self._count_schema_fields(items, depth + 1)
+
+        return max(count, 1)
+
+    def _count_values(self, data: Any) -> int:
+        """Count total values in data structure."""
+        if isinstance(data, dict):
+            return sum(self._count_values(v) for v in data.values())
+        elif isinstance(data, list):
+            return sum(self._count_values(item) for item in data)
+        else:
+            return 1
+
+    def _generate_reason(self, scores: Dict[str, float], evaluable: Dict[str, float]) -> str:
+        """Generate human-readable reason."""
+        issues = []
+        if scores["syntax"] < 1.0:
+            issues.append("syntax errors")
+        if "schema" in evaluable and scores["schema"] < 1.0:
+            issues.append("schema violations")
+        if "completeness" in evaluable and scores["completeness"] < 1.0:
+            issues.append("missing fields")
+        if "types" in evaluable and scores["types"] < 1.0:
+            issues.append("type errors")
+        if "values" in evaluable and scores["values"] < 1.0:
+            issues.append("value mismatches")
+
+        skipped = [k for k in ("schema", "completeness", "types", "values") if k not in evaluable]
+
+        if not issues and not skipped:
+            return "Fully valid structured output"
+        parts = []
+        if issues:
+            parts.append(f"Issues: {', '.join(issues)}")
+        if skipped:
+            parts.append(f"Not evaluated: {', '.join(skipped)}")
+        return "; ".join(parts) if parts else "Fully valid structured output"
+
+
+class QuickStructuredCheck(BaseMetric[JSONInput]):
+    """
+    Fast, lightweight structured output check.
+
+    Quick validation that just checks:
+    1. Is it valid JSON?
+    2. Does it have the expected keys?
+    3. Are the types roughly correct?
+
+    Score: 0.0 to 1.0
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "quick_structured_check"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.validator = JSONValidator()
+
+    def compute_one(self, inputs: JSONInput) -> Dict[str, Any]:
+        response = inputs.response
+        schema = inputs.schema
+        expected = inputs.expected
+
+        if not response or not response.strip():
+            return {"output": 0.0, "reason": "Empty response"}
+
+        # Quick syntax check
+        syntax_result = self.validator.validate_syntax(response)
+        if not syntax_result.syntax_valid:
+            return {"output": 0.0, "reason": "Invalid JSON"}
+
+        parsed = syntax_result.parsed
+        score = 0.5  # Base score for valid JSON
+
+        # Quick schema check
+        if schema:
+            required = schema.get("required", [])
+            if required and isinstance(parsed, dict):
+                present = sum(1 for f in required if f in parsed)
+                schema_score = present / len(required)
+                score = 0.5 + (0.5 * schema_score)
+            else:
+                score = 1.0
+        elif expected is not None:
+            # Quick key comparison
+            if isinstance(expected, dict) and isinstance(parsed, dict):
+                expected_keys = set(expected.keys())
+                actual_keys = set(parsed.keys())
+                if expected_keys:
+                    overlap = len(expected_keys & actual_keys) / len(expected_keys)
+                    score = 0.5 + (0.5 * overlap)
+                else:
+                    score = 1.0
+            else:
+                score = 1.0 if type(expected) is type(parsed) else 0.5
+
+        return {
+            "output": round(score, 4),
+            "reason": "Valid JSON" + (" with expected structure" if score >= 0.8 else ""),
+            "parsed": parsed,
+        }

fi/evals/metrics/structured/types.py

@@ -0,0 +1,108 @@
+"""
+Input types for structured output validation metrics.
+"""
+
+import warnings
+from typing import Optional, Dict, Any, List
+from pydantic import BaseModel, Field, ConfigDict
+from enum import Enum
+
+warnings.filterwarnings("ignore", message='Field name "schema" in .* shadows an attribute in parent "BaseModel"')
+
+
+class ValidationMode(Enum):
+    """Validation strictness modes."""
+    STRICT = "strict"      # Exact match, no extra fields, correct types
+    COERCE = "coerce"      # Allow type coercion (str -> int, etc.)
+    LENIENT = "lenient"    # Allow extra fields, flexible types
+
+
+class JSONInput(BaseModel):
+    """Input for JSON validation metrics."""
+    model_config = ConfigDict(extra="allow")
+
+    response: str = Field(..., description="LLM-generated JSON string")
+    schema: Optional[Dict[str, Any]] = Field(
+        None, description="JSON Schema to validate against"
+    )
+    expected: Optional[Dict[str, Any]] = Field(
+        None, description="Expected JSON object for comparison"
+    )
+    mode: str = Field(
+        "coerce", description="Validation strictness: strict, coerce, lenient"
+    )
+
+
+class PydanticInput(BaseModel):
+    """Input for Pydantic model validation."""
+    model_config = ConfigDict(extra="allow")
+
+    response: str = Field(..., description="LLM-generated JSON string")
+    model_class: Optional[str] = Field(
+        None, description="Fully qualified Pydantic model class name"
+    )
+    model_schema: Optional[Dict[str, Any]] = Field(
+        None, description="JSON Schema derived from Pydantic model"
+    )
+    mode: str = Field("coerce")
+
+
+class YAMLInput(BaseModel):
+    """Input for YAML validation metrics."""
+    model_config = ConfigDict(extra="allow")
+
+    response: str = Field(..., description="LLM-generated YAML string")
+    schema: Optional[Dict[str, Any]] = Field(
+        None, description="JSON Schema to validate against"
+    )
+    expected: Optional[Dict[str, Any]] = Field(
+        None, description="Expected YAML content as dict"
+    )
+
+
+class StructuredInput(BaseModel):
+    """Generic input for any structured format."""
+    model_config = ConfigDict(extra="allow")
+
+    response: str = Field(..., description="LLM-generated structured output")
+    format: str = Field("json", description="Format: json, xml, yaml, toml")
+    schema: Optional[Dict[str, Any]] = Field(None, description="Schema to validate")
+    expected: Optional[Any] = Field(None, description="Expected output")
+    mode: str = Field("coerce")
+
+
+class ValidationError(BaseModel):
+    """Single validation error."""
+    path: str = Field(..., description="JSON path to error (e.g., '$.user.name')")
+    message: str = Field(..., description="Error description")
+    error_type: str = Field(..., description="Error type: syntax, type, missing, extra")
+    expected: Optional[Any] = Field(None, description="Expected value/type")
+    actual: Optional[Any] = Field(None, description="Actual value/type")
+
+    def dict(self, **kwargs):
+        """Convert to dictionary."""
+        return {
+            "path": self.path,
+            "message": self.message,
+            "error_type": self.error_type,
+            "expected": self.expected,
+            "actual": self.actual,
+        }
+
+
+class ValidationResult(BaseModel):
+    """Complete validation result."""
+    model_config = ConfigDict(extra="allow")
+
+    valid: bool = Field(..., description="Overall validity")
+    errors: List[ValidationError] = Field(default_factory=list)
+    warnings: List[ValidationError] = Field(default_factory=list)
+
+    # Detailed scores
+    syntax_valid: bool = True
+    schema_valid: bool = True
+    type_valid: bool = True
+    completeness: float = 1.0  # 0-1, fraction of required fields present
+
+    # Parsed output (if successful)
+    parsed: Optional[Any] = None

fi/evals/metrics/structured/validators/__init__.py

@@ -0,0 +1,30 @@
+"""
+Validators for structured output formats.
+"""
+
+from .base import BaseValidator
+from .json_validator import JSONValidator
+from .pydantic_validator import PydanticValidator
+from .yaml_validator import YAMLValidator
+
+__all__ = [
+    "BaseValidator",
+    "JSONValidator",
+    "PydanticValidator",
+    "YAMLValidator",
+]
+
+# Validator registry for easy lookup
+VALIDATORS = {
+    "json": JSONValidator,
+    "yaml": YAMLValidator,
+    "pydantic": PydanticValidator,
+}
+
+
+def get_validator(format_name: str) -> BaseValidator:
+    """Get validator instance by format name."""
+    validator_class = VALIDATORS.get(format_name.lower())
+    if validator_class is None:
+        raise ValueError(f"Unknown format: {format_name}. Available: {list(VALIDATORS.keys())}")
+    return validator_class()

fi/evals/metrics/structured/validators/base.py

@@ -0,0 +1,189 @@
+"""
+Base validator interface for structured output validation.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+from ..types import ValidationResult, ValidationError, ValidationMode
+
+
+class BaseValidator(ABC):
+    """Abstract base class for format validators."""
+
+    format_name: str = "unknown"
+
+    @abstractmethod
+    def validate_syntax(self, content: str) -> ValidationResult:
+        """
+        Validate syntax only (is it parseable?).
+
+        Returns:
+            ValidationResult with syntax_valid set
+        """
+        pass
+
+    @abstractmethod
+    def validate_schema(
+        self,
+        content: str,
+        schema: Dict[str, Any],
+        mode: ValidationMode = ValidationMode.COERCE,
+    ) -> ValidationResult:
+        """
+        Validate against a schema.
+
+        Args:
+            content: Raw string content
+            schema: Schema to validate against
+            mode: Validation strictness
+
+        Returns:
+            ValidationResult with full validation details
+        """
+        pass
+
+    @abstractmethod
+    def parse(self, content: str) -> Any:
+        """
+        Parse content into Python object.
+
+        Raises:
+            ValueError: If content cannot be parsed
+        """
+        pass
+
+    def compare(
+        self,
+        content: str,
+        expected: Any,
+        mode: ValidationMode = ValidationMode.COERCE,
+    ) -> ValidationResult:
+        """
+        Compare parsed content against expected value.
+
+        Default implementation - can be overridden.
+        """
+        try:
+            parsed = self.parse(content)
+        except Exception as e:
+            return ValidationResult(
+                valid=False,
+                syntax_valid=False,
+                errors=[ValidationError(
+                    path="$",
+                    message=str(e),
+                    error_type="syntax",
+                )]
+            )
+
+        errors = self._compare_values(parsed, expected, "$", mode)
+
+        return ValidationResult(
+            valid=len(errors) == 0,
+            syntax_valid=True,
+            errors=errors,
+            parsed=parsed,
+        )
+
+    def _compare_values(
+        self,
+        actual: Any,
+        expected: Any,
+        path: str,
+        mode: ValidationMode,
+    ) -> List[ValidationError]:
+        """Recursively compare values."""
+        errors = []
+
+        # Handle None cases
+        if expected is None and actual is None:
+            return errors
+        if expected is None or actual is None:
+            if expected != actual:
+                errors.append(ValidationError(
+                    path=path,
+                    message="Value mismatch (None vs non-None)",
+                    error_type="value",
+                    expected=expected,
+                    actual=actual,
+                ))
+            return errors
+
+        # Type comparison
+        if type(actual) is not type(expected):
+            if mode == ValidationMode.STRICT:
+                errors.append(ValidationError(
+                    path=path,
+                    message="Type mismatch",
+                    error_type="type",
+                    expected=type(expected).__name__,
+                    actual=type(actual).__name__,
+                ))
+                return errors
+            elif mode == ValidationMode.COERCE:
+                # Try to coerce
+                try:
+                    actual = type(expected)(actual)
+                except (ValueError, TypeError):
+                    errors.append(ValidationError(
+                        path=path,
+                        message=f"Cannot coerce {type(actual).__name__} to {type(expected).__name__}",
+                        error_type="type",
+                        expected=type(expected).__name__,
+                        actual=type(actual).__name__,
+                    ))
+                    return errors
+
+        # Dict comparison
+        if isinstance(expected, dict):
+            # Check for missing keys
+            for key in expected:
+                if key not in actual:
+                    errors.append(ValidationError(
+                        path=f"{path}.{key}",
+                        message="Missing required field",
+                        error_type="missing",
+                        expected=key,
+                    ))
+                else:
+                    errors.extend(self._compare_values(
+                        actual[key], expected[key], f"{path}.{key}", mode
+                    ))
+
+            # Check for extra keys in strict mode
+            if mode == ValidationMode.STRICT:
+                for key in actual:
+                    if key not in expected:
+                        errors.append(ValidationError(
+                            path=f"{path}.{key}",
+                            message="Unexpected field",
+                            error_type="extra",
+                            actual=key,
+                        ))
+
+        # List comparison
+        elif isinstance(expected, list):
+            if len(actual) != len(expected):
+                errors.append(ValidationError(
+                    path=path,
+                    message="Array length mismatch",
+                    error_type="length",
+                    expected=len(expected),
+                    actual=len(actual),
+                ))
+            else:
+                for i, (a, e) in enumerate(zip(actual, expected)):
+                    errors.extend(self._compare_values(a, e, f"{path}[{i}]", mode))
+
+        # Scalar comparison
+        else:
+            if actual != expected:
+                errors.append(ValidationError(
+                    path=path,
+                    message="Value mismatch",
+                    error_type="value",
+                    expected=expected,
+                    actual=actual,
+                ))
+
+        return errors