solace-agent-mesh 1.11.2__py3-none-any.whl

solace_agent_mesh/evaluation/shared/__init__.py

@@ -0,0 +1,92 @@
+# Expose constants and validation rules
+from .constants import (
+    ALLOWED_TOPIC_INFIXES,
+    BLOCKED_TOPIC_INFIXES,
+    BROKER_REQUIRED_FIELDS,
+    DEFAULT_CATEGORY,
+    DEFAULT_CONNECTION_TIMEOUT,
+    DEFAULT_DESCRIPTION,
+    DEFAULT_RECONNECT_ATTEMPTS,
+    DEFAULT_RECONNECT_DELAY,
+    DEFAULT_RUN_COUNT,
+    DEFAULT_STARTUP_WAIT_TIME,
+    DEFAULT_TEST_TIMEOUT,
+    DEFAULT_WAIT_TIME,
+    DEFAULT_WORKERS,
+    EVALUATION_DIR,
+    MAX_ARTIFACT_SIZE_MB,
+    MAX_WAIT_TIME,
+    MAX_WORKERS,
+    MESSAGE_TIMEOUT,
+    REMOTE_REQUIRED_FIELDS,
+    ConnectionState,
+)
+from .exceptions import (
+    BrokerConnectionError,
+    CategorizationError,
+    ConfigurationError,
+    EvaluationError,
+    InvalidDataError,
+    MessageProcessingError,
+    MissingFileError,
+    SubscriberError,
+    TestCaseError,
+    TestCaseFileNotFoundError,
+    TestCaseParseError,
+)
+from .helpers import get_local_base_url
+
+# Expose main loader classes and functions
+from .test_case_loader import TestCase, load_test_case
+from .test_suite_loader import (
+    BrokerConfig,
+    EvaluationConfigLoader,
+    EvaluationOptions,
+    TestSuiteConfiguration,
+)
+
+__all__ = [
+    # Constants
+    "DEFAULT_STARTUP_WAIT_TIME",
+    "DEFAULT_TEST_TIMEOUT",
+    "DEFAULT_CONNECTION_TIMEOUT",
+    "DEFAULT_RECONNECT_ATTEMPTS",
+    "DEFAULT_RECONNECT_DELAY",
+    "DEFAULT_WAIT_TIME",
+    "DEFAULT_RUN_COUNT",
+    "DEFAULT_WORKERS",
+    "MAX_WORKERS",
+    "DEFAULT_CATEGORY",
+    "DEFAULT_DESCRIPTION",
+    "EVALUATION_DIR",
+    "MAX_ARTIFACT_SIZE_MB",
+    "MAX_WAIT_TIME",
+    "BROKER_REQUIRED_FIELDS",
+    "REMOTE_REQUIRED_FIELDS",
+    "ALLOWED_TOPIC_INFIXES",
+    "BLOCKED_TOPIC_INFIXES",
+    "MESSAGE_TIMEOUT",
+    "ConnectionState",
+    # Test Case Loading
+    "TestCase",
+    "load_test_case",
+    # Test Suite Loading
+    "TestSuiteConfiguration",
+    "EvaluationConfigLoader",
+    "BrokerConfig",
+    "EvaluationOptions",
+    # Exceptions
+    "EvaluationError",
+    "TestCaseError",
+    "TestCaseFileNotFoundError",
+    "TestCaseParseError",
+    "SubscriberError",
+    "BrokerConnectionError",
+    "MessageProcessingError",
+    "ConfigurationError",
+    "CategorizationError",
+    "MissingFileError",
+    "InvalidDataError",
+    # Helpers
+    "get_local_base_url",
+]

solace_agent_mesh/evaluation/shared/constants.py

@@ -0,0 +1,47 @@
+from enum import Enum
+from pathlib import Path
+
+EVALUATION_DIR = Path(__file__).parent.resolve()
+
+# Timeouts and Delays
+DEFAULT_STARTUP_WAIT_TIME = 60
+DEFAULT_TEST_TIMEOUT = 60
+DEFAULT_CONNECTION_TIMEOUT = 30
+DEFAULT_RECONNECT_ATTEMPTS = 3
+DEFAULT_RECONNECT_DELAY = 1.0
+DEFAULT_WAIT_TIME = 60
+
+# Test Suite Defaults
+DEFAULT_RUN_COUNT = 1
+DEFAULT_WORKERS = 4
+MAX_WORKERS = 8
+
+# Test Case Defaults
+DEFAULT_CATEGORY = "Other"
+DEFAULT_DESCRIPTION = "No description provided."
+MAX_WAIT_TIME = 300
+MAX_ARTIFACT_SIZE_MB = 10
+
+# Validation
+BROKER_REQUIRED_FIELDS = ["host", "vpn_name", "username", "password"]
+REMOTE_REQUIRED_FIELDS = ["EVAL_REMOTE_URL", "EVAL_NAMESPACE"]
+
+
+# Subscription Defaults
+ALLOWED_TOPIC_INFIXES: list[str] = [
+    "/agent/request/",
+    "/gateway/status/",
+    "/gateway/response/",
+]
+BLOCKED_TOPIC_INFIXES: list[str] = ["/discovery/"]
+MESSAGE_TIMEOUT: int = 1000
+
+
+class ConnectionState(Enum):
+    """Represents the connection state of the subscriber."""
+
+    DISCONNECTED = "disconnected"
+    CONNECTING = "connecting"
+    CONNECTED = "connected"
+    DISCONNECTING = "disconnecting"
+    ERROR = "error"

solace_agent_mesh/evaluation/shared/exceptions.py

@@ -0,0 +1,50 @@
+"""
+Centralized custom exceptions for the evaluation module.
+"""
+
+class EvaluationError(Exception):
+    """Base exception for all evaluation-related errors."""
+    pass
+
+# Test Case Errors
+class TestCaseError(EvaluationError):
+    """Base exception for test case-related errors."""
+    pass
+
+class TestCaseFileNotFoundError(TestCaseError):
+    """Raised when the test case file is not found."""
+    pass
+
+class TestCaseParseError(TestCaseError):
+    """Raised when the test case file cannot be parsed or validated."""
+    pass
+
+# Subscriber Errors
+class SubscriberError(EvaluationError):
+    """Base exception for subscriber-related errors."""
+    pass
+
+class BrokerConnectionError(SubscriberError):
+    """Raised when broker connection fails."""
+    pass
+
+class MessageProcessingError(SubscriberError):
+    """Raised when message processing fails."""
+    pass
+
+class ConfigurationError(SubscriberError):
+    """Raised when configuration is invalid."""
+    pass
+
+# Message Organizer Errors
+class CategorizationError(EvaluationError):
+    """Base exception for categorization errors."""
+    pass
+
+class MissingFileError(CategorizationError):
+    """Raised when required files are missing."""
+    pass
+
+class InvalidDataError(CategorizationError):
+    """Raised when data format is invalid."""
+    pass

solace_agent_mesh/evaluation/shared/helpers.py

@@ -0,0 +1,35 @@
+import logging
+import os
+
+from dotenv import load_dotenv
+
+log = logging.getLogger(__name__)
+
+
+def get_local_base_url() -> str:
+    """
+    Constructs the local API base URL from environment variables.
+    """
+    load_dotenv()
+    host = os.getenv("REST_API_HOST", "0.0.0.0")
+    port = os.getenv("REST_API_PORT", "8080")
+    return f"http://{host}:{port}"
+
+
+def resolve_env_vars(data: dict) -> dict:
+    """
+    Resolves environment variables in a dictionary.
+    Looks for keys ending in _VAR and replaces them with the corresponding environment variable value.
+    """
+    resolved = {}
+    for key, value in data.items():
+        if key.endswith("_VAR"):
+            env_var_name = key[:-4]  # Remove '_VAR' suffix
+            env_value = os.getenv(value)
+            if not env_value:
+                log.warning(f"Environment variable '{value}' not set for {env_var_name}")
+            resolved[env_var_name] = env_value
+        else:
+            # This is a direct value, include it as-is
+            resolved[key] = value
+    return resolved

solace_agent_mesh/evaluation/shared/test_case_loader.py

@@ -0,0 +1,167 @@
+"""
+Refactored test case loader with Pydantic-based validation.
+This module provides robust test case loading and validation using Pydantic.
+"""
+
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field, ValidationError, field_validator
+
+from .constants import (
+    DEFAULT_CATEGORY,
+    DEFAULT_DESCRIPTION,
+    DEFAULT_WAIT_TIME,
+    MAX_WAIT_TIME,
+)
+from .exceptions import (
+    TestCaseError,
+    TestCaseFileNotFoundError,
+    TestCaseParseError,
+)
+
+log = logging.getLogger(__name__)
+
+
+class Artifact(BaseModel):
+    """
+    Configuration for an individual artifact.
+    Uses Pydantic for built-in validation.
+    """
+    artifact_type: Literal["file", "url", "text"] = Field(..., alias="type")
+    path: str = Field(..., min_length=1)
+
+    @field_validator("path")
+    def validate_path_security(cls, v, values):
+        """
+        Validate file path security to prevent directory traversal.
+        """
+        if "artifact_type" in values.data and values.data["artifact_type"] == "file":
+            normalized_path = os.path.normpath(v)
+            if normalized_path.startswith("..") or os.path.isabs(normalized_path):
+                raise ValueError(
+                    f"Artifact path '{v}' is not safe (no absolute paths or directory traversal)"
+                )
+        return v
+
+    class Config:
+        validate_by_name = True
+
+
+class Evaluation(BaseModel):
+    """
+    Configuration for evaluation criteria.
+    """
+    expected_tools: list[str] = Field(default_factory=list)
+    expected_response: str = ""
+    criterion: str = ""
+
+    @field_validator("expected_tools")
+    def validate_tool_names(cls, tools: list[str]) -> list[str]:
+        """
+        Ensures that tool names are non-empty strings.
+        """
+        if any(not isinstance(tool, str) or not tool.strip() for tool in tools):
+            raise ValueError("All tool names must be non-empty strings.")
+        return tools
+
+class TestCase(BaseModel):
+    """
+    Complete test case configuration using Pydantic for validation.
+    """
+    test_case_id: str = Field(..., min_length=1)
+    query: str = Field(..., min_length=1)
+    target_agent: str = Field(..., min_length=1)
+    category: str = DEFAULT_CATEGORY
+    description: str = DEFAULT_DESCRIPTION
+    wait_time: int = Field(DEFAULT_WAIT_TIME, gt=0, le=MAX_WAIT_TIME)
+    artifacts: list[Artifact] = Field(default_factory=list)
+    evaluation: Evaluation = Field(default_factory=Evaluation)
+
+    def to_dict(self) -> dict[str, any]:
+        """Convert test case to dictionary format for JSON serialization."""
+        return self.model_dump(by_alias=True)
+
+def load_test_case(test_case_path: str) -> dict[str, any]:
+    """
+    Load a test case from a JSON file with Pydantic validation.
+
+    Args:
+        test_case_path: The full path to the test case file.
+
+    Returns:
+        A dictionary containing the validated test case data.
+
+    Raises:
+        TestCaseFileNotFoundError: If the file does not exist.
+        TestCaseParseError: If the file is not valid JSON or fails validation.
+        TestCaseError: For other unexpected errors.
+    """
+    path = Path(test_case_path)
+    if not path.is_file():
+        raise TestCaseFileNotFoundError(f"Test case file not found: {test_case_path}")
+
+    try:
+        with open(path) as f:
+            data = json.load(f)
+            test_case = TestCase.model_validate(data)
+            return test_case.to_dict()
+    except json.JSONDecodeError as e:
+        raise TestCaseParseError(f"Invalid JSON in test case file {test_case_path}: {e}") from e
+    except ValidationError as e:
+        raise TestCaseParseError(f"Test case validation failed for {test_case_path}:\n{e}") from e
+    except Exception as e:
+        raise TestCaseError(f"Error reading or processing test case file {test_case_path}: {e}") from e
+
+def validate_test_case_file(test_case_path: str) -> None:
+    """
+    Validates a test case file and logs errors if any. Exits on failure.
+
+    Args:
+        test_case_path: The full path to the test case file.
+
+    Raises:
+        SystemExit: If validation fails.
+    """
+    try:
+        load_test_case(test_case_path)
+        log.info(f"Test case '{test_case_path}' validation successful.")
+    except (TestCaseFileNotFoundError, TestCaseParseError, TestCaseError) as e:
+        log.error(f"Error: {e}")
+        sys.exit(1)
+
+def main():
+    """Main entry point for command-line usage and testing."""
+    if len(sys.argv) != 2:
+        log.error(f"Usage: python {sys.argv[0]} <path_to_test_case.json>")
+        log.error(f"Example: python {sys.argv[0]} /path/to/hello_world.test.json")
+        sys.exit(1)
+
+    test_case_path = sys.argv[1]
+
+    try:
+        # Load and validate test case
+        test_case_data = load_test_case(test_case_path)
+
+        log.info(f"Successfully loaded test case: {test_case_data['test_case_id']}")
+        log.info(f"Target Agent: {test_case_data['target_agent']}")
+        log.info(f"Category: {test_case_data['category']}")
+        query = test_case_data['query']
+        log.info(f"Query: {query[:100]}{'...' if len(query) > 100 else ''}")
+        log.info(f"Wait Time: {test_case_data['wait_time']} seconds")
+        log.info(f"Artifacts: {len(test_case_data['artifacts'])} artifact(s)")
+        log.info(f"Expected Tools: {len(test_case_data['evaluation']['expected_tools'])} tool(s)")
+
+    except (TestCaseFileNotFoundError, TestCaseParseError, TestCaseError) as e:
+        log.error(f"Error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        log.error(f"An unexpected error occurred: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

solace_agent_mesh/evaluation/shared/test_suite_loader.py

@@ -0,0 +1,280 @@
+"""
+Modern Pydantic-based configuration loader with comprehensive validation.
+Replaces complex custom validation with clean, declarative models.
+"""
+
+import json
+import logging
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
+from solace.messaging.config.solace_properties import (
+    authentication_properties,
+    service_properties,
+    transport_layer_properties,
+    transport_layer_security_properties,
+)
+
+from .constants import (
+    BROKER_REQUIRED_FIELDS,
+    DEFAULT_CONNECTION_TIMEOUT,
+    DEFAULT_RECONNECT_ATTEMPTS,
+    DEFAULT_RECONNECT_DELAY,
+    DEFAULT_RUN_COUNT,
+    DEFAULT_WORKERS,
+    MAX_WORKERS,
+    REMOTE_REQUIRED_FIELDS,
+)
+from .helpers import resolve_env_vars
+
+log = logging.getLogger(__name__)
+
+
+class EnvironmentVariables(BaseModel):
+    """Environment variable configuration with automatic resolution."""
+    variables: dict[str, str | None] = Field(default_factory=dict)
+
+    @model_validator(mode='before')
+    @classmethod
+    def build_variables(cls, data: any) -> any:
+        """Build a variables dictionary from raw data."""
+        if isinstance(data, dict) and 'variables' not in data:
+            return {"variables": resolve_env_vars(data)}
+        return data
+
+    def get(self, key: str, default: str | None = None) -> str | None:
+        """Get environment variable value with default."""
+        return self.variables.get(key, default)
+
+    def is_complete(self, required_vars: list[str]) -> bool:
+        """Check if all required environment variables are present."""
+        return all(self.variables.get(var) is not None for var in required_vars)
+
+
+class ModelConfiguration(BaseModel):
+    """Individual LLM model configuration with validation."""
+    name: str = Field(min_length=1, description="Model name cannot be empty")
+    environment: EnvironmentVariables = Field(alias="env")
+
+    @model_validator(mode='after')
+    def validate_essential_vars(self):
+        """Ensure essential environment variables are present."""
+        essential_vars = ["LLM_SERVICE_PLANNING_MODEL_NAME"]
+        if not any(var in self.environment.variables for var in essential_vars):
+            raise ValueError(f"Model '{self.name}' must have at least one of: {essential_vars}")
+        return self
+
+
+class RemoteConfig(BaseModel):
+    """Remote configuration with environment variable support."""
+    environment: EnvironmentVariables
+
+    @model_validator(mode='before')
+    @classmethod
+    def build_environment(cls, data: any) -> any:
+        if isinstance(data, dict):
+            # All fields are environment variables
+            return {"environment": data}
+        return data
+
+    @model_validator(mode='after')
+    def sanitize_namespace(self):
+        """Remove trailing slashes from EVAL_NAMESPACE."""
+        if self.environment and "EVAL_NAMESPACE" in self.environment.variables:
+            namespace = self.environment.variables.get("EVAL_NAMESPACE")
+            if namespace:
+                self.environment.variables["EVAL_NAMESPACE"] = namespace.rstrip("/")
+        return self
+
+
+class BrokerConfig(BaseModel):
+    """Broker connection configuration with validation and environment variable resolution."""
+    host: str | None = Field(default=None, alias="SOLACE_BROKER_URL")
+    vpn_name: str | None = Field(default=None, alias="SOLACE_BROKER_VPN")
+    username: str | None = Field(default=None, alias="SOLACE_BROKER_USERNAME")
+    password: str | None = Field(default=None, alias="SOLACE_BROKER_PASSWORD")
+    cert_validated: bool = False
+    connection_timeout: int = DEFAULT_CONNECTION_TIMEOUT
+    reconnect_attempts: int = DEFAULT_RECONNECT_ATTEMPTS
+    reconnect_delay: float = DEFAULT_RECONNECT_DELAY
+
+    @model_validator(mode='before')
+    @classmethod
+    def resolve_env_vars(cls, data: dict) -> dict:
+        """Resolve environment variables for broker configuration."""
+        return resolve_env_vars(data)
+
+    @model_validator(mode='after')
+    def check_required_fields(self):
+        """Ensure all required broker fields are present."""
+        missing_fields = [field for field in BROKER_REQUIRED_FIELDS if getattr(self, field) is None]
+        if missing_fields:
+            raise ValueError(f"Broker configuration is missing required fields: {missing_fields}")
+        return self
+
+    def to_solace_properties(self) -> dict[str, any]:
+        """Convert to Solace messaging properties."""
+        return {
+            transport_layer_properties.HOST: self.host,
+            service_properties.VPN_NAME: self.vpn_name,
+            authentication_properties.SCHEME_BASIC_USER_NAME: self.username,
+            authentication_properties.SCHEME_BASIC_PASSWORD: self.password,
+            transport_layer_security_properties.CERT_VALIDATED: self.cert_validated,
+        }
+
+
+class EvaluationOptions(BaseModel):
+    """Evaluation behavior settings with conditional validation."""
+    tool_matching_enabled: bool = Field(default=True)
+    response_matching_enabled: bool = Field(default=True)
+    llm_evaluation_enabled: bool = Field(default=False)
+    llm_evaluator_environment: EnvironmentVariables | None = Field(default=None)
+
+    @model_validator(mode='after')
+    def validate_llm_evaluator_config(self):
+        """Validate LLM evaluator configuration when enabled."""
+        if self.llm_evaluation_enabled:
+            if not self.llm_evaluator_environment:
+                raise ValueError("llm_evaluator_environment is required when llm_evaluation_enabled is true")
+
+            required_vars = [
+                "LLM_SERVICE_PLANNING_MODEL_NAME",
+                "LLM_SERVICE_ENDPOINT",
+                "LLM_SERVICE_API_KEY"
+            ]
+            if not self.llm_evaluator_environment.is_complete(required_vars):
+                raise ValueError(f"LLM evaluator requires environment variables: {required_vars}")
+        return self
+
+
+class TestSuiteConfiguration(BaseModel):
+    """Complete test suite configuration with comprehensive validation."""
+    broker: BrokerConfig
+    agent_configs: list[str] | None = Field(default=None, min_length=1, alias="agents")
+    model_configurations: list[ModelConfiguration] | None = Field(default=None, min_length=1, alias="llm_models")
+    remote: RemoteConfig | None = Field(default=None)
+    test_case_files: list[str] = Field(min_length=1, alias="test_cases")
+    results_directory: str = Field(min_length=1, alias="results_dir_name")
+    run_count: int = Field(default=DEFAULT_RUN_COUNT, ge=1, alias="runs")
+    workers: int = Field(default=DEFAULT_WORKERS, ge=1, le=MAX_WORKERS)
+    evaluation_options: EvaluationOptions = Field(default_factory=EvaluationOptions, alias="evaluation_settings")
+
+    @field_validator('agent_configs', 'test_case_files', mode='before')
+    @classmethod
+    def resolve_relative_paths(cls, v: list[str], info) -> list[str]:
+        """Convert relative paths to absolute paths."""
+        if not v:
+            return v
+        config_dir = getattr(info.context, 'config_dir', Path.cwd())
+        return [str(config_dir / p) if not Path(p).is_absolute() else p for p in v]
+
+    @model_validator(mode='after')
+    def add_eval_backend_if_missing(self):
+        """Add eval_backend.yaml if not present in agent configs."""
+        if self.agent_configs and not any(Path(p).name == "eval_backend.yaml" for p in self.agent_configs):
+            project_root = Path.cwd()
+            eval_backend_path = str(project_root / "configs" / "eval_backend.yaml")
+            self.agent_configs.append(eval_backend_path)
+        return self
+
+    @model_validator(mode='after')
+    def validate_configuration_mode(self):
+        """Validate that either remote or local configuration is correctly provided."""
+        is_remote = self.remote is not None
+        is_local = self.agent_configs is not None and self.model_configurations is not None
+
+        if is_remote and is_local:
+            raise ValueError("Configuration cannot have both 'remote' and local settings ('agents', 'llm_models').")
+
+        if is_remote and (self.agent_configs or self.model_configurations):
+            raise ValueError("When 'remote' is provided, 'agents' and 'llm_models' must not be set.")
+
+        if is_remote and not self.remote.environment.is_complete(REMOTE_REQUIRED_FIELDS):
+            raise ValueError(f"Remote configuration requires environment variables: {REMOTE_REQUIRED_FIELDS}")
+
+        if not is_remote and not is_local:
+            raise ValueError("Configuration must include either 'remote' or local settings ('agents', 'llm_models').")
+
+        return self
+
+
+class ConfigurationParser:
+    """Handles raw JSON parsing and transformation."""
+
+    def __init__(self, config_path: str):
+        self.config_path = Path(config_path)
+        self.config_dir = self.config_path.parent.resolve()
+
+    def load_raw_config(self) -> dict[str, any]:
+        """Load raw JSON configuration."""
+        try:
+            with open(self.config_path) as f:
+                return json.load(f)
+        except FileNotFoundError:
+            log.error(f"Configuration file not found: {self.config_path}")
+            sys.exit(1)
+        except json.JSONDecodeError as e:
+            log.error(f"Invalid JSON in configuration file: {e}")
+            sys.exit(1)
+
+    def transform_evaluation_settings(self, raw_settings: dict[str, any]) -> dict[str, any]:
+        """Transform nested evaluation settings structure."""
+        result = {
+            "tool_matching_enabled": raw_settings.get("tool_match", {}).get("enabled", True),
+            "response_matching_enabled": raw_settings.get("response_match", {}).get("enabled", True),
+            "llm_evaluation_enabled": raw_settings.get("llm_evaluator", {}).get("enabled", False),
+            "llm_evaluator_environment": None
+        }
+
+        # Handle LLM evaluator environment if enabled
+        if result["llm_evaluation_enabled"]:
+            env_data = raw_settings.get("llm_evaluator", {}).get("env", {})
+            if env_data:
+                result["llm_evaluator_environment"] = env_data
+
+        return result
+
+
+class EvaluationConfigLoader:
+    """Modern configuration loader using Pydantic validation."""
+
+    def __init__(self, config_path: str):
+        self.parser = ConfigurationParser(config_path)
+
+    def load_configuration(self) -> TestSuiteConfiguration:
+        """Load and validate configuration, returning Pydantic model."""
+        try:
+            # Load raw JSON
+            raw_config = self.parser.load_raw_config()
+
+            # Transform evaluation settings structure
+            if "evaluation_settings" in raw_config:
+                raw_config["evaluation_settings"] = self.parser.transform_evaluation_settings(
+                    raw_config["evaluation_settings"]
+                )
+
+            config = TestSuiteConfiguration.model_validate(
+                raw_config,
+                context={'config_dir': self.parser.config_dir}
+            )
+
+            log.info("Configuration loaded and validated successfully.")
+            return config
+
+        except ValidationError as e:
+            self._handle_validation_error(e)
+            sys.exit(1)
+
+    def get_evaluation_options(self) -> EvaluationOptions:
+        """Get evaluation options from configuration."""
+        config = self.load_configuration()
+        return config.evaluation_options
+
+    def _handle_validation_error(self, e: ValidationError):
+        """Convert Pydantic validation errors to user-friendly format."""
+        log.error("Configuration validation failed:")
+        for error in e.errors():
+            field_path = " -> ".join(str(loc) for loc in error['loc'])
+            message = error['msg']
+            log.error(f"  Field '{field_path}': {message}")