agent-control-evaluators 5.0.0__py3-none-any.whl

agent_control_evaluators/__init__.py

@@ -0,0 +1,76 @@
+"""Agent Control Evaluators.
+
+This package contains builtin evaluator implementations for agent-control.
+Built-in evaluators (regex, list, json, sql) are registered automatically on import.
+
+Available evaluators:
+    Built-in (no namespace):
+        - regex: Regular expression matching
+        - list: List-based value matching
+        - json: JSON validation
+        - sql: SQL query validation
+
+Naming convention:
+    - Built-in: "regex", "list", "json", "sql"
+    - External: "provider.name" (e.g., "galileo.luna2")
+    - Agent-scoped: "agent:name" (custom code deployed with agent)
+
+External evaluators are installed via separate packages (e.g., agent-control-evaluator-galileo).
+Custom evaluators are Evaluator classes deployed with the engine.
+Their schemas are registered via initAgent for validation purposes.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("agent-control-evaluators")
+except PackageNotFoundError:
+    __version__ = "0.0.0.dev"
+
+# Core infrastructure - export from _base and _registry
+from agent_control_evaluators._base import Evaluator, EvaluatorConfig, EvaluatorMetadata
+from agent_control_evaluators._discovery import (
+    discover_evaluators,
+    ensure_evaluators_discovered,
+    list_evaluators,
+    reset_evaluator_discovery,
+)
+from agent_control_evaluators._factory import clear_evaluator_cache, get_evaluator_instance
+from agent_control_evaluators._registry import (
+    clear_evaluators,
+    get_all_evaluators,
+    get_evaluator,
+    register_evaluator,
+)
+
+# Import built-in evaluators to auto-register them
+from agent_control_evaluators.json import JSONEvaluator, JSONEvaluatorConfig
+from agent_control_evaluators.list import ListEvaluator, ListEvaluatorConfig
+from agent_control_evaluators.regex import RegexEvaluator, RegexEvaluatorConfig
+from agent_control_evaluators.sql import SQLEvaluator, SQLEvaluatorConfig
+
+__all__ = [
+    # Core infrastructure
+    "Evaluator",
+    "EvaluatorConfig",
+    "EvaluatorMetadata",
+    "register_evaluator",
+    "get_evaluator",
+    "get_all_evaluators",
+    "clear_evaluators",
+    "discover_evaluators",
+    "ensure_evaluators_discovered",
+    "reset_evaluator_discovery",
+    "list_evaluators",
+    "get_evaluator_instance",
+    "clear_evaluator_cache",
+    # Built-in evaluators
+    "RegexEvaluator",
+    "RegexEvaluatorConfig",
+    "ListEvaluator",
+    "ListEvaluatorConfig",
+    "JSONEvaluator",
+    "JSONEvaluatorConfig",
+    "SQLEvaluator",
+    "SQLEvaluatorConfig",
+]

agent_control_evaluators/_base.py

@@ -0,0 +1,178 @@
+"""Evaluator base classes and metadata."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
+
+from agent_control_models import EvaluatorResult
+from agent_control_models.base import BaseModel
+
+if TYPE_CHECKING:
+    from typing import Self
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluatorConfig(BaseModel):
+    """Base class for typed evaluator configurations.
+
+    Extends the project's BaseModel to ensure consistent behavior
+    and enable type checking across all evaluator configs.
+
+    Example:
+        ```python
+        from agent_control_evaluators import EvaluatorConfig
+
+        class MyEvaluatorConfig(EvaluatorConfig):
+            pattern: str
+            threshold: float = 0.5
+        ```
+    """
+
+    pass
+
+
+ConfigT = TypeVar("ConfigT", bound=EvaluatorConfig)
+
+
+@dataclass
+class EvaluatorMetadata:
+    """Metadata about an evaluator.
+
+    Attributes:
+        name: Unique evaluator name (e.g., "regex", "galileo.luna2")
+        version: Evaluator version string
+        description: Human-readable description
+        requires_api_key: Whether the evaluator requires an API key
+        timeout_ms: Default timeout in milliseconds
+    """
+
+    name: str
+    version: str
+    description: str
+    requires_api_key: bool = False
+    timeout_ms: int = 10000
+
+
+class Evaluator(ABC, Generic[ConfigT]):  # noqa: UP046 - need Python 3.10 compat
+    """Base class for all evaluators (built-in, external, or custom).
+
+    All evaluators follow the same pattern:
+        1. Define metadata and config_model as class variables
+        2. Implement evaluate() method
+        3. Register with @register_evaluator decorator
+
+    IMPORTANT - Instance Caching & Thread Safety:
+        Evaluator instances are cached and reused across multiple evaluate() calls
+        when they have the same configuration. This means:
+
+        - DO NOT store mutable request-scoped state on `self`
+        - The evaluate() method may be called concurrently from multiple requests
+        - Any state stored in __init__ should be immutable or thread-safe
+        - Use local variables within evaluate() for request-specific state
+
+        Good pattern:
+            def __init__(self, config):
+                super().__init__(config)
+                self._compiled_regex = re.compile(config.pattern)  # OK: immutable
+
+            async def evaluate(self, data):
+                result = self._compiled_regex.search(data)  # OK: uses immutable state
+                return EvaluatorResult(matched=result is not None, ...)
+
+        Bad pattern:
+            def __init__(self, config):
+                super().__init__(config)
+                self.call_count = 0  # BAD: mutable state shared across requests
+
+            async def evaluate(self, data):
+                self.call_count += 1  # BAD: race condition, leaks between requests
+
+    Example:
+        ```python
+        from agent_control_evaluators import (
+            Evaluator,
+            EvaluatorConfig,
+            EvaluatorMetadata,
+            register_evaluator,
+        )
+        from agent_control_models import EvaluatorResult
+
+        class MyConfig(EvaluatorConfig):
+            threshold: float = 0.5
+
+        @register_evaluator
+        class MyEvaluator(Evaluator[MyConfig]):
+            metadata = EvaluatorMetadata(
+                name="my-evaluator",
+                version="1.0.0",
+                description="My custom evaluator",
+            )
+            config_model = MyConfig
+
+            async def evaluate(self, data: Any) -> EvaluatorResult:
+                return EvaluatorResult(
+                    matched=len(str(data)) > self.config.threshold,
+                    confidence=1.0,
+                    message="Evaluation complete"
+                )
+        ```
+    """
+
+    metadata: ClassVar[EvaluatorMetadata]
+    config_model: ClassVar[type[EvaluatorConfig]]
+
+    def __init__(self, config: ConfigT) -> None:
+        """Initialize evaluator with validated config.
+
+        Args:
+            config: Validated configuration (instance of config_model)
+        """
+        self.config: ConfigT = config
+
+    @classmethod
+    def from_dict(cls, config_dict: dict[str, Any]) -> Self:
+        """Create evaluator instance from raw config dict.
+
+        Validates config against config_model before creating instance.
+
+        Args:
+            config_dict: Raw configuration dictionary
+
+        Returns:
+            Evaluator instance with validated config
+        """
+        validated = cls.config_model(**config_dict)
+        return cls(validated)  # type: ignore[arg-type]
+
+    @abstractmethod
+    async def evaluate(self, data: Any) -> EvaluatorResult:
+        """Evaluate data and return result.
+
+        Args:
+            data: Data extracted by selector from the payload
+
+        Returns:
+            EvaluatorResult with matched status, confidence, and message
+        """
+        pass
+
+    def get_timeout_seconds(self) -> float:
+        """Get timeout in seconds from config or metadata default."""
+        timeout_ms: int = getattr(self.config, "timeout_ms", self.metadata.timeout_ms)
+        return float(timeout_ms) / 1000.0
+
+    @classmethod
+    def is_available(cls) -> bool:
+        """Check if evaluator dependencies are satisfied.
+
+        Override this method for evaluators with optional dependencies.
+        Return False to skip registration during discovery.
+
+        Returns:
+            True if evaluator can be used, False otherwise
+        """
+        return True

agent_control_evaluators/_discovery.py

@@ -0,0 +1,109 @@
+"""Evaluator discovery via entry points."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING, Any
+
+from agent_control_evaluators._registry import (
+    get_all_evaluators,
+    get_evaluator,
+    register_evaluator,
+)
+
+if TYPE_CHECKING:
+    from agent_control_evaluators._base import Evaluator
+
+logger = logging.getLogger(__name__)
+
+_DISCOVERY_COMPLETE = False
+_DISCOVERY_LOCK = threading.Lock()
+
+
+def discover_evaluators() -> int:
+    """Discover and register evaluators via entry points.
+
+    All evaluators (built-in and third-party) are discovered via the
+    'agent_control.evaluators' entry point group. Evaluators are only registered
+    if their `is_available()` method returns True.
+
+    Safe to call multiple times - only runs discovery once.
+    Thread-safe via lock.
+
+    Returns:
+        Number of evaluators discovered
+    """
+    global _DISCOVERY_COMPLETE
+
+    # Fast path without lock
+    if _DISCOVERY_COMPLETE:
+        return 0
+
+    with _DISCOVERY_LOCK:
+        # Double-check after acquiring lock
+        if _DISCOVERY_COMPLETE:
+            return 0
+
+        discovered = 0
+
+        # Discover ALL evaluators (built-in and third-party) via entry points.
+        # Only register evaluators where is_available() returns True.
+        try:
+            eps = entry_points(group="agent_control.evaluators")
+            for ep in eps:
+                try:
+                    evaluator_class = ep.load()
+                    name = evaluator_class.metadata.name
+
+                    # Skip if already registered
+                    if get_evaluator(name) is not None:
+                        continue
+
+                    # Check if evaluator dependencies are satisfied
+                    if not evaluator_class.is_available():
+                        logger.debug(f"Evaluator '{name}' not available, skipping")
+                        continue
+
+                    register_evaluator(evaluator_class)
+                    logger.debug(f"Registered evaluator: {name}")
+                    discovered += 1
+                except Exception as e:
+                    logger.warning(f"Failed to load evaluator '{ep.name}': {e}")
+        except Exception as e:
+            logger.debug(f"Entry point discovery not available: {e}")
+
+        _DISCOVERY_COMPLETE = True
+        logger.debug(f"Evaluator discovery complete: {discovered} new evaluators")
+        return discovered
+
+
+def ensure_evaluators_discovered() -> None:
+    """Ensure evaluator discovery has run. Call this before using evaluators."""
+    if not _DISCOVERY_COMPLETE:
+        discover_evaluators()
+
+
+def reset_evaluator_discovery() -> None:
+    """Reset discovery state. Useful for testing."""
+    global _DISCOVERY_COMPLETE
+    with _DISCOVERY_LOCK:
+        _DISCOVERY_COMPLETE = False
+
+
+# =============================================================================
+# Public evaluator API
+# =============================================================================
+
+
+def list_evaluators() -> dict[str, type[Evaluator[Any]]]:
+    """List all registered evaluators.
+
+    This function ensures evaluator discovery has run before returning results.
+
+    Returns:
+        Dictionary mapping evaluator names to evaluator classes
+    """
+    ensure_evaluators_discovered()
+    return get_all_evaluators()

agent_control_evaluators/_factory.py

@@ -0,0 +1,105 @@
+"""Evaluator factory with instance caching."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from collections import OrderedDict
+from typing import TYPE_CHECKING, Any
+
+from agent_control_evaluators._discovery import list_evaluators
+
+if TYPE_CHECKING:
+    from agent_control_models import EvaluatorSpec
+
+    from agent_control_evaluators._base import Evaluator
+
+logger = logging.getLogger(__name__)
+
+# Configuration
+DEFAULT_CACHE_SIZE = 100
+MIN_CACHE_SIZE = 1  # Minimum to avoid infinite loop in eviction
+
+
+def _parse_cache_size() -> int:
+    """Parse EVALUATOR_CACHE_SIZE from env with safe fallback."""
+    raw = os.environ.get("EVALUATOR_CACHE_SIZE")
+    if raw is None:
+        return DEFAULT_CACHE_SIZE
+    try:
+        return int(raw)
+    except ValueError:
+        logger.warning(
+            f"Invalid EVALUATOR_CACHE_SIZE '{raw}', using default {DEFAULT_CACHE_SIZE}"
+        )
+        return DEFAULT_CACHE_SIZE
+
+
+EVALUATOR_CACHE_SIZE = max(_parse_cache_size(), MIN_CACHE_SIZE)
+
+# LRU cache for evaluator instances: cache_key -> Evaluator instance
+_EVALUATOR_CACHE: OrderedDict[str, Evaluator[Any]] = OrderedDict()
+
+
+def _config_hash(config: dict[str, Any]) -> str:
+    """Create a hashable key from config dict."""
+    return json.dumps(config, sort_keys=True, default=str)
+
+
+def get_evaluator_instance(evaluator_spec: EvaluatorSpec) -> Evaluator[Any]:
+    """Get or create a cached evaluator instance from specification.
+
+    Uses LRU caching to reuse evaluator instances with the same config.
+    Cache key is: {evaluator_name}:{config_hash}
+
+    WARNING: Evaluator instances are cached and reused across requests!
+    Evaluator implementations MUST be stateless - do not store mutable
+    request-scoped state on the evaluator instance. See Evaluator
+    docstring for details on safe patterns.
+
+    Args:
+        evaluator_spec: The evaluator specification with name and config
+
+    Returns:
+        Evaluator instance (cached or new)
+
+    Raises:
+        ValueError: If evaluator not found
+    """
+    # Build cache key
+    cache_key = f"{evaluator_spec.name}:{_config_hash(evaluator_spec.config)}"
+
+    # Check cache
+    if cache_key in _EVALUATOR_CACHE:
+        # Move to end (most recently used)
+        _EVALUATOR_CACHE.move_to_end(cache_key)
+        logger.debug(f"Cache hit for evaluator: {evaluator_spec.name}")
+        return _EVALUATOR_CACHE[cache_key]
+
+    # Cache miss - create new instance
+    evaluators = list_evaluators()
+    evaluator_cls = evaluators.get(evaluator_spec.name)
+
+    if evaluator_cls is None:
+        raise ValueError(
+            f"Evaluator '{evaluator_spec.name}' not found. "
+            f"Available evaluators: {', '.join(evaluators.keys())}"
+        )
+
+    logger.debug(f"Cache miss, creating evaluator: {evaluator_spec.name}")
+    instance = evaluator_cls.from_dict(evaluator_spec.config)
+
+    # Evict oldest if cache is full
+    while len(_EVALUATOR_CACHE) >= EVALUATOR_CACHE_SIZE:
+        evicted_key, _ = _EVALUATOR_CACHE.popitem(last=False)
+        logger.debug(f"Evicted evaluator from cache: {evicted_key}")
+
+    # Cache the instance
+    _EVALUATOR_CACHE[cache_key] = instance
+    return instance
+
+
+def clear_evaluator_cache() -> None:
+    """Clear all cached evaluator instances. Useful for testing."""
+    _EVALUATOR_CACHE.clear()

agent_control_evaluators/_registry.py

@@ -0,0 +1,87 @@
+"""Evaluator registry for registration and lookup."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from agent_control_evaluators._base import Evaluator
+
+logger = logging.getLogger(__name__)
+
+# =============================================================================
+# Evaluator Registry
+# =============================================================================
+
+_EVALUATOR_REGISTRY: dict[str, type[Evaluator[Any]]] = {}
+
+
+def register_evaluator(
+    evaluator_class: type[Evaluator[Any]],
+) -> type[Evaluator[Any]]:
+    """Register an evaluator class by its metadata name.
+
+    Can be used as a decorator or called directly. Respects the evaluator's
+    is_available() method - evaluators with unavailable dependencies are
+    silently skipped.
+
+    Args:
+        evaluator_class: Evaluator class to register
+
+    Returns:
+        The same evaluator class (for decorator usage)
+
+    Raises:
+        ValueError: If evaluator name already registered with different class
+
+    Example:
+        ```python
+        @register_evaluator
+        class MyEvaluator(Evaluator[MyConfig]):
+            metadata = EvaluatorMetadata(name="my-evaluator", ...)
+            ...
+        ```
+    """
+    name = evaluator_class.metadata.name
+
+    # Check if evaluator dependencies are satisfied
+    if not evaluator_class.is_available():
+        logger.debug(f"Evaluator '{name}' not available (is_available=False), skipping")
+        return evaluator_class
+
+    if name in _EVALUATOR_REGISTRY:
+        # Allow re-registration of same class (e.g., during hot reload)
+        if _EVALUATOR_REGISTRY[name] is evaluator_class:
+            return evaluator_class
+        raise ValueError(f"Evaluator '{name}' is already registered")
+
+    _EVALUATOR_REGISTRY[name] = evaluator_class
+    logger.debug(f"Registered evaluator: {name} v{evaluator_class.metadata.version}")
+    return evaluator_class
+
+
+def get_evaluator(name: str) -> type[Evaluator[Any]] | None:
+    """Get a registered evaluator by name.
+
+    Args:
+        name: Evaluator name to look up
+
+    Returns:
+        Evaluator class if found, None otherwise
+    """
+    return _EVALUATOR_REGISTRY.get(name)
+
+
+def get_all_evaluators() -> dict[str, type[Evaluator[Any]]]:
+    """Get all registered evaluators.
+
+    Returns:
+        Dictionary mapping evaluator names to evaluator classes
+    """
+    return dict(_EVALUATOR_REGISTRY)
+
+
+def clear_evaluators() -> None:
+    """Clear all registered evaluators. Useful for testing."""
+    _EVALUATOR_REGISTRY.clear()

agent_control_evaluators/json/__init__.py

@@ -0,0 +1,6 @@
+"""JSON validation evaluator."""
+
+from agent_control_evaluators.json.config import JSONEvaluatorConfig
+from agent_control_evaluators.json.evaluator import JSONEvaluator
+
+__all__ = ["JSONEvaluator", "JSONEvaluatorConfig"]