agent-control-evaluators 5.0.0__tar.gz

agent_control_evaluators-5.0.0/.gitignore

@@ -0,0 +1,86 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv
+
+# UV
+.uv/
+uv.lock
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+coverage-*.xml
+
+# Testing
+.pytest_cache/
+.coverage
+coverage-*.xml
+htmlcov/
+.tox/
+.mypy_cache/
+.ruff_cache/
+
+# Playwright
+playwright-report/
+playwright/.cache/
+test-results/
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# Logs
+*.log
+logs/
+
+# Database
+*.db
+*.sqlite3
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Intellij
+*.iml
+
+## CLAUDE
+.claude
+
+# Local notes
+rearch_plan.md

agent_control_evaluators-5.0.0/Makefile

@@ -0,0 +1,33 @@
+.PHONY: help sync test lint lint-fix typecheck build publish
+
+PACKAGE := agent-control-evaluators
+
+help:
+	@echo "Agent Control Evaluators - Makefile commands"
+	@echo ""
+	@echo "  make test            - run pytest"
+	@echo "  make lint            - run ruff check"
+	@echo "  make lint-fix        - run ruff check --fix"
+	@echo "  make typecheck       - run mypy"
+	@echo "  make build           - build package"
+
+sync:
+	uv sync
+
+test:
+	uv run pytest --cov=src --cov-report=xml:../../coverage-evaluators.xml -q
+
+lint:
+	uv run ruff check --config ../../pyproject.toml src/
+
+lint-fix:
+	uv run ruff check --config ../../pyproject.toml --fix src/
+
+typecheck:
+	uv run mypy --config-file ../../pyproject.toml src/
+
+build:
+	uv build
+
+publish:
+	uv publish

agent_control_evaluators-5.0.0/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: agent-control-evaluators
+Version: 5.0.0
+Summary: Builtin evaluators for agent-control
+Author: Agent Control Team
+License: Apache-2.0
+Requires-Python: >=3.12
+Requires-Dist: agent-control-models
+Requires-Dist: google-re2>=1.1
+Requires-Dist: jsonschema>=4.0.0
+Requires-Dist: pydantic>=2.12.4
+Requires-Dist: sqlglot[rs]>=20.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Provides-Extra: galileo
+Requires-Dist: agent-control-evaluator-galileo>=3.0.0; extra == 'galileo'
+Description-Content-Type: text/markdown
+
+# Agent Control Evaluators
+
+Built-in evaluators for agent-control.
+
+## Installation
+
+```bash
+pip install agent-control-evaluators
+```
+
+## Available Evaluators
+
+| Name | Description |
+|------|-------------|
+| `regex` | Regular expression pattern matching |
+| `list` | List-based value matching (allow/deny) |
+| `json` | JSON validation (schema, required fields, types) |
+| `sql` | SQL query validation |
+
+## Usage
+
+Evaluators are automatically discovered via Python entry points:
+
+```python
+from agent_control_evaluators import discover_evaluators, list_evaluators
+
+# Load all available evaluators
+discover_evaluators()
+
+# See what's available
+print(list_evaluators())
+# {'regex': <class 'RegexEvaluator'>, 'list': ..., 'json': ..., 'sql': ...}
+```
+
+## External Evaluators
+
+Additional evaluators are available via separate packages:
+
+- `agent-control-evaluator-galileo` - Galileo Luna2 evaluator
+
+```bash
+# Direct install
+pip install agent-control-evaluator-galileo
+
+# Or via convenience extra
+pip install agent-control-evaluators[galileo]
+```
+
+## Creating Custom Evaluators
+
+See [AGENTS.md](../../AGENTS.md) for guidance on creating new evaluators.

agent_control_evaluators-5.0.0/README.md

@@ -0,0 +1,51 @@
+# Agent Control Evaluators
+
+Built-in evaluators for agent-control.
+
+## Installation
+
+```bash
+pip install agent-control-evaluators
+```
+
+## Available Evaluators
+
+| Name | Description |
+|------|-------------|
+| `regex` | Regular expression pattern matching |
+| `list` | List-based value matching (allow/deny) |
+| `json` | JSON validation (schema, required fields, types) |
+| `sql` | SQL query validation |
+
+## Usage
+
+Evaluators are automatically discovered via Python entry points:
+
+```python
+from agent_control_evaluators import discover_evaluators, list_evaluators
+
+# Load all available evaluators
+discover_evaluators()
+
+# See what's available
+print(list_evaluators())
+# {'regex': <class 'RegexEvaluator'>, 'list': ..., 'json': ..., 'sql': ...}
+```
+
+## External Evaluators
+
+Additional evaluators are available via separate packages:
+
+- `agent-control-evaluator-galileo` - Galileo Luna2 evaluator
+
+```bash
+# Direct install
+pip install agent-control-evaluator-galileo
+
+# Or via convenience extra
+pip install agent-control-evaluators[galileo]
+```
+
+## Creating Custom Evaluators
+
+See [AGENTS.md](../../AGENTS.md) for guidance on creating new evaluators.

agent_control_evaluators-5.0.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "agent-control-evaluators"
+version = "5.0.0"
+description = "Builtin evaluators for agent-control"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Agent Control Team" }]
+dependencies = [
+    "agent-control-models",
+    "pydantic>=2.12.4",
+    "google-re2>=1.1",
+    "jsonschema>=4.0.0",
+    "sqlglot[rs]>=20.0.0",
+]
+
+[project.optional-dependencies]
+galileo = ["agent-control-evaluator-galileo>=3.0.0"]
+dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0"]
+
+[project.entry-points."agent_control.evaluators"]
+regex = "agent_control_evaluators.regex:RegexEvaluator"
+list = "agent_control_evaluators.list:ListEvaluator"
+json = "agent_control_evaluators.json:JSONEvaluator"
+sql = "agent_control_evaluators.sql:SQLEvaluator"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_control_evaluators"]
+
+[tool.uv.sources]
+agent-control-models = { workspace = true }
+# For local dev: use local galileo package instead of PyPI
+agent-control-evaluator-galileo = { path = "../extra/galileo", editable = true }

agent_control_evaluators-5.0.0/src/agent_control_evaluators/__init__.py

@@ -0,0 +1,76 @@
+"""Agent Control Evaluators.
+
+This package contains builtin evaluator implementations for agent-control.
+Built-in evaluators (regex, list, json, sql) are registered automatically on import.
+
+Available evaluators:
+    Built-in (no namespace):
+        - regex: Regular expression matching
+        - list: List-based value matching
+        - json: JSON validation
+        - sql: SQL query validation
+
+Naming convention:
+    - Built-in: "regex", "list", "json", "sql"
+    - External: "provider.name" (e.g., "galileo.luna2")
+    - Agent-scoped: "agent:name" (custom code deployed with agent)
+
+External evaluators are installed via separate packages (e.g., agent-control-evaluator-galileo).
+Custom evaluators are Evaluator classes deployed with the engine.
+Their schemas are registered via initAgent for validation purposes.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("agent-control-evaluators")
+except PackageNotFoundError:
+    __version__ = "0.0.0.dev"
+
+# Core infrastructure - export from _base and _registry
+from agent_control_evaluators._base import Evaluator, EvaluatorConfig, EvaluatorMetadata
+from agent_control_evaluators._discovery import (
+    discover_evaluators,
+    ensure_evaluators_discovered,
+    list_evaluators,
+    reset_evaluator_discovery,
+)
+from agent_control_evaluators._factory import clear_evaluator_cache, get_evaluator_instance
+from agent_control_evaluators._registry import (
+    clear_evaluators,
+    get_all_evaluators,
+    get_evaluator,
+    register_evaluator,
+)
+
+# Import built-in evaluators to auto-register them
+from agent_control_evaluators.json import JSONEvaluator, JSONEvaluatorConfig
+from agent_control_evaluators.list import ListEvaluator, ListEvaluatorConfig
+from agent_control_evaluators.regex import RegexEvaluator, RegexEvaluatorConfig
+from agent_control_evaluators.sql import SQLEvaluator, SQLEvaluatorConfig
+
+__all__ = [
+    # Core infrastructure
+    "Evaluator",
+    "EvaluatorConfig",
+    "EvaluatorMetadata",
+    "register_evaluator",
+    "get_evaluator",
+    "get_all_evaluators",
+    "clear_evaluators",
+    "discover_evaluators",
+    "ensure_evaluators_discovered",
+    "reset_evaluator_discovery",
+    "list_evaluators",
+    "get_evaluator_instance",
+    "clear_evaluator_cache",
+    # Built-in evaluators
+    "RegexEvaluator",
+    "RegexEvaluatorConfig",
+    "ListEvaluator",
+    "ListEvaluatorConfig",
+    "JSONEvaluator",
+    "JSONEvaluatorConfig",
+    "SQLEvaluator",
+    "SQLEvaluatorConfig",
+]

agent_control_evaluators-5.0.0/src/agent_control_evaluators/_base.py

@@ -0,0 +1,178 @@
+"""Evaluator base classes and metadata."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
+
+from agent_control_models import EvaluatorResult
+from agent_control_models.base import BaseModel
+
+if TYPE_CHECKING:
+    from typing import Self
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluatorConfig(BaseModel):
+    """Base class for typed evaluator configurations.
+
+    Extends the project's BaseModel to ensure consistent behavior
+    and enable type checking across all evaluator configs.
+
+    Example:
+        ```python
+        from agent_control_evaluators import EvaluatorConfig
+
+        class MyEvaluatorConfig(EvaluatorConfig):
+            pattern: str
+            threshold: float = 0.5
+        ```
+    """
+
+    pass
+
+
+ConfigT = TypeVar("ConfigT", bound=EvaluatorConfig)
+
+
+@dataclass
+class EvaluatorMetadata:
+    """Metadata about an evaluator.
+
+    Attributes:
+        name: Unique evaluator name (e.g., "regex", "galileo.luna2")
+        version: Evaluator version string
+        description: Human-readable description
+        requires_api_key: Whether the evaluator requires an API key
+        timeout_ms: Default timeout in milliseconds
+    """
+
+    name: str
+    version: str
+    description: str
+    requires_api_key: bool = False
+    timeout_ms: int = 10000
+
+
+class Evaluator(ABC, Generic[ConfigT]):  # noqa: UP046 - need Python 3.10 compat
+    """Base class for all evaluators (built-in, external, or custom).
+
+    All evaluators follow the same pattern:
+        1. Define metadata and config_model as class variables
+        2. Implement evaluate() method
+        3. Register with @register_evaluator decorator
+
+    IMPORTANT - Instance Caching & Thread Safety:
+        Evaluator instances are cached and reused across multiple evaluate() calls
+        when they have the same configuration. This means:
+
+        - DO NOT store mutable request-scoped state on `self`
+        - The evaluate() method may be called concurrently from multiple requests
+        - Any state stored in __init__ should be immutable or thread-safe
+        - Use local variables within evaluate() for request-specific state
+
+        Good pattern:
+            def __init__(self, config):
+                super().__init__(config)
+                self._compiled_regex = re.compile(config.pattern)  # OK: immutable
+
+            async def evaluate(self, data):
+                result = self._compiled_regex.search(data)  # OK: uses immutable state
+                return EvaluatorResult(matched=result is not None, ...)
+
+        Bad pattern:
+            def __init__(self, config):
+                super().__init__(config)
+                self.call_count = 0  # BAD: mutable state shared across requests
+
+            async def evaluate(self, data):
+                self.call_count += 1  # BAD: race condition, leaks between requests
+
+    Example:
+        ```python
+        from agent_control_evaluators import (
+            Evaluator,
+            EvaluatorConfig,
+            EvaluatorMetadata,
+            register_evaluator,
+        )
+        from agent_control_models import EvaluatorResult
+
+        class MyConfig(EvaluatorConfig):
+            threshold: float = 0.5
+
+        @register_evaluator
+        class MyEvaluator(Evaluator[MyConfig]):
+            metadata = EvaluatorMetadata(
+                name="my-evaluator",
+                version="1.0.0",
+                description="My custom evaluator",
+            )
+            config_model = MyConfig
+
+            async def evaluate(self, data: Any) -> EvaluatorResult:
+                return EvaluatorResult(
+                    matched=len(str(data)) > self.config.threshold,
+                    confidence=1.0,
+                    message="Evaluation complete"
+                )
+        ```
+    """
+
+    metadata: ClassVar[EvaluatorMetadata]
+    config_model: ClassVar[type[EvaluatorConfig]]
+
+    def __init__(self, config: ConfigT) -> None:
+        """Initialize evaluator with validated config.
+
+        Args:
+            config: Validated configuration (instance of config_model)
+        """
+        self.config: ConfigT = config
+
+    @classmethod
+    def from_dict(cls, config_dict: dict[str, Any]) -> Self:
+        """Create evaluator instance from raw config dict.
+
+        Validates config against config_model before creating instance.
+
+        Args:
+            config_dict: Raw configuration dictionary
+
+        Returns:
+            Evaluator instance with validated config
+        """
+        validated = cls.config_model(**config_dict)
+        return cls(validated)  # type: ignore[arg-type]
+
+    @abstractmethod
+    async def evaluate(self, data: Any) -> EvaluatorResult:
+        """Evaluate data and return result.
+
+        Args:
+            data: Data extracted by selector from the payload
+
+        Returns:
+            EvaluatorResult with matched status, confidence, and message
+        """
+        pass
+
+    def get_timeout_seconds(self) -> float:
+        """Get timeout in seconds from config or metadata default."""
+        timeout_ms: int = getattr(self.config, "timeout_ms", self.metadata.timeout_ms)
+        return float(timeout_ms) / 1000.0
+
+    @classmethod
+    def is_available(cls) -> bool:
+        """Check if evaluator dependencies are satisfied.
+
+        Override this method for evaluators with optional dependencies.
+        Return False to skip registration during discovery.
+
+        Returns:
+            True if evaluator can be used, False otherwise
+        """
+        return True

agent_control_evaluators-5.0.0/src/agent_control_evaluators/_discovery.py

@@ -0,0 +1,109 @@
+"""Evaluator discovery via entry points."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING, Any
+
+from agent_control_evaluators._registry import (
+    get_all_evaluators,
+    get_evaluator,
+    register_evaluator,
+)
+
+if TYPE_CHECKING:
+    from agent_control_evaluators._base import Evaluator
+
+logger = logging.getLogger(__name__)
+
+_DISCOVERY_COMPLETE = False
+_DISCOVERY_LOCK = threading.Lock()
+
+
+def discover_evaluators() -> int:
+    """Discover and register evaluators via entry points.
+
+    All evaluators (built-in and third-party) are discovered via the
+    'agent_control.evaluators' entry point group. Evaluators are only registered
+    if their `is_available()` method returns True.
+
+    Safe to call multiple times - only runs discovery once.
+    Thread-safe via lock.
+
+    Returns:
+        Number of evaluators discovered
+    """
+    global _DISCOVERY_COMPLETE
+
+    # Fast path without lock
+    if _DISCOVERY_COMPLETE:
+        return 0
+
+    with _DISCOVERY_LOCK:
+        # Double-check after acquiring lock
+        if _DISCOVERY_COMPLETE:
+            return 0
+
+        discovered = 0
+
+        # Discover ALL evaluators (built-in and third-party) via entry points.
+        # Only register evaluators where is_available() returns True.
+        try:
+            eps = entry_points(group="agent_control.evaluators")
+            for ep in eps:
+                try:
+                    evaluator_class = ep.load()
+                    name = evaluator_class.metadata.name
+
+                    # Skip if already registered
+                    if get_evaluator(name) is not None:
+                        continue
+
+                    # Check if evaluator dependencies are satisfied
+                    if not evaluator_class.is_available():
+                        logger.debug(f"Evaluator '{name}' not available, skipping")
+                        continue
+
+                    register_evaluator(evaluator_class)
+                    logger.debug(f"Registered evaluator: {name}")
+                    discovered += 1
+                except Exception as e:
+                    logger.warning(f"Failed to load evaluator '{ep.name}': {e}")
+        except Exception as e:
+            logger.debug(f"Entry point discovery not available: {e}")
+
+        _DISCOVERY_COMPLETE = True
+        logger.debug(f"Evaluator discovery complete: {discovered} new evaluators")
+        return discovered
+
+
+def ensure_evaluators_discovered() -> None:
+    """Ensure evaluator discovery has run. Call this before using evaluators."""
+    if not _DISCOVERY_COMPLETE:
+        discover_evaluators()
+
+
+def reset_evaluator_discovery() -> None:
+    """Reset discovery state. Useful for testing."""
+    global _DISCOVERY_COMPLETE
+    with _DISCOVERY_LOCK:
+        _DISCOVERY_COMPLETE = False
+
+
+# =============================================================================
+# Public evaluator API
+# =============================================================================
+
+
+def list_evaluators() -> dict[str, type[Evaluator[Any]]]:
+    """List all registered evaluators.
+
+    This function ensures evaluator discovery has run before returning results.
+
+    Returns:
+        Dictionary mapping evaluator names to evaluator classes
+    """
+    ensure_evaluators_discovered()
+    return get_all_evaluators()