adversarial-workflow 0.6.6__py3-none-any.whl → 0.9.0__py3-none-any.whl

adversarial_workflow/evaluators/discovery.py

@@ -4,6 +4,10 @@ YAML parsing and discovery for custom evaluators.
 This module handles discovering evaluator definitions from
 .adversarial/evaluators/*.yml files and parsing them into
 EvaluatorConfig objects.
+
+Supports dual-field model specification (ADV-0015):
+- Legacy: model + api_key_env fields (backwards compatible)
+- New: model_requirement field (resolved via ModelResolver)
 """
 
 from __future__ import annotations
@@ -14,7 +18,7 @@ from pathlib import Path
 
 import yaml
 
-from .config import EvaluatorConfig
+from .config import EvaluatorConfig, ModelRequirement
 
 logger = logging.getLogger(__name__)
 
@@ -54,26 +58,39 @@ def parse_evaluator_yaml(yml_file: Path) -> EvaluatorConfig:
         raise EvaluatorParseError(f"YAML must be a mapping, got {type(data).__name__}: {yml_file}")
 
     # Validate required fields exist
-    required = [
+    # model and api_key_env are only required if model_requirement is not present
+    always_required = [
         "name",
         "description",
-        "model",
-        "api_key_env",
         "prompt",
         "output_suffix",
     ]
-    missing = [f for f in required if f not in data]
+    has_model_requirement = "model_requirement" in data
+    if not has_model_requirement:
+        # Legacy format: model and api_key_env are required
+        always_required.extend(["model", "api_key_env"])
+
+    missing = [f for f in always_required if f not in data]
     if missing:
         raise EvaluatorParseError(f"Missing required fields: {', '.join(missing)}")
 
     # Validate required fields are strings (YAML can parse 'yes' as bool, '123' as int)
-    for field in required:
+    for field in always_required:
         value = data[field]
         if not isinstance(value, str):
             raise EvaluatorParseError(
                 f"Field '{field}' must be a string, got {type(value).__name__}: {value!r}"
             )
 
+    # Validate model and api_key_env are strings if present (even when optional)
+    for field in ["model", "api_key_env"]:
+        if field in data and data[field] is not None:
+            value = data[field]
+            if not isinstance(value, str):
+                raise EvaluatorParseError(
+                    f"Field '{field}' must be a string, got {type(value).__name__}: {value!r}"
+                )
+
     # Validate name format (valid CLI command name)
     name = data["name"]
     if not re.match(r"^[a-zA-Z][a-zA-Z0-9_-]*$", name):
@@ -143,6 +160,67 @@ def parse_evaluator_yaml(yml_file: Path) -> EvaluatorConfig:
             )
             data["timeout"] = 600
 
+    # Parse model_requirement if present (ADV-0015)
+    model_requirement = None
+    if "model_requirement" in data:
+        req_data = data["model_requirement"]
+
+        # Validate model_requirement is a mapping
+        if not isinstance(req_data, dict):
+            raise EvaluatorParseError(
+                f"model_requirement must be a mapping, got {type(req_data).__name__}"
+            )
+
+        # Validate required fields in model_requirement
+        if "family" not in req_data:
+            raise EvaluatorParseError("model_requirement.family is required")
+        if "tier" not in req_data:
+            raise EvaluatorParseError("model_requirement.tier is required")
+
+        # Validate family and tier are strings
+        family = req_data["family"]
+        tier = req_data["tier"]
+        if not isinstance(family, str):
+            raise EvaluatorParseError(
+                f"model_requirement.family must be a string, got {type(family).__name__}"
+            )
+        if not isinstance(tier, str):
+            raise EvaluatorParseError(
+                f"model_requirement.tier must be a string, got {type(tier).__name__}"
+            )
+
+        # Validate optional min_version is string if present
+        min_version = req_data.get("min_version", "")
+        # Reject booleans explicitly (YAML parses 'yes'/'no'/'true'/'false' as bool)
+        if isinstance(min_version, bool):
+            raise EvaluatorParseError(
+                f"model_requirement.min_version must be a string, got bool: {min_version!r}"
+            )
+        # Convert integers to strings (YAML parses '0' as int 0)
+        if isinstance(min_version, int):
+            min_version = str(min_version)
+        elif min_version and not isinstance(min_version, str):
+            raise EvaluatorParseError(
+                f"model_requirement.min_version must be a string, got {type(min_version).__name__}"
+            )
+
+        # Validate optional min_context is integer if present
+        min_context = req_data.get("min_context", 0)
+        # Reject booleans explicitly (YAML parses 'yes'/'no'/'true'/'false' as bool)
+        if isinstance(min_context, bool):
+            raise EvaluatorParseError("model_requirement.min_context must be an integer, got bool")
+        if min_context and not isinstance(min_context, int):
+            raise EvaluatorParseError(
+                f"model_requirement.min_context must be an integer, got {type(min_context).__name__}"
+            )
+
+        model_requirement = ModelRequirement(
+            family=family,
+            tier=tier,
+            min_version=min_version,
+            min_context=min_context,
+        )
+
     # Filter to known fields only (log unknown fields)
     known_fields = {
         "name",
@@ -156,17 +234,27 @@ def parse_evaluator_yaml(yml_file: Path) -> EvaluatorConfig:
         "aliases",
         "version",
         "timeout",
+        "model_requirement",  # ADV-0015
     }
     unknown = set(data.keys()) - known_fields
     if unknown:
         logger.warning("Unknown fields in %s: %s", yml_file.name, ", ".join(sorted(unknown)))
 
-    # Build filtered data dict
-    filtered_data = {k: v for k, v in data.items() if k in known_fields}
+    # Build filtered data dict (exclude model_requirement as it's handled separately)
+    scalar_fields = known_fields - {"model_requirement"}
+    filtered_data = {k: v for k, v in data.items() if k in scalar_fields}
+
+    # Set defaults for optional model/api_key_env when model_requirement is present
+    # Also handle explicit null values (YAML parses empty or null as None)
+    if "model" not in filtered_data or filtered_data["model"] is None:
+        filtered_data["model"] = ""
+    if "api_key_env" not in filtered_data or filtered_data["api_key_env"] is None:
+        filtered_data["api_key_env"] = ""
 
-    # Create config with metadata
+    # Create config with metadata and model_requirement
     config = EvaluatorConfig(
         **filtered_data,
+        model_requirement=model_requirement,
         source="local",
         config_file=str(yml_file),
     )

adversarial_workflow/evaluators/resolver.py

@@ -0,0 +1,211 @@
+"""
+Model resolver for evaluator configurations (ADV-0015: Model Routing Layer - Phase 1).
+
+This module provides the ModelResolver class that resolves model requirements
+to actual model IDs using an embedded registry. It supports:
+- model_requirement field (new structured format)
+- model field (legacy string format)
+- Fallback from model_requirement to model on resolution failure
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from adversarial_workflow.evaluators.config import EvaluatorConfig, ModelRequirement
+
+
+class ResolutionError(Exception):
+    """Raised when model resolution fails."""
+
+
+class ModelResolver:
+    """Resolves model requirements to actual model IDs.
+
+    Uses an embedded registry (matching adversarial-evaluator-library/providers/registry.yml)
+    to map family/tier pairs to concrete model identifiers.
+
+    Resolution order:
+    1. If model_requirement present: resolve via registry
+    2. If resolution fails AND model present: warn + fallback to legacy
+    3. If resolution fails AND no model: raise ResolutionError
+    4. If no model_requirement AND model present: use legacy directly
+    5. If neither: raise ResolutionError
+    """
+
+    # Default registry - matches adversarial-evaluator-library/providers/registry.yml
+    # Updated 2026-02-03 per Library team handoff (ADR-0005)
+    DEFAULT_REGISTRY: ClassVar[dict[str, dict[str, dict[str, list[str] | str]]]] = {
+        "claude": {
+            "opus": {
+                "models": ["claude-4-opus-20260115", "claude-opus-4-5-20251101"],
+                "prefix": "anthropic/",
+            },
+            "sonnet": {
+                "models": ["claude-4-sonnet-20260115"],
+                "prefix": "anthropic/",
+            },
+            "haiku": {
+                "models": ["claude-4-haiku-20260115"],
+                "prefix": "anthropic/",
+            },
+        },
+        "gpt": {
+            "flagship": {
+                "models": ["gpt-4o", "gpt-4o-2024-08-06"],
+                "prefix": "",
+            },
+            "standard": {
+                "models": ["gpt-4-turbo", "gpt-4"],
+                "prefix": "",
+            },
+            "mini": {
+                "models": ["gpt-4o-mini"],
+                "prefix": "",
+            },
+        },
+        "o": {
+            "flagship": {
+                "models": ["o1", "o1-2024-12-17"],
+                "prefix": "",
+            },
+            "mini": {
+                "models": ["o3-mini"],
+                "prefix": "",
+            },
+        },
+        "gemini": {
+            "pro": {
+                "models": ["gemini-2.5-pro"],
+                "prefix": "gemini/",
+            },
+            "flash": {
+                "models": ["gemini-2.5-flash"],
+                "prefix": "gemini/",
+            },
+        },
+        "mistral": {
+            "large": {
+                "models": ["mistral-large-latest"],
+                "prefix": "mistral/",
+            },
+            "small": {
+                "models": ["mistral-small-latest"],
+                "prefix": "mistral/",
+            },
+        },
+        "codestral": {
+            "latest": {
+                "models": ["codestral-latest"],
+                "prefix": "mistral/",
+            },
+        },
+        "llama": {
+            "large": {
+                "models": ["llama-3.3-70b"],
+                "prefix": "",  # varies by host
+            },
+            "medium": {
+                "models": ["llama-3.1-8b"],
+                "prefix": "",
+            },
+        },
+    }
+
+    # API key environment variable mapping by family
+    API_KEY_MAP: ClassVar[dict[str, str]] = {
+        "claude": "ANTHROPIC_API_KEY",
+        "gpt": "OPENAI_API_KEY",
+        "o": "OPENAI_API_KEY",
+        "gemini": "GEMINI_API_KEY",
+        "mistral": "MISTRAL_API_KEY",
+        "codestral": "MISTRAL_API_KEY",
+        "llama": "TOGETHER_API_KEY",
+    }
+
+    def resolve(self, config: EvaluatorConfig) -> tuple[str, str]:
+        """Resolve evaluator config to (model_id, api_key_env).
+
+        Args:
+            config: EvaluatorConfig with model and/or model_requirement
+
+        Returns:
+            (model_id, api_key_env) tuple
+
+        Raises:
+            ResolutionError: If resolution fails and no fallback available
+        """
+        if config.model_requirement:
+            try:
+                return self._resolve_requirement(config.model_requirement)
+            except ResolutionError as e:
+                if config.model:
+                    # Fall back to legacy with warning
+                    warnings.warn(
+                        f"model_requirement resolution failed for {config.name}: {e}. "
+                        f"Falling back to legacy model field: {config.model}",
+                        UserWarning,
+                        stacklevel=2,
+                    )
+                    return (config.model, config.api_key_env)
+                raise
+
+        # Legacy only
+        if config.model:
+            return (config.model, config.api_key_env)
+
+        raise ResolutionError("No model or model_requirement specified")
+
+    def _resolve_requirement(self, req: ModelRequirement) -> tuple[str, str]:
+        """Resolve requirement to model ID using registry.
+
+        Args:
+            req: ModelRequirement with family and tier
+
+        Returns:
+            (model_id, api_key_env) tuple
+
+        Raises:
+            ResolutionError: If family or tier not found in registry
+        """
+        # TODO(Phase 2): ModelRequirement.min_version and ModelRequirement.min_context
+        # are currently parsed but not used for filtering. Phase 1 only performs
+        # family/tier matching. Phase 2 will implement filtering by min_version
+        # and min_context requirements.
+        family = self.DEFAULT_REGISTRY.get(req.family)
+        if not family:
+            raise ResolutionError(f"Unknown model family: {req.family}")
+
+        tier_data = family.get(req.tier)
+        if not tier_data:
+            raise ResolutionError(f"Unknown tier '{req.tier}' for family '{req.family}'")
+
+        # Return first (latest) model in tier
+        models = tier_data.get("models", [])
+        if not models:
+            raise ResolutionError(f"No models defined for {req.family}/{req.tier}")
+        # Registry type is list[str] | str for flexibility; actual values are always lists
+        model_id = models[0]  # type: ignore[index]
+
+        # Apply provider prefix for LiteLLM compatibility
+        prefix = tier_data.get("prefix", "")
+        if prefix:
+            model_id = f"{prefix}{model_id}"
+
+        # Determine API key env from family
+        api_key_env = self._get_api_key_env(req.family)
+
+        return (model_id, api_key_env)
+
+    def _get_api_key_env(self, family: str) -> str:
+        """Get default API key environment variable for family.
+
+        Args:
+            family: Model family name
+
+        Returns:
+            Environment variable name for API key
+        """
+        return self.API_KEY_MAP.get(family, f"{family.upper()}_API_KEY")

adversarial_workflow/evaluators/runner.py

@@ -1,4 +1,9 @@
-"""Generic evaluator runner."""
+"""Generic evaluator runner.
+
+Supports dual-field model specification (ADV-0015):
+- Legacy: model + api_key_env fields (backwards compatible)
+- New: model_requirement field (resolved via ModelResolver)
+"""
 
 from __future__ import annotations
 
@@ -14,6 +19,7 @@ from ..utils.colors import BOLD, GREEN, RED, RESET, YELLOW
 from ..utils.config import load_config
 from ..utils.validation import validate_evaluation_output
 from .config import EvaluatorConfig
+from .resolver import ModelResolver, ResolutionError
 
 
 def run_evaluator(config: EvaluatorConfig, file_path: str, timeout: int = 180) -> int:
@@ -43,20 +49,28 @@ def run_evaluator(config: EvaluatorConfig, file_path: str, timeout: int = 180) -
         return 1
     project_config = load_config()
 
-    # 3. Check aider available
+    # 3. Resolve model (ADV-0015: dual-field support)
+    resolver = ModelResolver()
+    try:
+        resolved_model, resolved_api_key_env = resolver.resolve(config)
+    except ResolutionError as e:
+        print(f"{RED}Error: {e}{RESET}")
+        return 1
+
+    # 4. Check aider available
     if not shutil.which("aider"):
         print(f"{RED}Error: Aider not found{RESET}")
         _print_aider_help()
         return 1
 
-    # 4. Check API key
-    api_key = os.environ.get(config.api_key_env)
+    # 5. Check API key (using resolved api_key_env)
+    api_key = os.environ.get(resolved_api_key_env)
     if not api_key:
-        print(f"{RED}Error: {config.api_key_env} not set{RESET}")
-        print(f"   Set in .env or export {config.api_key_env}=your-key")
+        print(f"{RED}Error: {resolved_api_key_env} not set{RESET}")
+        print(f"   Set in .env or export {resolved_api_key_env}=your-key")
         return 1
 
-    # 5. Pre-flight file size check
+    # 6. Pre-flight file size check
     line_count, estimated_tokens = _check_file_size(file_path)
     if line_count > 500 or estimated_tokens > 20000:
         _warn_large_file(line_count, estimated_tokens)
@@ -65,11 +79,11 @@ def run_evaluator(config: EvaluatorConfig, file_path: str, timeout: int = 180) -
                 print("Evaluation cancelled.")
                 return 0
 
-    # 6. Determine execution method
+    # 7. Determine execution method
     if config.source == "builtin":
         return _run_builtin_evaluator(config, file_path, project_config, timeout)
     else:
-        return _run_custom_evaluator(config, file_path, project_config, timeout)
+        return _run_custom_evaluator(config, file_path, project_config, timeout, resolved_model)
 
 
 def _run_builtin_evaluator(
@@ -99,8 +113,17 @@ def _run_custom_evaluator(
     file_path: str,
     project_config: dict,
     timeout: int,
+    resolved_model: str,
 ) -> int:
-    """Run a custom evaluator by invoking aider directly."""
+    """Run a custom evaluator by invoking aider directly.
+
+    Args:
+        config: Evaluator configuration
+        file_path: Path to file to evaluate
+        project_config: Project configuration dict
+        timeout: Timeout in seconds
+        resolved_model: Resolved model ID from ModelResolver
+    """
     # Prepare output path
     logs_dir = Path(project_config["log_directory"])
     logs_dir.mkdir(parents=True, exist_ok=True)
@@ -131,13 +154,13 @@ def _run_custom_evaluator(
     prefix = config.log_prefix or config.name.upper()
 
     try:
-        print(f"{prefix}: Using model {config.model}")
+        print(f"{prefix}: Using model {resolved_model}")
 
         # Build aider command
         cmd = [
             "aider",
             "--model",
-            config.model,
+            resolved_model,
             "--yes",
             "--no-detect-urls",
             "--no-git",
@@ -168,7 +191,7 @@ def _run_custom_evaluator(
 
 **Source**: {file_path}
 **Evaluator**: {config.name}
-**Model**: {config.model}
+**Model**: {resolved_model}
 **Generated**: {timestamp}
 
 ---

adversarial_workflow/library/__init__.py

@@ -0,0 +1,56 @@
+"""Evaluator library client for adversarial-workflow.
+
+This module provides functionality to browse, install, and update evaluator
+configurations from the community adversarial-evaluator-library.
+
+Philosophy: "Copy, Don't Link"
+- Evaluators are copied to projects, not referenced at runtime
+- Projects remain self-contained and work offline
+- Users can customize their local copies freely
+- Updates are explicit and user-controlled
+"""
+
+from .cache import DEFAULT_CACHE_DIR, DEFAULT_CACHE_TTL, CacheManager
+from .client import (
+    DEFAULT_LIBRARY_URL,
+    LibraryClient,
+    LibraryClientError,
+    NetworkError,
+    ParseError,
+)
+from .commands import (
+    library_check_updates,
+    library_info,
+    library_install,
+    library_list,
+    library_update,
+)
+from .config import LibraryConfig, get_library_config
+from .models import EvaluatorEntry, IndexData, InstalledEvaluatorMeta, UpdateInfo
+
+__all__ = [
+    # Client
+    "LibraryClient",
+    "LibraryClientError",
+    "NetworkError",
+    "ParseError",
+    "DEFAULT_LIBRARY_URL",
+    # Models
+    "EvaluatorEntry",
+    "IndexData",
+    "InstalledEvaluatorMeta",
+    "UpdateInfo",
+    # Cache
+    "CacheManager",
+    "DEFAULT_CACHE_DIR",
+    "DEFAULT_CACHE_TTL",
+    # Config
+    "LibraryConfig",
+    "get_library_config",
+    # Commands
+    "library_list",
+    "library_info",
+    "library_install",
+    "library_check_updates",
+    "library_update",
+]