adversarial-workflow 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

adversarial_workflow/evaluators/discovery.py

@@ -0,0 +1,212 @@
+"""
+YAML parsing and discovery for custom evaluators.
+
+This module handles discovering evaluator definitions from
+.adversarial/evaluators/*.yml files and parsing them into
+EvaluatorConfig objects.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+
+import yaml
+
+from .config import EvaluatorConfig
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluatorParseError(Exception):
+    """Raised when evaluator YAML is invalid."""
+
+
+def parse_evaluator_yaml(yml_file: Path) -> EvaluatorConfig:
+    """Parse a YAML file into an EvaluatorConfig.
+
+    Args:
+        yml_file: Path to the YAML file
+
+    Returns:
+        EvaluatorConfig instance
+
+    Raises:
+        EvaluatorParseError: If YAML is invalid or missing required fields
+        yaml.YAMLError: If YAML syntax is invalid
+    """
+    # Read file with explicit UTF-8 encoding
+    try:
+        content = yml_file.read_text(encoding="utf-8")
+    except UnicodeDecodeError as e:
+        raise EvaluatorParseError(
+            f"File encoding error (not UTF-8): {yml_file}"
+        ) from e
+
+    # Parse YAML
+    data = yaml.safe_load(content)
+
+    # Check for empty YAML
+    if data is None or (isinstance(data, str) and not data.strip()):
+        raise EvaluatorParseError(f"Empty or invalid YAML file: {yml_file}")
+
+    # Ensure parsed data is a dict (YAML can parse scalars, lists, etc.)
+    if not isinstance(data, dict):
+        raise EvaluatorParseError(
+            f"YAML must be a mapping, got {type(data).__name__}: {yml_file}"
+        )
+
+    # Validate required fields exist
+    required = ["name", "description", "model", "api_key_env", "prompt", "output_suffix"]
+    missing = [f for f in required if f not in data]
+    if missing:
+        raise EvaluatorParseError(f"Missing required fields: {', '.join(missing)}")
+
+    # Validate required fields are strings (YAML can parse 'yes' as bool, '123' as int)
+    for field in required:
+        value = data[field]
+        if not isinstance(value, str):
+            raise EvaluatorParseError(
+                f"Field '{field}' must be a string, got {type(value).__name__}: {value!r}"
+            )
+
+    # Validate name format (valid CLI command name)
+    name = data["name"]
+    if not re.match(r"^[a-zA-Z][a-zA-Z0-9_-]*$", name):
+        raise EvaluatorParseError(
+            f"Invalid evaluator name '{name}': must start with letter, "
+            "contain only letters, numbers, hyphens, underscores"
+        )
+
+    # Normalize aliases (handle None, string, or list)
+    aliases = data.get("aliases")
+    if aliases is None:
+        data["aliases"] = []
+    elif isinstance(aliases, str):
+        data["aliases"] = [aliases]
+    elif not isinstance(aliases, list):
+        raise EvaluatorParseError(
+            f"aliases must be string or list, got {type(aliases).__name__}"
+        )
+
+    # Validate alias names - must be strings with valid format
+    for alias in data.get("aliases", []):
+        if not isinstance(alias, str):
+            raise EvaluatorParseError(
+                f"Alias must be a string, got {type(alias).__name__}: {alias!r}"
+            )
+        if not re.match(r"^[a-zA-Z][a-zA-Z0-9_-]*$", alias):
+            raise EvaluatorParseError(
+                f"Invalid alias '{alias}': must start with letter, "
+                "contain only letters, numbers, hyphens, underscores"
+            )
+
+    # Validate prompt is non-empty
+    prompt = data.get("prompt", "")
+    if not prompt or not prompt.strip():
+        raise EvaluatorParseError("prompt cannot be empty")
+
+    # Validate optional string fields if present (YAML can parse '2' as int, 'yes' as bool)
+    optional_string_fields = ["log_prefix", "fallback_model", "version"]
+    for field in optional_string_fields:
+        if field in data and data[field] is not None:
+            value = data[field]
+            if not isinstance(value, str):
+                raise EvaluatorParseError(
+                    f"Field '{field}' must be a string, got {type(value).__name__}: {value!r}"
+                )
+
+    # Filter to known fields only (log unknown fields)
+    known_fields = {
+        "name",
+        "description",
+        "model",
+        "api_key_env",
+        "prompt",
+        "output_suffix",
+        "log_prefix",
+        "fallback_model",
+        "aliases",
+        "version",
+    }
+    unknown = set(data.keys()) - known_fields
+    if unknown:
+        logger.warning(
+            "Unknown fields in %s: %s", yml_file.name, ", ".join(sorted(unknown))
+        )
+
+    # Build filtered data dict
+    filtered_data = {k: v for k, v in data.items() if k in known_fields}
+
+    # Create config with metadata
+    config = EvaluatorConfig(
+        **filtered_data,
+        source="local",
+        config_file=str(yml_file),
+    )
+
+    return config
+
+
+def discover_local_evaluators(
+    base_path: Path | None = None,
+) -> dict[str, EvaluatorConfig]:
+    """Discover evaluators from .adversarial/evaluators/*.yml
+
+    Args:
+        base_path: Project root (default: current directory)
+
+    Returns:
+        Dict mapping evaluator name (and aliases) to EvaluatorConfig
+    """
+    if base_path is None:
+        base_path = Path.cwd()
+
+    evaluators: dict[str, EvaluatorConfig] = {}
+    local_dir = base_path / ".adversarial" / "evaluators"
+
+    if not local_dir.exists():
+        return evaluators
+
+    # Get yml files with error handling for permission/access issues
+    try:
+        yml_files = sorted(local_dir.glob("*.yml"))
+    except OSError as e:
+        logger.warning("Could not read evaluators directory: %s", e)
+        return evaluators
+
+    for yml_file in yml_files:
+        try:
+            config = parse_evaluator_yaml(yml_file)
+
+            # Check for name conflicts
+            if config.name in evaluators:
+                logger.warning(
+                    "Evaluator '%s' in %s conflicts with existing; skipping",
+                    config.name,
+                    yml_file.name,
+                )
+                continue
+
+            # Register primary name
+            evaluators[config.name] = config
+
+            # Register aliases (point to same config object)
+            for alias in config.aliases:
+                if alias in evaluators:
+                    logger.warning(
+                        "Alias '%s' conflicts with existing evaluator; skipping alias",
+                        alias,
+                    )
+                    continue
+                evaluators[alias] = config
+
+        except EvaluatorParseError as e:
+            logger.warning("Skipping %s: %s", yml_file.name, e)
+        except yaml.YAMLError as e:
+            logger.warning("Skipping %s: YAML syntax error: %s", yml_file.name, e)
+        except OSError as e:
+            logger.warning("Could not load %s: %s", yml_file.name, e)
+
+    return evaluators

adversarial_workflow/evaluators/runner.py

@@ -0,0 +1,313 @@
+"""Generic evaluator runner."""
+
+from __future__ import annotations
+
+import os
+import platform
+import shutil
+import subprocess
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .config import EvaluatorConfig
+from ..utils.colors import RESET, BOLD, GREEN, YELLOW, RED
+from ..utils.config import load_config
+from ..utils.validation import validate_evaluation_output
+
+
+def run_evaluator(config: EvaluatorConfig, file_path: str, timeout: int = 180) -> int:
+    """Run an evaluator on a file.
+
+    Args:
+        config: Evaluator configuration
+        file_path: Path to file to evaluate
+        timeout: Timeout in seconds (default: 180)
+
+    Returns:
+        0 on success, non-zero on failure
+    """
+    prefix = config.log_prefix or config.name.upper()
+    print(f"{prefix}: Evaluating {file_path}")
+    print()
+
+    # 1. Validate file exists
+    if not os.path.exists(file_path):
+        print(f"{RED}Error: File not found: {file_path}{RESET}")
+        return 1
+
+    # 2. Load project config (check initialization first)
+    config_path = Path(".adversarial/config.yml")
+    if not config_path.exists():
+        print(f"{RED}Error: Not initialized. Run 'adversarial init' first.{RESET}")
+        return 1
+    project_config = load_config()
+
+    # 3. Check aider available
+    if not shutil.which("aider"):
+        print(f"{RED}Error: Aider not found{RESET}")
+        _print_aider_help()
+        return 1
+
+    # 4. Check API key
+    api_key = os.environ.get(config.api_key_env)
+    if not api_key:
+        print(f"{RED}Error: {config.api_key_env} not set{RESET}")
+        print(f"   Set in .env or export {config.api_key_env}=your-key")
+        return 1
+
+    # 5. Pre-flight file size check
+    line_count, estimated_tokens = _check_file_size(file_path)
+    if line_count > 500 or estimated_tokens > 20000:
+        _warn_large_file(line_count, estimated_tokens)
+        if line_count > 700:
+            if not _confirm_continue():
+                print("Evaluation cancelled.")
+                return 0
+
+    # 6. Determine execution method
+    if config.source == "builtin":
+        return _run_builtin_evaluator(config, file_path, project_config, timeout)
+    else:
+        return _run_custom_evaluator(config, file_path, project_config, timeout)
+
+
+def _run_builtin_evaluator(
+    config: EvaluatorConfig,
+    file_path: str,
+    project_config: dict,
+    timeout: int,
+) -> int:
+    """Run a built-in evaluator using existing shell scripts."""
+    script_map = {
+        "evaluate": ".adversarial/scripts/evaluate_plan.sh",
+        "proofread": ".adversarial/scripts/proofread_content.sh",
+        "review": ".adversarial/scripts/code_review.sh",
+    }
+
+    script = script_map.get(config.name)
+    if not script or not os.path.exists(script):
+        print(f"{RED}Error: Script not found: {script}{RESET}")
+        print("   Fix: Run 'adversarial init' to reinstall scripts")
+        return 1
+
+    return _execute_script(script, file_path, config, project_config, timeout)
+
+
+def _run_custom_evaluator(
+    config: EvaluatorConfig,
+    file_path: str,
+    project_config: dict,
+    timeout: int,
+) -> int:
+    """Run a custom evaluator by invoking aider directly."""
+    # Prepare output path
+    logs_dir = Path(project_config["log_directory"])
+    logs_dir.mkdir(parents=True, exist_ok=True)
+
+    file_basename = Path(file_path).stem
+    output_file = logs_dir / f"{file_basename}-{config.output_suffix}.md"
+
+    # Read input file
+    file_content = Path(file_path).read_text()
+
+    # Build full prompt
+    full_prompt = f"""{config.prompt}
+
+---
+
+## Document to Evaluate
+
+**File**: {file_path}
+
+{file_content}
+"""
+
+    # Create temp file for prompt
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
+        f.write(full_prompt)
+        prompt_file = f.name
+
+    prefix = config.log_prefix or config.name.upper()
+
+    try:
+        print(f"{prefix}: Using model {config.model}")
+
+        # Build aider command
+        cmd = [
+            "aider",
+            "--model", config.model,
+            "--yes",
+            "--no-git",
+            "--no-auto-commits",
+            "--message-file", prompt_file,
+            "--read", file_path,
+        ]
+
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            env=os.environ,
+        )
+
+        # Check for errors
+        output = result.stdout + result.stderr
+        if "RateLimitError" in output or "tokens per min" in output:
+            _print_rate_limit_error(file_path)
+            return 1
+
+        # Write output
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+        header = f"""# {config.output_suffix.replace('-', ' ').replace('_', ' ').title()}
+
+**Source**: {file_path}
+**Evaluator**: {config.name}
+**Model**: {config.model}
+**Generated**: {timestamp}
+
+---
+
+"""
+        output_file.write_text(header + result.stdout)
+
+        print(f"{prefix}: Output written to {output_file}")
+
+        # Validate output and determine verdict
+        is_valid, verdict, message = validate_evaluation_output(str(output_file))
+
+        if not is_valid:
+            print(f"{RED}Evaluation failed: {message}{RESET}")
+            return 1
+
+        return _report_verdict(verdict, output_file, config)
+
+    except subprocess.TimeoutExpired:
+        _print_timeout_error(timeout)
+        return 1
+    except FileNotFoundError:
+        _print_platform_error()
+        return 1
+    finally:
+        Path(prompt_file).unlink(missing_ok=True)
+
+
+def _execute_script(
+    script: str,
+    file_path: str,
+    config: EvaluatorConfig,
+    project_config: dict,
+    timeout: int,
+) -> int:
+    """Execute a shell script evaluator."""
+    try:
+        result = subprocess.run(
+            [script, file_path],
+            text=True,
+            capture_output=True,
+            timeout=timeout,
+        )
+
+        # Check for rate limit errors
+        output = result.stdout + result.stderr
+        if "RateLimitError" in output or "tokens per min" in output:
+            _print_rate_limit_error(file_path)
+            return 1
+
+    except subprocess.TimeoutExpired:
+        _print_timeout_error(timeout)
+        return 1
+    except FileNotFoundError:
+        _print_platform_error()
+        return 1
+
+    # Validate output
+    file_basename = Path(file_path).stem
+    log_file = Path(project_config["log_directory"]) / f"{file_basename}-{config.output_suffix}.md"
+
+    is_valid, verdict, message = validate_evaluation_output(str(log_file))
+
+    if not is_valid:
+        print(f"{RED}Evaluation failed: {message}{RESET}")
+        return 1
+
+    return _report_verdict(verdict, log_file, config)
+
+
+def _report_verdict(verdict: str | None, log_file: Path, config: EvaluatorConfig) -> int:
+    """Report the evaluation verdict to terminal."""
+    print()
+    if verdict == "APPROVED":
+        print(f"{GREEN}Evaluation APPROVED!{RESET}")
+        print(f"   Review output: {log_file}")
+        return 0
+    elif verdict == "NEEDS_REVISION":
+        print(f"{YELLOW}Evaluation NEEDS_REVISION{RESET}")
+        print(f"   Details: {log_file}")
+        return 1
+    elif verdict == "REJECTED":
+        print(f"{RED}Evaluation REJECTED{RESET}")
+        print(f"   Details: {log_file}")
+        return 1
+    else:
+        print(f"{YELLOW}Evaluation complete (verdict: {verdict}){RESET}")
+        print(f"   Review output: {log_file}")
+        return 0
+
+
+# Helper functions
+def _check_file_size(file_path: str) -> tuple[int, int]:
+    """Return (line_count, estimated_tokens)."""
+    with open(file_path, "r") as f:
+        lines = f.readlines()
+        f.seek(0)
+        content = f.read()
+    return len(lines), len(content) // 4
+
+
+def _warn_large_file(line_count: int, tokens: int) -> None:
+    """Print large file warning."""
+    print(f"{YELLOW}Large file detected:{RESET}")
+    print(f"   Lines: {line_count:,}")
+    print(f"   Estimated tokens: ~{tokens:,}")
+    print()
+
+
+def _confirm_continue() -> bool:
+    """Ask user to confirm continuing with large file."""
+    response = input("Continue anyway? [y/N]: ").strip().lower()
+    return response in ["y", "yes"]
+
+
+def _print_aider_help() -> None:
+    """Print aider installation help."""
+    print()
+    print(f"{BOLD}FIX:{RESET}")
+    print("   1. Install aider: pip install aider-chat")
+    print("   2. Verify: aider --version")
+
+
+def _print_rate_limit_error(file_path: str) -> None:
+    """Print rate limit error with suggestions."""
+    print(f"{RED}Error: API rate limit exceeded{RESET}")
+    print()
+    print(f"{BOLD}SOLUTIONS:{RESET}")
+    print("   1. Split into smaller documents (<500 lines)")
+    print("   2. Upgrade your API tier")
+    print("   3. Wait and retry")
+
+
+def _print_timeout_error(timeout: int) -> None:
+    """Print timeout error."""
+    print(f"{RED}Error: Evaluation timed out (>{timeout}s){RESET}")
+
+
+def _print_platform_error() -> None:
+    """Print platform compatibility error."""
+    if platform.system() == "Windows":
+        print(f"{RED}Error: Windows not supported{RESET}")
+        print("   Use WSL (Windows Subsystem for Linux)")
+    else:
+        print(f"{RED}Error: Script not found{RESET}")
+        print("   Run: adversarial init")

adversarial_workflow/utils/__init__.py

@@ -0,0 +1,17 @@
+"""Shared utilities for adversarial-workflow."""
+
+from .colors import BOLD, CYAN, GRAY, GREEN, RED, RESET, YELLOW
+from .config import load_config
+from .validation import validate_evaluation_output
+
+__all__ = [
+    "BOLD",
+    "CYAN",
+    "GRAY",
+    "GREEN",
+    "RED",
+    "RESET",
+    "YELLOW",
+    "load_config",
+    "validate_evaluation_output",
+]

adversarial_workflow/utils/colors.py

@@ -0,0 +1,9 @@
+"""Terminal color constants."""
+
+RESET = "\033[0m"
+BOLD = "\033[1m"
+GREEN = "\033[92m"
+YELLOW = "\033[93m"
+RED = "\033[91m"
+CYAN = "\033[96m"
+GRAY = "\033[90m"

adversarial_workflow/utils/config.py

@@ -0,0 +1,44 @@
+"""Configuration loading utilities."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import yaml
+
+
+def load_config(config_path: str = ".adversarial/config.yml") -> dict[str, Any]:
+    """Load configuration from YAML file with environment variable overrides."""
+    # Default configuration
+    config: dict[str, Any] = {
+        "evaluator_model": "gpt-4o",
+        "task_directory": "tasks/",
+        "test_command": "pytest",
+        "log_directory": ".adversarial/logs/",
+        "artifacts_directory": ".adversarial/artifacts/",
+    }
+
+    # Load from file if exists
+    if os.path.exists(config_path):
+        with open(config_path) as f:
+            file_config = yaml.safe_load(f) or {}
+            if not isinstance(file_config, dict):
+                raise ValueError(
+                    f"Config file must be a mapping, got {type(file_config).__name__}"
+                )
+            config.update(file_config)
+
+    # Override with environment variables
+    env_overrides = {
+        "ADVERSARIAL_EVALUATOR_MODEL": "evaluator_model",
+        "ADVERSARIAL_TEST_COMMAND": "test_command",
+        "ADVERSARIAL_LOG_DIR": "log_directory",
+    }
+
+    for env_var, config_key in env_overrides.items():
+        value = os.getenv(env_var)
+        if value:
+            config[config_key] = value
+
+    return config