adversarial-workflow 0.5.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

adversarial_workflow/__init__.py

@@ -12,7 +12,7 @@ Usage:
     adversarial validate "pytest"
 """
 
-__version__ = "0.5.0"
+__version__ = "0.6.1"
 __author__ = "Fredrik Matheson"
 __license__ = "MIT"
 

adversarial_workflow/cli.py

@@ -27,9 +27,9 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
 import yaml
-from dotenv import load_dotenv
+from dotenv import load_dotenv, dotenv_values
 
-__version__ = "0.5.0"
+__version__ = "0.6.1"
 
 # ANSI color codes for better output
 RESET = "\033[0m"
@@ -800,26 +800,37 @@ def check() -> int:
     issues: List[Dict] = []
     good_checks: List[str] = []
 
-    # Check for .env file first (before loading environment variables)
+    # Check for .env file (note: already loaded by main() at startup)
     env_file = Path(".env")
     env_loaded = False
-    env_keys_before = set(os.environ.keys())
 
     if env_file.exists():
         try:
+            # Load .env into environment (idempotent - safe to call again after main())
             load_dotenv(env_file)
-            env_keys_after = set(os.environ.keys())
-            new_keys = env_keys_after - env_keys_before
+            # Use dotenv_values() to count variables directly from file
+            # This gives accurate count regardless of what was already in environment
+            env_vars = dotenv_values(env_file)
             env_loaded = True
             good_checks.append(
-                f".env file found and loaded ({len(new_keys)} variables)"
+                f".env file found ({len(env_vars)} variables configured)"
             )
-        except Exception as e:
+        except (FileNotFoundError, PermissionError) as e:
+            # File access errors
             issues.append(
                 {
                     "severity": "WARNING",
-                    "message": f".env file found but could not be loaded: {e}",
-                    "fix": "Check .env file format and permissions",
+                    "message": f".env file found but could not be read: {e}",
+                    "fix": "Check .env file permissions",
+                }
+            )
+        except (OSError, ValueError) as e:
+            # Covers UnicodeDecodeError (ValueError subclass) and other OS errors
+            issues.append(
+                {
+                    "severity": "WARNING",
+                    "message": f".env file found but could not be parsed: {e}",
+                    "fix": "Check .env file encoding (should be UTF-8)",
                 }
             )
     else:
@@ -2086,225 +2097,8 @@ def evaluate(task_file: str) -> int:
         return 0
 
 
-def proofread(doc_file: str) -> int:
-    """Run proofreading review on teaching/documentation content."""
-
-    print(f"📖 Proofreading document: {doc_file}")
-    print()
-
-    # Error 1: Document file not found
-    if not os.path.exists(doc_file):
-        print(f"{RED}❌ ERROR: Document file not found: {doc_file}{RESET}")
-        print("   Usage: adversarial proofread <document_file>")
-        print("   Example: adversarial proofread docs/guide/concept.md")
-        return 1
-
-    # Error 2: Config not loaded
-    try:
-        config = load_config()
-    except FileNotFoundError:
-        print(f"{RED}❌ ERROR: Not initialized. Run 'adversarial init' first.{RESET}")
-        return 1
-
-    # Error 3: Aider not available
-    if not shutil.which("aider"):
-        print(f"{RED}❌ ERROR: Aider not found{RESET}")
-        print()
-        print(f"{BOLD}WHY:{RESET}")
-        print("   Proofreader uses aider (AI pair programming tool) to:")
-        print("   • Review teaching content quality")
-        print("   • Check clarity and accuracy")
-        print("   • Evaluate pedagogical effectiveness")
-        print()
-        print(f"{BOLD}FIX:{RESET}")
-        print("   1. Install aider: pip install aider-chat")
-        print("   2. Verify installation: aider --version")
-        print("   3. Then retry: adversarial proofread ...")
-        print()
-        print(f"{BOLD}HELP:{RESET}")
-        print("   Aider docs: https://aider.chat/docs/install.html")
-        return 1
-
-    # Pre-flight check for file size
-    with open(doc_file, "r") as f:
-        line_count = len(f.readlines())
-        f.seek(0)
-        file_size = len(f.read())
-
-    # Estimate tokens (1 token ≈ 4 characters)
-    estimated_tokens = file_size // 4
-
-    # Warn if file is large (>500 lines or >20k tokens)
-    if line_count > 500 or estimated_tokens > 20000:
-        print(f"{YELLOW}⚠️  Large file detected:{RESET}")
-        print(f"   Lines: {line_count:,}")
-        print(f"   Estimated tokens: ~{estimated_tokens:,}")
-        print()
-        print(f"{BOLD}Note:{RESET} Files over 500 lines may exceed OpenAI rate limits.")
-        print(
-            f"      If proofreading fails, consider splitting into smaller documents."
-        )
-        print()
-
-        # Give user a chance to cancel for very large files
-        if line_count > 700:
-            print(f"{RED}⚠️  WARNING: File is very large (>{line_count} lines){RESET}")
-            print(f"   This will likely fail on Tier 1 OpenAI accounts (30k TPM limit)")
-            print(f"   Recommended: Split into files <500 lines each")
-            print()
-            response = input("Continue anyway? [y/N]: ").strip().lower()
-            if response not in ["y", "yes"]:
-                print("Proofreading cancelled.")
-                return 0
-            print()
-
-    # Error 4: Script execution fails
-    script = ".adversarial/scripts/proofread_content.sh"
-    if not os.path.exists(script):
-        print(f"{RED}❌ ERROR: Script not found: {script}{RESET}")
-        print("   Fix: Run 'adversarial init' to reinstall scripts")
-        return 1
-
-    try:
-        result = subprocess.run(
-            [script, doc_file], text=True, capture_output=True, timeout=180  # 3 minutes
-        )
-
-        # Check for rate limit errors in output
-        output = result.stdout + result.stderr
-        if "RateLimitError" in output or "tokens per min (TPM)" in output:
-            print(f"{RED}❌ ERROR: OpenAI rate limit exceeded{RESET}")
-            print()
-            print(f"{BOLD}WHY:{RESET}")
-            print(
-                "   Your document file is too large for your OpenAI organization's rate limit"
-            )
-            print()
-
-            # Extract file size for helpful message
-            with open(doc_file, "r") as f:
-                line_count = len(f.readlines())
-
-            print(f"{BOLD}FILE SIZE:{RESET}")
-            print(f"   Lines: {line_count:,}")
-            print(f"   Recommended limit: 500 lines")
-            print()
-            print(f"{BOLD}SOLUTIONS:{RESET}")
-            print("   1. Split your document into smaller files (<500 lines each)")
-            print("   2. Upgrade your OpenAI tier (Tier 2 supports ~1,000 lines)")
-            print("   3. Use manual review for this comprehensive document")
-            print()
-            print(f"{BOLD}MORE INFO:{RESET}")
-            print("   https://platform.openai.com/docs/guides/rate-limits")
-            return 1
-
-    except subprocess.TimeoutExpired:
-        print(f"{RED}❌ ERROR: Proofreading timed out (>3 minutes){RESET}")
-        print()
-        print(f"{BOLD}WHY:{RESET}")
-        print("   The AI model took too long to respond")
-        print()
-        print(f"{BOLD}POSSIBLE CAUSES:{RESET}")
-        print("   • Network issues connecting to API")
-        print("   • Document file too large (>1000 lines)")
-        print("   • API rate limiting")
-        print()
-        print(f"{BOLD}FIX:{RESET}")
-        print("   1. Check your network connection")
-        print("   2. Try a smaller document file")
-        print("   3. Wait a few minutes and retry")
-        return 1
-    except FileNotFoundError as e:
-        # Check if this is a bash/platform issue
-        if platform.system() == "Windows":
-            print(f"{RED}❌ ERROR: Cannot execute workflow scripts{RESET}")
-            print()
-            print(f"{BOLD}WHY:{RESET}")
-            print("   Native Windows (PowerShell/CMD) cannot run bash scripts")
-            print("   This package requires Unix shell (bash) for workflow automation")
-            print()
-            print(f"{BOLD}FIX:{RESET}")
-            print("   Option 1 (RECOMMENDED): Use WSL (Windows Subsystem for Linux)")
-            print(
-                "     1. Install WSL: https://learn.microsoft.com/windows/wsl/install"
-            )
-            print("     2. Open WSL terminal")
-            print("     3. Reinstall package in WSL: pip install adversarial-workflow")
-            print()
-            print("   Option 2: Try Git Bash (not officially supported)")
-            print("     • May have compatibility issues")
-            print("     • WSL is strongly recommended")
-            print()
-            print(f"{BOLD}HELP:{RESET}")
-            print("   See platform requirements: README.md#platform-support")
-        else:
-            print(f"{RED}❌ ERROR: Script not found: {script}{RESET}")
-            print()
-            print(f"{BOLD}WHY:{RESET}")
-            print("   Workflow scripts are missing or corrupted")
-            print()
-            print(f"{BOLD}FIX:{RESET}")
-            print("   Run: adversarial init")
-            print("   This will reinstall all workflow scripts")
-        return 1
-
-    # Error 5: Proofreading rejected
-    if result.returncode != 0:
-        print()
-        print("📋 Proofreading complete (needs revision)")
-        print(f"   Details: {config['log_directory']}")
-        return result.returncode
-
-    # Error 6: Validation - Check if proofreading actually ran (not just empty output)
-    # Extract document name from filename to find log file
-    doc_basename = os.path.basename(doc_file)
-    doc_name = os.path.splitext(doc_basename)[0]
 
-    log_file = os.path.join(config["log_directory"], f"{doc_name}-PROOFREADING.md")
 
-    is_valid, verdict, message = validate_evaluation_output(log_file)
-    if not is_valid:
-        print()
-        print(f"{RED}❌ Proofreading failed: {message}{RESET}")
-        print()
-        print(f"{BOLD}WHY:{RESET}")
-        print("   The proofreading script ran but didn't produce valid output")
-        print("   This usually means Aider encountered an error before running GPT-4o")
-        print()
-        print(f"{BOLD}LOG FILE:{RESET}")
-        print(f"   {log_file}")
-        print()
-        print(f"{BOLD}FIX:{RESET}")
-        print("   1. Check the log file for error messages")
-        print("   2. Ensure your API keys are valid: adversarial check")
-        print("   3. Try running the proofreading again")
-        print()
-        return 1
-
-    # Verify token count (warn if suspiciously low)
-    verify_token_count(doc_file, log_file)
-
-    # Report based on actual verdict from proofreading
-    print()
-    if verdict == "APPROVED":
-        print(f"{GREEN}✅ Proofreading APPROVED!{RESET}")
-        print(f"   Document is ready for publication")
-        print(f"   Review output: {log_file}")
-        return 0
-    elif verdict == "NEEDS_REVISION":
-        print(f"{YELLOW}⚠️  Proofreading NEEDS_REVISION{RESET}")
-        print(f"   Review feedback and update document")
-        print(f"   Details: {log_file}")
-        return 1
-    elif verdict == "REJECTED":
-        print(f"{RED}❌ Proofreading REJECTED{RESET}")
-        print(f"   Document has fundamental issues - major revision needed")
-        print(f"   Details: {log_file}")
-        return 1
-    else:  # UNKNOWN or other
-        print(f"{YELLOW}⚠️  Proofreading complete (verdict: {verdict}){RESET}")
-        print(f"   Review output: {log_file}")
-        return 0
 
 
 def review() -> int:
@@ -3041,8 +2835,74 @@ def split(task_file: str, strategy: str = "sections", max_lines: int = 500, dry_
         print(f"{RED}Error during file splitting: {e}{RESET}")
         return 1
 
+
+def list_evaluators() -> int:
+    """List all available evaluators (built-in and local)."""
+    from adversarial_workflow.evaluators import (
+        BUILTIN_EVALUATORS,
+        discover_local_evaluators,
+    )
+
+    # Print built-in evaluators
+    print(f"{BOLD}Built-in Evaluators:{RESET}")
+    for name, config in sorted(BUILTIN_EVALUATORS.items()):
+        print(f"  {name:14} {config.description}")
+
+    print()
+
+    # Print local evaluators
+    local_evaluators = discover_local_evaluators()
+    if local_evaluators:
+        print(f"{BOLD}Local Evaluators{RESET} (.adversarial/evaluators/):")
+
+        # Group by primary name (skip aliases)
+        seen_configs = set()
+        for _, config in sorted(local_evaluators.items()):
+            if id(config) in seen_configs:
+                continue
+            seen_configs.add(id(config))
+
+            print(f"  {config.name:14} {config.description}")
+            if config.aliases:
+                print(f"    aliases: {', '.join(config.aliases)}")
+            print(f"    model: {config.model}")
+            if config.version != "1.0.0":
+                print(f"    version: {config.version}")
+    else:
+        print(f"{GRAY}No local evaluators found.{RESET}")
+        print()
+        print("Create .adversarial/evaluators/*.yml to add custom evaluators.")
+        print("See: https://github.com/movito/adversarial-workflow#custom-evaluators")
+
+    return 0
+
 def main():
     """Main CLI entry point."""
+    import logging
+    import sys
+
+    # Load .env file before any commands run
+    # Wrapped in try/except so CLI remains usable even with malformed .env
+    try:
+        load_dotenv()
+    except Exception as e:
+        print(f"Warning: Could not load .env file: {e}", file=sys.stderr)
+
+    from adversarial_workflow.evaluators import (
+        get_all_evaluators,
+        run_evaluator,
+        BUILTIN_EVALUATORS,
+    )
+
+    logger = logging.getLogger(__name__)
+
+    # Commands that cannot be overridden by evaluators
+    # Note: 'review' is special - it reviews git changes without a file argument
+    STATIC_COMMANDS = {
+        "init", "check", "doctor", "health", "quickstart",
+        "agent", "split", "validate", "review", "list-evaluators"
+    }
+
     parser = argparse.ArgumentParser(
         description="Adversarial Workflow - Multi-stage AI code review",
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -3112,17 +2972,7 @@ For more information: https://github.com/movito/adversarial-workflow
         "--path", default=".", help="Project path (default: current directory)"
     )
 
-    # evaluate command
-    eval_parser = subparsers.add_parser("evaluate", help="Run Phase 1: Plan evaluation")
-    eval_parser.add_argument("task_file", help="Task file to evaluate")
-
-    # proofread command
-    proofread_parser = subparsers.add_parser(
-        "proofread", help="Proofread teaching content and documentation"
-    )
-    proofread_parser.add_argument("doc_file", help="Document file to proofread")
-
-    # review command
+    # review command (static - reviews git changes, no file argument)
     subparsers.add_parser("review", help="Run Phase 3: Code review")
 
     # validate command
@@ -3151,13 +3001,74 @@ For more information: https://github.com/movito/adversarial-workflow
         help="Preview splits without creating files"
     )
 
+    # list-evaluators command
+    subparsers.add_parser(
+        "list-evaluators",
+        help="List all available evaluators (built-in and local)",
+    )
+
+    # Dynamic evaluator registration
+    try:
+        evaluators = get_all_evaluators()
+    except Exception as e:
+        logger.warning("Evaluator discovery failed: %s", e)
+        evaluators = BUILTIN_EVALUATORS
+
+    registered_configs = set()  # Track by id() to avoid duplicate alias registration
+
+    for name, config in evaluators.items():
+        # Skip if name conflicts with static command
+        if name in STATIC_COMMANDS:
+            logger.warning("Evaluator '%s' conflicts with CLI command; skipping", name)
+            # Mark as registered to prevent alias re-registration attempts
+            registered_configs.add(id(config))
+            continue
+
+        # Skip if this config was already registered (aliases share config object)
+        if id(config) in registered_configs:
+            continue
+        registered_configs.add(id(config))
+
+        # Filter aliases that conflict with static commands
+        aliases = [a for a in (config.aliases or []) if a not in STATIC_COMMANDS]
+        if config.aliases and len(aliases) != len(config.aliases):
+            skipped = [a for a in config.aliases if a in STATIC_COMMANDS]
+            logger.warning(
+                "Skipping evaluator aliases that conflict with static commands: %s",
+                skipped,
+            )
+
+        # Create subparser for this evaluator
+        eval_parser = subparsers.add_parser(
+            config.name,
+            help=config.description,
+            aliases=aliases,
+        )
+        eval_parser.add_argument("file", help="File to evaluate")
+        eval_parser.add_argument(
+            "--timeout", "-t",
+            type=int,
+            default=180,
+            help="Timeout in seconds (default: 180)"
+        )
+        # Store config for later execution
+        eval_parser.set_defaults(evaluator_config=config)
+
     args = parser.parse_args()
 
     if not args.command:
         parser.print_help()
         return 0
 
-    # Execute command
+    # Check for evaluator command first (has evaluator_config attribute)
+    if hasattr(args, "evaluator_config"):
+        return run_evaluator(
+            args.evaluator_config,
+            args.file,
+            timeout=args.timeout,
+        )
+
+    # Execute static commands
     if args.command == "init":
         if args.interactive:
             return init_interactive(args.path)
@@ -3177,21 +3088,19 @@ For more information: https://github.com/movito/adversarial-workflow
             print(f"{RED}Error: agent command requires a subcommand{RESET}")
             print("Usage: adversarial agent onboard")
             return 1
-    elif args.command == "evaluate":
-        return evaluate(args.task_file)
-    elif args.command == "proofread":
-        return proofread(args.doc_file)
     elif args.command == "review":
         return review()
     elif args.command == "validate":
         return validate(args.test_command)
     elif args.command == "split":
         return split(
-            args.task_file, 
-            strategy=args.strategy, 
-            max_lines=args.max_lines, 
+            args.task_file,
+            strategy=args.strategy,
+            max_lines=args.max_lines,
             dry_run=args.dry_run
         )
+    elif args.command == "list-evaluators":
+        return list_evaluators()
     else:
         parser.print_help()
         return 1

adversarial_workflow/evaluators/__init__.py

@@ -0,0 +1,45 @@
+"""Evaluators module for adversarial-workflow plugin architecture."""
+
+from .config import EvaluatorConfig
+from .discovery import (
+    discover_local_evaluators,
+    parse_evaluator_yaml,
+    EvaluatorParseError,
+)
+from .runner import run_evaluator
+from .builtins import BUILTIN_EVALUATORS
+
+
+def get_all_evaluators() -> dict[str, EvaluatorConfig]:
+    """Get all available evaluators (built-in + local).
+
+    Local evaluators override built-in evaluators with the same name.
+    Aliases from local evaluators are also included in the returned dictionary.
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+
+    evaluators: dict[str, EvaluatorConfig] = {}
+
+    # Add built-in evaluators first
+    evaluators.update(BUILTIN_EVALUATORS)
+
+    # Discover and add local evaluators (may override built-ins)
+    local = discover_local_evaluators()
+    for name, config in local.items():
+        if name in BUILTIN_EVALUATORS:
+            logger.info("Local evaluator '%s' overrides built-in", name)
+        evaluators[name] = config
+
+    return evaluators
+
+
+__all__ = [
+    "EvaluatorConfig",
+    "EvaluatorParseError",
+    "run_evaluator",
+    "get_all_evaluators",
+    "discover_local_evaluators",
+    "parse_evaluator_yaml",
+    "BUILTIN_EVALUATORS",
+]

adversarial_workflow/evaluators/builtins.py

@@ -0,0 +1,36 @@
+"""Built-in evaluator configurations."""
+
+from __future__ import annotations
+
+from .config import EvaluatorConfig
+
+# Built-in evaluators use shell scripts - prompts are in the scripts
+BUILTIN_EVALUATORS: dict[str, EvaluatorConfig] = {
+    "evaluate": EvaluatorConfig(
+        name="evaluate",
+        description="Plan evaluation (GPT-4o)",
+        model="gpt-4o",
+        api_key_env="OPENAI_API_KEY",
+        prompt="",  # Prompt is in shell script
+        output_suffix="PLAN-EVALUATION",
+        source="builtin",
+    ),
+    "proofread": EvaluatorConfig(
+        name="proofread",
+        description="Teaching content review (GPT-4o)",
+        model="gpt-4o",
+        api_key_env="OPENAI_API_KEY",
+        prompt="",  # Prompt is in shell script
+        output_suffix="PROOFREADING",
+        source="builtin",
+    ),
+    "review": EvaluatorConfig(
+        name="review",
+        description="Code review (GPT-4o)",
+        model="gpt-4o",
+        api_key_env="OPENAI_API_KEY",
+        prompt="",  # Prompt is in shell script
+        output_suffix="CODE-REVIEW",
+        source="builtin",
+    ),
+}

adversarial_workflow/evaluators/config.py

@@ -0,0 +1,49 @@
+"""
+EvaluatorConfig dataclass for evaluator definitions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class EvaluatorConfig:
+    """Configuration for an evaluator (built-in or custom).
+
+    This dataclass represents the configuration for any evaluator,
+    whether built-in (evaluate, proofread, review) or custom
+    (defined in .adversarial/evaluators/*.yml).
+
+    Attributes:
+        name: Command name (e.g., "evaluate", "athena")
+        description: Help text shown in CLI
+        model: Model to use (e.g., "gpt-4o", "gemini-2.5-pro")
+        api_key_env: Environment variable name for API key
+        prompt: The evaluation prompt template
+        output_suffix: Log file suffix (e.g., "PLAN-EVALUATION")
+        log_prefix: CLI output prefix (e.g., "ATHENA")
+        fallback_model: Fallback model if primary fails
+        aliases: Alternative command names
+        version: Evaluator version
+        source: "builtin" or "local" (set internally)
+        config_file: Path to YAML file if local (set internally)
+    """
+
+    # Required fields
+    name: str
+    description: str
+    model: str
+    api_key_env: str
+    prompt: str
+    output_suffix: str
+
+    # Optional fields with defaults
+    log_prefix: str = ""
+    fallback_model: str | None = None
+    aliases: list[str] = field(default_factory=list)
+    version: str = "1.0.0"
+
+    # Metadata (set internally during discovery, not from YAML)
+    source: str = "builtin"
+    config_file: str | None = None