adversarial-workflow 0.5.0__tar.gz → 0.6.0__tar.gz

{adversarial_workflow-0.5.0 → adversarial_workflow-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adversarial-workflow
-Version: 0.5.0
+Version: 0.6.0
 Summary: Multi-stage AI code review system preventing phantom work - Author/Evaluator pattern
 Author: Fredrik Matheson
 License: MIT
@@ -382,8 +382,68 @@ adversarial split task.md               # Split large files into smaller parts
 adversarial split task.md --dry-run     # Preview split without creating files
 adversarial review                      # Phase 3: Review implementation
 adversarial validate "pytest"           # Phase 4: Validate with tests
+adversarial list-evaluators             # List all available evaluators
 ```
 
+## Custom Evaluators
+
+Starting with v0.6.0, you can define project-specific evaluators without modifying the package.
+
+### Creating a Custom Evaluator
+
+1. Create the evaluators directory:
+   ```bash
+   mkdir -p .adversarial/evaluators
+   ```
+
+2. Create a YAML definition:
+   ```yaml
+   # .adversarial/evaluators/athena.yml
+   name: athena
+   description: Knowledge evaluation using Gemini 2.5 Pro
+   model: gemini-2.5-pro
+   api_key_env: GEMINI_API_KEY
+   output_suffix: KNOWLEDGE-EVALUATION
+   prompt: |
+     You are Athena, a knowledge evaluation specialist...
+
+   # Optional
+   aliases:
+     - knowledge
+   ```
+
+3. Use it like any built-in evaluator:
+   ```bash
+   adversarial athena docs/research-plan.md
+   ```
+
+### Evaluator YAML Schema
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `name` | Yes | Command name |
+| `description` | Yes | Help text shown in CLI |
+| `model` | Yes | Model to use (e.g., `gpt-4o`, `gemini-2.5-pro`) |
+| `api_key_env` | Yes | Environment variable for API key |
+| `output_suffix` | Yes | Log file suffix (e.g., `KNOWLEDGE-EVAL`) |
+| `prompt` | Yes | The evaluation prompt |
+| `aliases` | No | Alternative command names |
+| `log_prefix` | No | CLI output prefix |
+| `fallback_model` | No | Fallback model if primary fails |
+| `version` | No | Evaluator version (default: 1.0.0) |
+
+### Listing Available Evaluators
+
+```bash
+adversarial list-evaluators
+```
+
+### Example: Athena Knowledge Evaluator
+
+See [docs/examples/athena.yml](docs/examples/athena.yml) for a complete example of a knowledge-focused evaluator using Gemini 2.5 Pro.
+
+For full documentation on custom evaluators, see [docs/CUSTOM_EVALUATORS.md](docs/CUSTOM_EVALUATORS.md).
+
 ## Configuration
 
 ### Option 1: YAML Config (persistent)

{adversarial_workflow-0.5.0 → adversarial_workflow-0.6.0}/README.md

@@ -347,8 +347,68 @@ adversarial split task.md               # Split large files into smaller parts
 adversarial split task.md --dry-run     # Preview split without creating files
 adversarial review                      # Phase 3: Review implementation
 adversarial validate "pytest"           # Phase 4: Validate with tests
+adversarial list-evaluators             # List all available evaluators
 ```
 
+## Custom Evaluators
+
+Starting with v0.6.0, you can define project-specific evaluators without modifying the package.
+
+### Creating a Custom Evaluator
+
+1. Create the evaluators directory:
+   ```bash
+   mkdir -p .adversarial/evaluators
+   ```
+
+2. Create a YAML definition:
+   ```yaml
+   # .adversarial/evaluators/athena.yml
+   name: athena
+   description: Knowledge evaluation using Gemini 2.5 Pro
+   model: gemini-2.5-pro
+   api_key_env: GEMINI_API_KEY
+   output_suffix: KNOWLEDGE-EVALUATION
+   prompt: |
+     You are Athena, a knowledge evaluation specialist...
+
+   # Optional
+   aliases:
+     - knowledge
+   ```
+
+3. Use it like any built-in evaluator:
+   ```bash
+   adversarial athena docs/research-plan.md
+   ```
+
+### Evaluator YAML Schema
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `name` | Yes | Command name |
+| `description` | Yes | Help text shown in CLI |
+| `model` | Yes | Model to use (e.g., `gpt-4o`, `gemini-2.5-pro`) |
+| `api_key_env` | Yes | Environment variable for API key |
+| `output_suffix` | Yes | Log file suffix (e.g., `KNOWLEDGE-EVAL`) |
+| `prompt` | Yes | The evaluation prompt |
+| `aliases` | No | Alternative command names |
+| `log_prefix` | No | CLI output prefix |
+| `fallback_model` | No | Fallback model if primary fails |
+| `version` | No | Evaluator version (default: 1.0.0) |
+
+### Listing Available Evaluators
+
+```bash
+adversarial list-evaluators
+```
+
+### Example: Athena Knowledge Evaluator
+
+See [docs/examples/athena.yml](docs/examples/athena.yml) for a complete example of a knowledge-focused evaluator using Gemini 2.5 Pro.
+
+For full documentation on custom evaluators, see [docs/CUSTOM_EVALUATORS.md](docs/CUSTOM_EVALUATORS.md).
+
 ## Configuration
 
 ### Option 1: YAML Config (persistent)

{adversarial_workflow-0.5.0 → adversarial_workflow-0.6.0}/adversarial_workflow/__init__.py

@@ -12,7 +12,7 @@ Usage:
     adversarial validate "pytest"
 """
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"
 __author__ = "Fredrik Matheson"
 __license__ = "MIT"
 

{adversarial_workflow-0.5.0 → adversarial_workflow-0.6.0}/adversarial_workflow/cli.py

@@ -29,7 +29,7 @@ from typing import Dict, List, Optional, Tuple
 import yaml
 from dotenv import load_dotenv
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"
 
 # ANSI color codes for better output
 RESET = "\033[0m"
@@ -2086,225 +2086,8 @@ def evaluate(task_file: str) -> int:
         return 0
 
 
-def proofread(doc_file: str) -> int:
-    """Run proofreading review on teaching/documentation content."""
 
-    print(f"📖 Proofreading document: {doc_file}")
-    print()
-
-    # Error 1: Document file not found
-    if not os.path.exists(doc_file):
-        print(f"{RED}❌ ERROR: Document file not found: {doc_file}{RESET}")
-        print("   Usage: adversarial proofread <document_file>")
-        print("   Example: adversarial proofread docs/guide/concept.md")
-        return 1
-
-    # Error 2: Config not loaded
-    try:
-        config = load_config()
-    except FileNotFoundError:
-        print(f"{RED}❌ ERROR: Not initialized. Run 'adversarial init' first.{RESET}")
-        return 1
-
-    # Error 3: Aider not available
-    if not shutil.which("aider"):
-        print(f"{RED}❌ ERROR: Aider not found{RESET}")
-        print()
-        print(f"{BOLD}WHY:{RESET}")
-        print("   Proofreader uses aider (AI pair programming tool) to:")
-        print("   • Review teaching content quality")
-        print("   • Check clarity and accuracy")
-        print("   • Evaluate pedagogical effectiveness")
-        print()
-        print(f"{BOLD}FIX:{RESET}")
-        print("   1. Install aider: pip install aider-chat")
-        print("   2. Verify installation: aider --version")
-        print("   3. Then retry: adversarial proofread ...")
-        print()
-        print(f"{BOLD}HELP:{RESET}")
-        print("   Aider docs: https://aider.chat/docs/install.html")
-        return 1
-
-    # Pre-flight check for file size
-    with open(doc_file, "r") as f:
-        line_count = len(f.readlines())
-        f.seek(0)
-        file_size = len(f.read())
 
-    # Estimate tokens (1 token ≈ 4 characters)
-    estimated_tokens = file_size // 4
-
-    # Warn if file is large (>500 lines or >20k tokens)
-    if line_count > 500 or estimated_tokens > 20000:
-        print(f"{YELLOW}⚠️  Large file detected:{RESET}")
-        print(f"   Lines: {line_count:,}")
-        print(f"   Estimated tokens: ~{estimated_tokens:,}")
-        print()
-        print(f"{BOLD}Note:{RESET} Files over 500 lines may exceed OpenAI rate limits.")
-        print(
-            f"      If proofreading fails, consider splitting into smaller documents."
-        )
-        print()
-
-        # Give user a chance to cancel for very large files
-        if line_count > 700:
-            print(f"{RED}⚠️  WARNING: File is very large (>{line_count} lines){RESET}")
-            print(f"   This will likely fail on Tier 1 OpenAI accounts (30k TPM limit)")
-            print(f"   Recommended: Split into files <500 lines each")
-            print()
-            response = input("Continue anyway? [y/N]: ").strip().lower()
-            if response not in ["y", "yes"]:
-                print("Proofreading cancelled.")
-                return 0
-            print()
-
-    # Error 4: Script execution fails
-    script = ".adversarial/scripts/proofread_content.sh"
-    if not os.path.exists(script):
-        print(f"{RED}❌ ERROR: Script not found: {script}{RESET}")
-        print("   Fix: Run 'adversarial init' to reinstall scripts")
-        return 1
-
-    try:
-        result = subprocess.run(
-            [script, doc_file], text=True, capture_output=True, timeout=180  # 3 minutes
-        )
-
-        # Check for rate limit errors in output
-        output = result.stdout + result.stderr
-        if "RateLimitError" in output or "tokens per min (TPM)" in output:
-            print(f"{RED}❌ ERROR: OpenAI rate limit exceeded{RESET}")
-            print()
-            print(f"{BOLD}WHY:{RESET}")
-            print(
-                "   Your document file is too large for your OpenAI organization's rate limit"
-            )
-            print()
-
-            # Extract file size for helpful message
-            with open(doc_file, "r") as f:
-                line_count = len(f.readlines())
-
-            print(f"{BOLD}FILE SIZE:{RESET}")
-            print(f"   Lines: {line_count:,}")
-            print(f"   Recommended limit: 500 lines")
-            print()
-            print(f"{BOLD}SOLUTIONS:{RESET}")
-            print("   1. Split your document into smaller files (<500 lines each)")
-            print("   2. Upgrade your OpenAI tier (Tier 2 supports ~1,000 lines)")
-            print("   3. Use manual review for this comprehensive document")
-            print()
-            print(f"{BOLD}MORE INFO:{RESET}")
-            print("   https://platform.openai.com/docs/guides/rate-limits")
-            return 1
-
-    except subprocess.TimeoutExpired:
-        print(f"{RED}❌ ERROR: Proofreading timed out (>3 minutes){RESET}")
-        print()
-        print(f"{BOLD}WHY:{RESET}")
-        print("   The AI model took too long to respond")
-        print()
-        print(f"{BOLD}POSSIBLE CAUSES:{RESET}")
-        print("   • Network issues connecting to API")
-        print("   • Document file too large (>1000 lines)")
-        print("   • API rate limiting")
-        print()
-        print(f"{BOLD}FIX:{RESET}")
-        print("   1. Check your network connection")
-        print("   2. Try a smaller document file")
-        print("   3. Wait a few minutes and retry")
-        return 1
-    except FileNotFoundError as e:
-        # Check if this is a bash/platform issue
-        if platform.system() == "Windows":
-            print(f"{RED}❌ ERROR: Cannot execute workflow scripts{RESET}")
-            print()
-            print(f"{BOLD}WHY:{RESET}")
-            print("   Native Windows (PowerShell/CMD) cannot run bash scripts")
-            print("   This package requires Unix shell (bash) for workflow automation")
-            print()
-            print(f"{BOLD}FIX:{RESET}")
-            print("   Option 1 (RECOMMENDED): Use WSL (Windows Subsystem for Linux)")
-            print(
-                "     1. Install WSL: https://learn.microsoft.com/windows/wsl/install"
-            )
-            print("     2. Open WSL terminal")
-            print("     3. Reinstall package in WSL: pip install adversarial-workflow")
-            print()
-            print("   Option 2: Try Git Bash (not officially supported)")
-            print("     • May have compatibility issues")
-            print("     • WSL is strongly recommended")
-            print()
-            print(f"{BOLD}HELP:{RESET}")
-            print("   See platform requirements: README.md#platform-support")
-        else:
-            print(f"{RED}❌ ERROR: Script not found: {script}{RESET}")
-            print()
-            print(f"{BOLD}WHY:{RESET}")
-            print("   Workflow scripts are missing or corrupted")
-            print()
-            print(f"{BOLD}FIX:{RESET}")
-            print("   Run: adversarial init")
-            print("   This will reinstall all workflow scripts")
-        return 1
-
-    # Error 5: Proofreading rejected
-    if result.returncode != 0:
-        print()
-        print("📋 Proofreading complete (needs revision)")
-        print(f"   Details: {config['log_directory']}")
-        return result.returncode
-
-    # Error 6: Validation - Check if proofreading actually ran (not just empty output)
-    # Extract document name from filename to find log file
-    doc_basename = os.path.basename(doc_file)
-    doc_name = os.path.splitext(doc_basename)[0]
-
-    log_file = os.path.join(config["log_directory"], f"{doc_name}-PROOFREADING.md")
-
-    is_valid, verdict, message = validate_evaluation_output(log_file)
-    if not is_valid:
-        print()
-        print(f"{RED}❌ Proofreading failed: {message}{RESET}")
-        print()
-        print(f"{BOLD}WHY:{RESET}")
-        print("   The proofreading script ran but didn't produce valid output")
-        print("   This usually means Aider encountered an error before running GPT-4o")
-        print()
-        print(f"{BOLD}LOG FILE:{RESET}")
-        print(f"   {log_file}")
-        print()
-        print(f"{BOLD}FIX:{RESET}")
-        print("   1. Check the log file for error messages")
-        print("   2. Ensure your API keys are valid: adversarial check")
-        print("   3. Try running the proofreading again")
-        print()
-        return 1
-
-    # Verify token count (warn if suspiciously low)
-    verify_token_count(doc_file, log_file)
-
-    # Report based on actual verdict from proofreading
-    print()
-    if verdict == "APPROVED":
-        print(f"{GREEN}✅ Proofreading APPROVED!{RESET}")
-        print(f"   Document is ready for publication")
-        print(f"   Review output: {log_file}")
-        return 0
-    elif verdict == "NEEDS_REVISION":
-        print(f"{YELLOW}⚠️  Proofreading NEEDS_REVISION{RESET}")
-        print(f"   Review feedback and update document")
-        print(f"   Details: {log_file}")
-        return 1
-    elif verdict == "REJECTED":
-        print(f"{RED}❌ Proofreading REJECTED{RESET}")
-        print(f"   Document has fundamental issues - major revision needed")
-        print(f"   Details: {log_file}")
-        return 1
-    else:  # UNKNOWN or other
-        print(f"{YELLOW}⚠️  Proofreading complete (verdict: {verdict}){RESET}")
-        print(f"   Review output: {log_file}")
-        return 0
 
 
 def review() -> int:
@@ -3041,8 +2824,66 @@ def split(task_file: str, strategy: str = "sections", max_lines: int = 500, dry_
         print(f"{RED}Error during file splitting: {e}{RESET}")
         return 1
 
+
+def list_evaluators() -> int:
+    """List all available evaluators (built-in and local)."""
+    from adversarial_workflow.evaluators import (
+        BUILTIN_EVALUATORS,
+        discover_local_evaluators,
+    )
+
+    # Print built-in evaluators
+    print(f"{BOLD}Built-in Evaluators:{RESET}")
+    for name, config in sorted(BUILTIN_EVALUATORS.items()):
+        print(f"  {name:14} {config.description}")
+
+    print()
+
+    # Print local evaluators
+    local_evaluators = discover_local_evaluators()
+    if local_evaluators:
+        print(f"{BOLD}Local Evaluators{RESET} (.adversarial/evaluators/):")
+
+        # Group by primary name (skip aliases)
+        seen_configs = set()
+        for _, config in sorted(local_evaluators.items()):
+            if id(config) in seen_configs:
+                continue
+            seen_configs.add(id(config))
+
+            print(f"  {config.name:14} {config.description}")
+            if config.aliases:
+                print(f"    aliases: {', '.join(config.aliases)}")
+            print(f"    model: {config.model}")
+            if config.version != "1.0.0":
+                print(f"    version: {config.version}")
+    else:
+        print(f"{GRAY}No local evaluators found.{RESET}")
+        print()
+        print("Create .adversarial/evaluators/*.yml to add custom evaluators.")
+        print("See: https://github.com/movito/adversarial-workflow#custom-evaluators")
+
+    return 0
+
 def main():
     """Main CLI entry point."""
+    import logging
+
+    from adversarial_workflow.evaluators import (
+        get_all_evaluators,
+        run_evaluator,
+        BUILTIN_EVALUATORS,
+    )
+
+    logger = logging.getLogger(__name__)
+
+    # Commands that cannot be overridden by evaluators
+    # Note: 'review' is special - it reviews git changes without a file argument
+    STATIC_COMMANDS = {
+        "init", "check", "doctor", "health", "quickstart",
+        "agent", "split", "validate", "review", "list-evaluators"
+    }
+
     parser = argparse.ArgumentParser(
         description="Adversarial Workflow - Multi-stage AI code review",
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -3112,17 +2953,7 @@ For more information: https://github.com/movito/adversarial-workflow
         "--path", default=".", help="Project path (default: current directory)"
     )
 
-    # evaluate command
-    eval_parser = subparsers.add_parser("evaluate", help="Run Phase 1: Plan evaluation")
-    eval_parser.add_argument("task_file", help="Task file to evaluate")
-
-    # proofread command
-    proofread_parser = subparsers.add_parser(
-        "proofread", help="Proofread teaching content and documentation"
-    )
-    proofread_parser.add_argument("doc_file", help="Document file to proofread")
-
-    # review command
+    # review command (static - reviews git changes, no file argument)
     subparsers.add_parser("review", help="Run Phase 3: Code review")
 
     # validate command
@@ -3151,13 +2982,74 @@ For more information: https://github.com/movito/adversarial-workflow
         help="Preview splits without creating files"
     )
 
+    # list-evaluators command
+    subparsers.add_parser(
+        "list-evaluators",
+        help="List all available evaluators (built-in and local)",
+    )
+
+    # Dynamic evaluator registration
+    try:
+        evaluators = get_all_evaluators()
+    except Exception as e:
+        logger.warning("Evaluator discovery failed: %s", e)
+        evaluators = BUILTIN_EVALUATORS
+
+    registered_configs = set()  # Track by id() to avoid duplicate alias registration
+
+    for name, config in evaluators.items():
+        # Skip if name conflicts with static command
+        if name in STATIC_COMMANDS:
+            logger.warning("Evaluator '%s' conflicts with CLI command; skipping", name)
+            # Mark as registered to prevent alias re-registration attempts
+            registered_configs.add(id(config))
+            continue
+
+        # Skip if this config was already registered (aliases share config object)
+        if id(config) in registered_configs:
+            continue
+        registered_configs.add(id(config))
+
+        # Filter aliases that conflict with static commands
+        aliases = [a for a in (config.aliases or []) if a not in STATIC_COMMANDS]
+        if config.aliases and len(aliases) != len(config.aliases):
+            skipped = [a for a in config.aliases if a in STATIC_COMMANDS]
+            logger.warning(
+                "Skipping evaluator aliases that conflict with static commands: %s",
+                skipped,
+            )
+
+        # Create subparser for this evaluator
+        eval_parser = subparsers.add_parser(
+            config.name,
+            help=config.description,
+            aliases=aliases,
+        )
+        eval_parser.add_argument("file", help="File to evaluate")
+        eval_parser.add_argument(
+            "--timeout", "-t",
+            type=int,
+            default=180,
+            help="Timeout in seconds (default: 180)"
+        )
+        # Store config for later execution
+        eval_parser.set_defaults(evaluator_config=config)
+
     args = parser.parse_args()
 
     if not args.command:
         parser.print_help()
         return 0
 
-    # Execute command
+    # Check for evaluator command first (has evaluator_config attribute)
+    if hasattr(args, "evaluator_config"):
+        return run_evaluator(
+            args.evaluator_config,
+            args.file,
+            timeout=args.timeout,
+        )
+
+    # Execute static commands
     if args.command == "init":
         if args.interactive:
             return init_interactive(args.path)
@@ -3177,21 +3069,19 @@ For more information: https://github.com/movito/adversarial-workflow
             print(f"{RED}Error: agent command requires a subcommand{RESET}")
             print("Usage: adversarial agent onboard")
             return 1
-    elif args.command == "evaluate":
-        return evaluate(args.task_file)
-    elif args.command == "proofread":
-        return proofread(args.doc_file)
     elif args.command == "review":
         return review()
     elif args.command == "validate":
         return validate(args.test_command)
     elif args.command == "split":
         return split(
-            args.task_file, 
-            strategy=args.strategy, 
-            max_lines=args.max_lines, 
+            args.task_file,
+            strategy=args.strategy,
+            max_lines=args.max_lines,
             dry_run=args.dry_run
         )
+    elif args.command == "list-evaluators":
+        return list_evaluators()
     else:
         parser.print_help()
         return 1

adversarial_workflow-0.6.0/adversarial_workflow/evaluators/__init__.py

@@ -0,0 +1,45 @@
+"""Evaluators module for adversarial-workflow plugin architecture."""
+
+from .config import EvaluatorConfig
+from .discovery import (
+    discover_local_evaluators,
+    parse_evaluator_yaml,
+    EvaluatorParseError,
+)
+from .runner import run_evaluator
+from .builtins import BUILTIN_EVALUATORS
+
+
+def get_all_evaluators() -> dict[str, EvaluatorConfig]:
+    """Get all available evaluators (built-in + local).
+
+    Local evaluators override built-in evaluators with the same name.
+    Aliases from local evaluators are also included in the returned dictionary.
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+
+    evaluators: dict[str, EvaluatorConfig] = {}
+
+    # Add built-in evaluators first
+    evaluators.update(BUILTIN_EVALUATORS)
+
+    # Discover and add local evaluators (may override built-ins)
+    local = discover_local_evaluators()
+    for name, config in local.items():
+        if name in BUILTIN_EVALUATORS:
+            logger.info("Local evaluator '%s' overrides built-in", name)
+        evaluators[name] = config
+
+    return evaluators
+
+
+__all__ = [
+    "EvaluatorConfig",
+    "EvaluatorParseError",
+    "run_evaluator",
+    "get_all_evaluators",
+    "discover_local_evaluators",
+    "parse_evaluator_yaml",
+    "BUILTIN_EVALUATORS",
+]

adversarial_workflow-0.6.0/adversarial_workflow/evaluators/builtins.py

@@ -0,0 +1,36 @@
+"""Built-in evaluator configurations."""
+
+from __future__ import annotations
+
+from .config import EvaluatorConfig
+
+# Built-in evaluators use shell scripts - prompts are in the scripts
+BUILTIN_EVALUATORS: dict[str, EvaluatorConfig] = {
+    "evaluate": EvaluatorConfig(
+        name="evaluate",
+        description="Plan evaluation (GPT-4o)",
+        model="gpt-4o",
+        api_key_env="OPENAI_API_KEY",
+        prompt="",  # Prompt is in shell script
+        output_suffix="PLAN-EVALUATION",
+        source="builtin",
+    ),
+    "proofread": EvaluatorConfig(
+        name="proofread",
+        description="Teaching content review (GPT-4o)",
+        model="gpt-4o",
+        api_key_env="OPENAI_API_KEY",
+        prompt="",  # Prompt is in shell script
+        output_suffix="PROOFREADING",
+        source="builtin",
+    ),
+    "review": EvaluatorConfig(
+        name="review",
+        description="Code review (GPT-4o)",
+        model="gpt-4o",
+        api_key_env="OPENAI_API_KEY",
+        prompt="",  # Prompt is in shell script
+        output_suffix="CODE-REVIEW",
+        source="builtin",
+    ),
+}