adversarial-workflow 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

adversarial_workflow/utils/file_splitter.py

@@ -0,0 +1,378 @@
+"""File splitting utility for large task specifications.
+
+This module provides functionality to split large markdown files into smaller,
+independently evaluable chunks to work around OpenAI's rate limits.
+"""
+
+import re
+import os
+from pathlib import Path
+from typing import List, Dict, Any
+
+
+def analyze_task_file(file_path: str) -> Dict[str, Any]:
+    """Analyze file structure and suggest split points.
+    
+    Args:
+        file_path: Path to the markdown file to analyze
+        
+    Returns:
+        Dict containing:
+        - total_lines: Total number of lines
+        - sections: List of detected sections with metadata
+        - estimated_tokens: Rough token estimate (lines * 4)
+        - suggested_splits: List of suggested split points
+        
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        ValueError: If file is empty or too small
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+    
+    with open(file_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    if not content.strip():
+        raise ValueError("File is empty or too small")
+    
+    lines = content.split('\n')
+    total_lines = len(lines)
+    
+    # Detect markdown sections
+    sections = []
+    current_section = None
+    current_start = 1
+    
+    for i, line in enumerate(lines, 1):
+        # Check for markdown headings (# or ##)
+        if re.match(r'^#+\s+', line.strip()):
+            # Close previous section
+            if current_section is not None:
+                current_section['end_line'] = i - 1
+                current_section['line_count'] = current_section['end_line'] - current_section['start_line'] + 1
+                sections.append(current_section)
+            
+            # Start new section
+            heading_level = len(line.lstrip().split()[0])  # Count # characters
+            title = re.sub(r'^#+\s+', '', line.strip())
+            current_section = {
+                'title': title,
+                'heading_level': heading_level,
+                'start_line': i,
+                'end_line': None,
+                'line_count': 0
+            }
+            current_start = i
+    
+    # Close final section
+    if current_section is not None:
+        current_section['end_line'] = total_lines
+        current_section['line_count'] = current_section['end_line'] - current_section['start_line'] + 1
+        sections.append(current_section)
+    
+    # If no sections found, treat entire file as one section
+    if not sections:
+        sections = [{
+            'title': 'Full Document',
+            'heading_level': 1,
+            'start_line': 1,
+            'end_line': total_lines,
+            'line_count': total_lines
+        }]
+    
+    # Estimate tokens (rough approximation: 1 line ≈ 4 tokens)
+    estimated_tokens = total_lines * 4
+    
+    # Suggest splits if file is large
+    suggested_splits = []
+    if total_lines > 500:
+        # Suggest section-based splits
+        suggested_splits = _suggest_section_splits(sections, max_lines=500)
+    
+    return {
+        'total_lines': total_lines,
+        'sections': sections,
+        'estimated_tokens': estimated_tokens,
+        'suggested_splits': suggested_splits
+    }
+
+
+def split_by_sections(content: str, max_lines: int = 500) -> List[Dict[str, Any]]:
+    """Split file by markdown sections.
+    
+    Args:
+        content: The markdown content to split
+        max_lines: Maximum lines per split
+        
+    Returns:
+        List of split dictionaries with metadata
+    """
+    lines = content.split('\n')
+    total_lines = len(lines)
+    
+    if total_lines <= max_lines:
+        return [{
+            'content': content,
+            'title': 'Full Document',
+            'start_line': 1,
+            'end_line': total_lines,
+            'line_count': total_lines
+        }]
+    
+    splits = []
+    current_split_lines = []
+    current_start = 1
+    current_title = "Part"
+    split_count = 1
+    
+    for i, line in enumerate(lines, 1):
+        current_split_lines.append(line)
+        
+        # Check if we hit a section boundary and are near limit
+        is_section_boundary = re.match(r'^#+\s+', line.strip())
+        approaching_limit = len(current_split_lines) >= max_lines * 0.8
+        
+        if len(current_split_lines) >= max_lines or (is_section_boundary and approaching_limit):
+            # Create split
+            split_content = '\n'.join(current_split_lines)
+            splits.append({
+                'content': split_content,
+                'title': f"Part {split_count}",
+                'start_line': current_start,
+                'end_line': i,
+                'line_count': len(current_split_lines)
+            })
+            
+            # Reset for next split
+            current_split_lines = []
+            current_start = i + 1
+            split_count += 1
+    
+    # Handle remaining lines
+    if current_split_lines:
+        split_content = '\n'.join(current_split_lines)
+        splits.append({
+            'content': split_content,
+            'title': f"Part {split_count}",
+            'start_line': current_start,
+            'end_line': total_lines,
+            'line_count': len(current_split_lines)
+        })
+    
+    return splits
+
+
+def split_by_phases(content: str) -> List[Dict[str, Any]]:
+    """Split file by implementation phases.
+    
+    Args:
+        content: The markdown content to split
+        
+    Returns:
+        List of split dictionaries, one per phase
+    """
+    lines = content.split('\n')
+    splits = []
+    current_split_lines = []
+    current_phase = None
+    current_start = 1
+    
+    for i, line in enumerate(lines, 1):
+        # Check for phase markers
+        phase_match = re.search(r'#+\s+Phase\s+(\d+)', line, re.IGNORECASE)
+        
+        if phase_match:
+            # Close previous split
+            if current_split_lines:
+                split_content = '\n'.join(current_split_lines)
+                title = f"Phase {current_phase}" if current_phase else "Overview"
+                splits.append({
+                    'content': split_content,
+                    'title': title,
+                    'phase_number': current_phase,
+                    'start_line': current_start,
+                    'end_line': i - 1,
+                    'line_count': len(current_split_lines)
+                })
+            
+            # Start new split
+            current_phase = int(phase_match.group(1))
+            current_split_lines = [line]
+            current_start = i
+        else:
+            current_split_lines.append(line)
+    
+    # Handle final split
+    if current_split_lines:
+        split_content = '\n'.join(current_split_lines)
+        title = f"Phase {current_phase}" if current_phase else "Full Document"
+        phase_info = {'phase_number': current_phase} if current_phase else {}
+        splits.append({
+            'content': split_content,
+            'title': title,
+            'start_line': current_start,
+            'end_line': len(lines),
+            'line_count': len(current_split_lines),
+            **phase_info
+        })
+    
+    # If no phases found, return entire content
+    if not splits:
+        splits = [{
+            'content': content,
+            'title': 'Full Document',
+            'start_line': 1,
+            'end_line': len(lines),
+            'line_count': len(lines)
+        }]
+    
+    return splits
+
+
+def split_at_lines(content: str, line_numbers: List[int]) -> List[Dict[str, Any]]:
+    """Split at specified line numbers.
+    
+    Args:
+        content: The content to split
+        line_numbers: Line numbers where splits should occur
+        
+    Returns:
+        List of split dictionaries
+    """
+    lines = content.split('\n')
+    total_lines = len(lines)
+    
+    if not line_numbers:
+        return [{
+            'content': content,
+            'title': 'Full Document',
+            'start_line': 1,
+            'end_line': total_lines,
+            'line_count': total_lines
+        }]
+    
+    # Sort and deduplicate line numbers
+    split_points = sorted(set(line_numbers))
+    
+    splits = []
+    current_start = 1
+    
+    for split_line in split_points:
+        if split_line >= total_lines:
+            continue
+            
+        # Create split from current_start to split_line
+        split_lines = lines[current_start - 1:split_line]
+        split_content = '\n'.join(split_lines)
+        
+        splits.append({
+            'content': split_content,
+            'title': f"Lines {current_start}-{split_line}",
+            'start_line': current_start,
+            'end_line': split_line,
+            'line_count': len(split_lines)
+        })
+        
+        current_start = split_line + 1
+    
+    # Handle remaining lines after final split
+    if current_start <= total_lines:
+        remaining_lines = lines[current_start - 1:]
+        split_content = '\n'.join(remaining_lines)
+        
+        splits.append({
+            'content': split_content,
+            'title': f"Lines {current_start}-{total_lines}",
+            'start_line': current_start,
+            'end_line': total_lines,
+            'line_count': len(remaining_lines)
+        })
+    
+    return splits
+
+
+def generate_split_files(original: str, splits: List[Dict[str, Any]], output_dir: str) -> List[str]:
+    """Generate split files with metadata and cross-references.
+    
+    Args:
+        original: Original filename
+        splits: List of split dictionaries
+        output_dir: Directory to write split files
+        
+    Returns:
+        List of created file paths
+    """
+    os.makedirs(output_dir, exist_ok=True)
+    
+    created_files = []
+    original_name = Path(original).stem
+    original_ext = Path(original).suffix
+    
+    for i, split in enumerate(splits, 1):
+        # Generate filename
+        filename = f"{original_name}-part{i}{original_ext}"
+        file_path = os.path.join(output_dir, filename)
+        
+        # Create content with metadata header
+        metadata_header = f"""<!-- Split from {original} -->
+<!-- Part {i} of {len(splits)} -->
+<!-- Lines {split['start_line']}-{split['end_line']} ({split['line_count']} lines) -->
+
+"""
+        
+        full_content = metadata_header + split['content']
+        
+        # Write file
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(full_content)
+        
+        created_files.append(file_path)
+    
+    return created_files
+
+
+def _suggest_section_splits(sections: List[Dict[str, Any]], max_lines: int = 500) -> List[Dict[str, Any]]:
+    """Suggest optimal split points based on sections.
+    
+    Args:
+        sections: List of section metadata
+        max_lines: Maximum lines per split
+        
+    Returns:
+        List of suggested split configurations
+    """
+    suggestions = []
+    current_chunk_lines = 0
+    current_chunk_sections = []
+    
+    for section in sections:
+        section_lines = section['line_count']
+        
+        # If adding this section would exceed limit, finish current chunk
+        if current_chunk_lines + section_lines > max_lines and current_chunk_sections:
+            suggestions.append({
+                'sections': current_chunk_sections.copy(),
+                'total_lines': current_chunk_lines,
+                'start_line': current_chunk_sections[0]['start_line'],
+                'end_line': current_chunk_sections[-1]['end_line']
+            })
+            
+            # Start new chunk
+            current_chunk_sections = [section]
+            current_chunk_lines = section_lines
+        else:
+            # Add section to current chunk
+            current_chunk_sections.append(section)
+            current_chunk_lines += section_lines
+    
+    # Add final chunk
+    if current_chunk_sections:
+        suggestions.append({
+            'sections': current_chunk_sections,
+            'total_lines': current_chunk_lines,
+            'start_line': current_chunk_sections[0]['start_line'],
+            'end_line': current_chunk_sections[-1]['end_line']
+        })
+    
+    return suggestions

adversarial_workflow/utils/validation.py

@@ -0,0 +1,76 @@
+"""Output validation utilities."""
+
+from __future__ import annotations
+
+import os
+import re
+
+
+def validate_evaluation_output(
+    log_file_path: str,
+) -> tuple[bool, str | None, str]:
+    """
+    Validate that evaluation log contains actual evaluation content.
+
+    Args:
+        log_file_path: Path to the evaluation log file
+
+    Returns:
+        (is_valid, verdict, message):
+            - is_valid: True if valid evaluation, False if failed
+            - verdict: "APPROVED", "NEEDS_REVISION", "REJECTED", or None
+            - message: Descriptive message about validation result
+    """
+    if not os.path.exists(log_file_path):
+        return False, None, f"Log file not found: {log_file_path}"
+
+    with open(log_file_path) as f:
+        content = f.read()
+
+    # Check minimum content size
+    if len(content) < 500:
+        return (
+            False,
+            None,
+            f"Log file too small ({len(content)} bytes) - evaluation likely failed",
+        )
+
+    # Check for evaluation markers (case-insensitive)
+    content_lower = content.lower()
+    evaluation_markers = [
+        "verdict:",
+        "approved",
+        "needs_revision",
+        "rejected",
+        "evaluation summary",
+        "strengths",
+        "concerns",
+    ]
+
+    has_evaluation_content = any(marker in content_lower for marker in evaluation_markers)
+    if not has_evaluation_content:
+        return (
+            False,
+            None,
+            "Log file missing evaluation content - no verdict or analysis found",
+        )
+
+    # Extract verdict
+    verdict = None
+    verdict_patterns = [
+        r"Verdict:\s*(APPROVED|NEEDS_REVISION|REJECTED)",
+        r"\*\*Verdict\*\*:\s*(APPROVED|NEEDS_REVISION|REJECTED)",
+        r"^(APPROVED|NEEDS_REVISION|REJECTED)\s*$",
+    ]
+
+    for pattern in verdict_patterns:
+        match = re.search(pattern, content, re.MULTILINE | re.IGNORECASE)
+        if match:
+            verdict = match.group(1).upper()
+            break
+
+    if verdict:
+        return True, verdict, f"Valid evaluation with verdict: {verdict}"
+    else:
+        # Has content but no clear verdict
+        return True, None, "Evaluation complete (verdict not detected)"

{adversarial_workflow-0.5.0.dist-info → adversarial_workflow-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adversarial-workflow
-Version: 0.5.0
+Version: 0.6.0
 Summary: Multi-stage AI code review system preventing phantom work - Author/Evaluator pattern
 Author: Fredrik Matheson
 License: MIT
@@ -382,8 +382,68 @@ adversarial split task.md               # Split large files into smaller parts
 adversarial split task.md --dry-run     # Preview split without creating files
 adversarial review                      # Phase 3: Review implementation
 adversarial validate "pytest"           # Phase 4: Validate with tests
+adversarial list-evaluators             # List all available evaluators
 ```
 
+## Custom Evaluators
+
+Starting with v0.6.0, you can define project-specific evaluators without modifying the package.
+
+### Creating a Custom Evaluator
+
+1. Create the evaluators directory:
+   ```bash
+   mkdir -p .adversarial/evaluators
+   ```
+
+2. Create a YAML definition:
+   ```yaml
+   # .adversarial/evaluators/athena.yml
+   name: athena
+   description: Knowledge evaluation using Gemini 2.5 Pro
+   model: gemini-2.5-pro
+   api_key_env: GEMINI_API_KEY
+   output_suffix: KNOWLEDGE-EVALUATION
+   prompt: |
+     You are Athena, a knowledge evaluation specialist...
+
+   # Optional
+   aliases:
+     - knowledge
+   ```
+
+3. Use it like any built-in evaluator:
+   ```bash
+   adversarial athena docs/research-plan.md
+   ```
+
+### Evaluator YAML Schema
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `name` | Yes | Command name |
+| `description` | Yes | Help text shown in CLI |
+| `model` | Yes | Model to use (e.g., `gpt-4o`, `gemini-2.5-pro`) |
+| `api_key_env` | Yes | Environment variable for API key |
+| `output_suffix` | Yes | Log file suffix (e.g., `KNOWLEDGE-EVAL`) |
+| `prompt` | Yes | The evaluation prompt |
+| `aliases` | No | Alternative command names |
+| `log_prefix` | No | CLI output prefix |
+| `fallback_model` | No | Fallback model if primary fails |
+| `version` | No | Evaluator version (default: 1.0.0) |
+
+### Listing Available Evaluators
+
+```bash
+adversarial list-evaluators
+```
+
+### Example: Athena Knowledge Evaluator
+
+See [docs/examples/athena.yml](docs/examples/athena.yml) for a complete example of a knowledge-focused evaluator using Gemini 2.5 Pro.
+
+For full documentation on custom evaluators, see [docs/CUSTOM_EVALUATORS.md](docs/CUSTOM_EVALUATORS.md).
+
 ## Configuration
 
 ### Option 1: YAML Config (persistent)

{adversarial_workflow-0.5.0.dist-info → adversarial_workflow-0.6.0.dist-info}/RECORD

@@ -1,6 +1,11 @@
-adversarial_workflow/__init__.py,sha256=YIuv4LILLb06wwTUFqCxAR4T-nLSJHvnUwR0ipzXsGY,596
+adversarial_workflow/__init__.py,sha256=0e_PxWBE6XJGkei7qZJRwQhWx51NyDoCPHJTn5upXmY,596
 adversarial_workflow/__main__.py,sha256=Ibb0CngDCh4mpCe8Zxnf3kyKnMddBxQy2JAk_kfTUMQ,119
-adversarial_workflow/cli.py,sha256=9WqM71vAHrfhS73Minn2HFoKbUVBK9KDZyd3W7i-V0o,114528
+adversarial_workflow/cli.py,sha256=ssE7xXFgY_0SRvLFGbb38trmxqwXLT1m5jo3VZKToxA,109198
+adversarial_workflow/evaluators/__init__.py,sha256=vB4gGaoP46a-ZLOeoVKjR6WohAsgeif4JMhaak9AIPo,1266
+adversarial_workflow/evaluators/builtins.py,sha256=u5LokYLe8ruEW2tunhOQaNSkpcZ9Ee2IeTkaC0dZDSY,1102
+adversarial_workflow/evaluators/config.py,sha256=05qYPIiIpCxXBVJzs70WQQLxi8I7MedfhE_oydXEcq0,1520
+adversarial_workflow/evaluators/discovery.py,sha256=V5vyFLKfh3Q9MVEipWMdD0tzsW3xC3RttVS_oEeWIb8,6801
+adversarial_workflow/evaluators/runner.py,sha256=27fdz49wdkNyBfdjr9VcZZA53rw270XYQWp_2y5s0PU,9244
 adversarial_workflow/templates/.aider.conf.yml.template,sha256=jT2jWIgsnmS3HLhoQWMTO3GV07bUcsT2keYw60jqiDw,183
 adversarial_workflow/templates/.env.example.template,sha256=TmTlcgz44uZqIbqgXqdfHMl-0vVn96F_EGNohClFkb8,1821
 adversarial_workflow/templates/README.template,sha256=FQAMPO99eIt_kgQfwhGHcrK736rm_MEvWSbPnqBSjAE,1349
@@ -15,9 +20,14 @@ adversarial_workflow/templates/agent-context/README.md.template,sha256=gF31N-s_t
 adversarial_workflow/templates/agent-context/agent-handoffs-minimal.json.template,sha256=DCSi4NFl0z0OwQxIw6FnCIVPTXz3AvSvl76ZkIZKWC4,1466
 adversarial_workflow/templates/agent-context/agent-handoffs.json.template,sha256=rdRX79xdNKyTGd8_g6pvSdQK5VC5sDiErq_OZFslSXI,3904
 adversarial_workflow/templates/agent-context/current-state.json.template,sha256=UH3SQGjXGNzScqHnQcPrhrI9ZvjQBC3oBp4S9TilzoY,2325
-adversarial_workflow-0.5.0.dist-info/licenses/LICENSE,sha256=M-dOQlre-NmicyPa55hYOJUW8roGpCKEgtq-z0z1KCA,1073
-adversarial_workflow-0.5.0.dist-info/METADATA,sha256=qYAgfQEnxvcdPy7xBLeR72ZFt2SMTEfbeC1OT-icJi8,27029
-adversarial_workflow-0.5.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-adversarial_workflow-0.5.0.dist-info/entry_points.txt,sha256=9H-iZ-yF1uKZ8P0G1suc6kWR0NvK7uPZJbhN7nvt1sE,62
-adversarial_workflow-0.5.0.dist-info/top_level.txt,sha256=8irutNxLRjUbTlzfAibIpz7_ovkkF2h8ES69NQpv24c,21
-adversarial_workflow-0.5.0.dist-info/RECORD,,
+adversarial_workflow/utils/__init__.py,sha256=Pnm-a_jqoMVOxHdvVWXeVrL0IKI-zkY7EAdbQmZAkSI,352
+adversarial_workflow/utils/colors.py,sha256=uRrG6KfIDBLo0F5_vPwms9NCm9-x8YXBiyZ4naCr868,160
+adversarial_workflow/utils/config.py,sha256=NBoC_-YYukEVo6BgpX2cDyeqV-3tnn_sHNU9L1AuSLQ,1341
+adversarial_workflow/utils/file_splitter.py,sha256=rVRMHJgzJ7uNiytimqbBY8PAr-SevXdRqUpr4xf6LdM,12061
+adversarial_workflow/utils/validation.py,sha256=0QfuRd-kurcadUCd9XQvO-N8RsmLp6ONQnc0vaQTUBA,2188
+adversarial_workflow-0.6.0.dist-info/licenses/LICENSE,sha256=M-dOQlre-NmicyPa55hYOJUW8roGpCKEgtq-z0z1KCA,1073
+adversarial_workflow-0.6.0.dist-info/METADATA,sha256=2GWjMoaxNkF9PF16I4QN0BDTupwoXtT_6uZpxW_wvTI,28799
+adversarial_workflow-0.6.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+adversarial_workflow-0.6.0.dist-info/entry_points.txt,sha256=9H-iZ-yF1uKZ8P0G1suc6kWR0NvK7uPZJbhN7nvt1sE,62
+adversarial_workflow-0.6.0.dist-info/top_level.txt,sha256=8irutNxLRjUbTlzfAibIpz7_ovkkF2h8ES69NQpv24c,21
+adversarial_workflow-0.6.0.dist-info/RECORD,,

{adversarial_workflow-0.5.0.dist-info → adversarial_workflow-0.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.1)
 Root-Is-Purelib: true
 Tag: py3-none-any