swegen 0.1.0__py3-none-any.whl

swegen/tools/validate.py

@@ -0,0 +1,523 @@
+from __future__ import annotations
+
+import asyncio
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from harbor.models.environment_type import EnvironmentType
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+from rich.table import Table
+
+from .harbor_runner import parse_harbor_outcome, run_harbor_agent
+
+DOCKER_CLEANUP_CMD = "docker system prune -af"
+
+
+@dataclass
+class ValidateArgs:
+    path: Path
+    task: str | None
+    jobs_dir: Path
+    agent: str  # "both" | "nop" | "oracle"
+    timeout_multiplier: float | None = None
+    verbose: bool = False
+    quiet: bool = False
+    environment: EnvironmentType = EnvironmentType.DOCKER
+    max_parallel: int = 8
+    show_passed: bool = False
+    output_file: Path | None = None  # Write results to file as they complete
+    docker_prune_batch: int = 5  # Run docker cleanup after every N tasks (0 to disable)
+
+
+@dataclass
+class ValidationResult:
+    """Result of validating a single task."""
+
+    task_id: str
+    nop_reward: float | None
+    oracle_reward: float | None
+    nop_exit_code: int
+    oracle_exit_code: int
+    passed: bool
+    error: str | None = None
+
+
+def run_validate(args: ValidateArgs) -> None:
+    """Main entry point - routes to single or batch validation."""
+    dataset_path, task_id, task_dir = _resolve_paths(args)
+
+    if task_id is None:
+        _run_batch_mode(args, dataset_path)
+    else:
+        _run_single_mode(args, dataset_path, task_id, task_dir)
+
+
+def _resolve_paths(args: ValidateArgs) -> tuple[Path, str | None, Path | None]:
+    """Resolve paths and determine if single or batch mode.
+
+    Returns: (dataset_path, task_id, task_dir)
+             task_id/task_dir are None for batch mode
+    """
+    path = args.path.resolve()
+
+    if args.task:
+        # Explicit task ID: single mode
+        return path, args.task, path / args.task
+
+    if path.is_dir() and (path / "tests" / "test.sh").exists():
+        # Path is a task directory: single mode
+        return path.parent, path.name, path
+
+    if path.is_dir():
+        # Check if directory contains tasks: batch mode
+        tasks = [d for d in path.iterdir() if d.is_dir() and (d / "tests" / "test.sh").exists()]
+        if tasks:
+            return path, None, None
+        raise SystemExit(
+            f"No tasks found in directory: {path}\nExpected directories with tests/test.sh"
+        )
+
+    raise SystemExit(
+        "Path must be:\n"
+        "  1. A task directory (containing tests/test.sh), or\n"
+        "  2. A dataset directory with multiple tasks"
+    )
+
+
+# ============================================================================
+# SINGLE TASK MODE
+# ============================================================================
+
+
+def _run_single_mode(args: ValidateArgs, dataset_path: Path, task_id: str, task_dir: Path) -> None:
+    """Validate a single task with traditional output."""
+    jobs_dir = args.jobs_dir.resolve()
+    jobs_dir.mkdir(parents=True, exist_ok=True)
+
+    # Run regular validation
+    print("[validate] Running regular validation...")
+    nop_reward, oracle_reward = _run_agents(
+        task_id, dataset_path, jobs_dir, args.agent, args.timeout_multiplier, args.environment
+    )
+
+    # Check results
+    if args.agent == "both":
+        if nop_reward != 0 or oracle_reward != 1:
+            print("\n[validate] FAILED: Harbor validation did not meet expectations")
+            print(f"  NOP: expected reward=0, got reward={nop_reward}")
+            print(f"  ORACLE: expected reward=1, got reward={oracle_reward}")
+            sys.exit(1)
+        else:
+            print("\n[validate] PASSED: Harbor validation met expectations")
+            print(f"  NOP: reward={nop_reward} ✓")
+            print(f"  ORACLE: reward={oracle_reward} ✓")
+
+
+def _run_agents(
+    task_id: str,
+    dataset_path: Path,
+    jobs_dir: Path,
+    agent: str,
+    timeout_multiplier: float | None,
+    environment: EnvironmentType = EnvironmentType.DOCKER,
+) -> tuple[float | None, float | None]:
+    """Run NOP and/or Oracle agents, return (nop_reward, oracle_reward)."""
+    nop_reward = oracle_reward = None
+
+    if agent in ("nop", "both"):
+        # When running both, keep image for nop so oracle can reuse it
+        delete_after = agent == "nop"  # Only delete if ONLY running nop
+        code, job_result = run_harbor_agent(
+            task_id,
+            dataset_path,
+            jobs_dir,
+            "nop",
+            timeout_multiplier,
+            delete_after=delete_after,
+            environment=environment,
+        )
+        nop_reward = parse_harbor_outcome(job_result).reward
+        print(f"[validate] nop exit={code}, reward={nop_reward}")
+
+    if agent in ("oracle", "both"):
+        # Oracle always deletes (cleanup)
+        code, job_result = run_harbor_agent(
+            task_id,
+            dataset_path,
+            jobs_dir,
+            "oracle",
+            timeout_multiplier,
+            delete_after=True,
+            environment=environment,
+        )
+        oracle_reward = parse_harbor_outcome(job_result).reward
+        print(f"[validate] oracle exit={code}, reward={oracle_reward}")
+
+    return nop_reward, oracle_reward
+
+
+# ============================================================================
+# BATCH MODE
+# ============================================================================
+
+
+def _run_batch_mode(args: ValidateArgs, dataset_path: Path) -> None:
+    """Validate all tasks in parallel with clean output."""
+    console = Console()
+    jobs_dir = args.jobs_dir.resolve()
+    jobs_dir.mkdir(parents=True, exist_ok=True)
+
+    # Find tasks
+    task_dirs = [
+        d for d in dataset_path.iterdir() if d.is_dir() and (d / "tests" / "test.sh").exists()
+    ]
+    if not task_dirs:
+        console.print("[yellow]No tasks found[/yellow]")
+        return
+
+    console.print(f"[blue]Found {len(task_dirs)} task(s) to validate[/blue]")
+    console.print(f"[blue]Parallel: {args.max_parallel} | Agent: {args.agent}[/blue]")
+    if args.output_file:
+        console.print(f"[blue]Output: {args.output_file}[/blue]")
+    # Show docker prune setting for local docker
+    if args.environment == EnvironmentType.DOCKER and args.docker_prune_batch > 0:
+        console.print(f"[blue]Docker prune: every {args.docker_prune_batch} tasks[/blue]")
+    console.print()
+
+    # Run validations
+    results = asyncio.run(
+        _validate_batch(
+            task_dirs,
+            dataset_path,
+            jobs_dir,
+            args.agent,
+            args.max_parallel,
+            args.timeout_multiplier,
+            args.environment,
+            console,
+            args.output_file,
+            args.docker_prune_batch,
+        )
+    )
+
+    # Print results
+    _print_results(results, args.agent, args.show_passed, console)
+
+    # Exit with failure if any tasks failed
+    if not all(r.passed for r in results):
+        sys.exit(1)
+
+
+async def _validate_batch(
+    task_dirs: list[Path],
+    dataset_path: Path,
+    jobs_dir: Path,
+    agent: str,
+    max_parallel: int,
+    timeout_multiplier: float | None,
+    environment: EnvironmentType,
+    console: Console,
+    output_file: Path | None = None,
+    docker_prune_batch: int = 5,
+) -> list[ValidationResult]:
+    """Run validations in parallel with progress bar."""
+    semaphore = asyncio.Semaphore(max_parallel)
+    
+    # Track completed count for docker pruning
+    completed_count = 0
+    prune_lock = asyncio.Lock()
+
+    # Lock and file handle for sequential writes
+    write_lock = asyncio.Lock()
+    file_handle = None
+    if output_file:
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        file_handle = open(output_file, "w")
+        # Write header
+        file_handle.write(f"# Validation results - {len(task_dirs)} tasks\n")
+        file_handle.write("# Format: TASK_ID: NOP=<reward> ORACLE=<reward> <STATUS>\n\n")
+        file_handle.flush()
+
+    async def write_result(result: ValidationResult) -> None:
+        """Write a single result to file (thread-safe)."""
+        if file_handle is None:
+            return
+        async with write_lock:
+            line = _format_result_line(result, agent)
+            file_handle.write(line + "\n")
+            file_handle.flush()  # Ensure immediate write to disk
+
+    async def validate_one(task_dir: Path) -> ValidationResult:
+        async with semaphore:
+            try:
+                nop_reward = oracle_reward = None
+                nop_code = oracle_code = 0
+
+                # Run NOP (capture_output=True to suppress Harbor's verbose output)
+                if agent in ("nop", "both"):
+                    # When running both, keep image for nop so oracle can reuse it
+                    delete_after = agent == "nop"  # Only delete if ONLY running nop
+                    nop_code, job = await asyncio.to_thread(
+                        run_harbor_agent,
+                        task_dir.name,
+                        dataset_path,
+                        jobs_dir,
+                        "nop",
+                        timeout_multiplier,
+                        True,
+                        delete_after,
+                        environment,
+                    )
+                    nop_reward = parse_harbor_outcome(job).reward
+
+                # Run Oracle (capture_output=True to suppress Harbor's verbose output)
+                if agent in ("oracle", "both"):
+                    # Oracle always deletes (cleanup)
+                    oracle_code, job = await asyncio.to_thread(
+                        run_harbor_agent,
+                        task_dir.name,
+                        dataset_path,
+                        jobs_dir,
+                        "oracle",
+                        timeout_multiplier,
+                        True,
+                        True,
+                        environment,
+                    )
+                    oracle_reward = parse_harbor_outcome(job).reward
+
+                # Determine pass/fail
+                passed = _check_passed(agent, nop_reward, oracle_reward)
+
+                result = ValidationResult(
+                    task_id=task_dir.name,
+                    nop_reward=nop_reward,
+                    oracle_reward=oracle_reward,
+                    nop_exit_code=nop_code,
+                    oracle_exit_code=oracle_code,
+                    passed=passed,
+                )
+            except Exception as e:
+                result = ValidationResult(
+                    task_id=task_dir.name,
+                    nop_reward=None,
+                    oracle_reward=None,
+                    nop_exit_code=-1,
+                    oracle_exit_code=-1,
+                    passed=False,
+                    error=str(e),
+                )
+
+            # Write to file immediately
+            await write_result(result)
+            return result
+
+    async def maybe_prune_docker(count: int) -> None:
+        """Run docker prune if conditions are met (local docker only, every N tasks)."""
+        if environment != EnvironmentType.DOCKER:
+            return
+        if docker_prune_batch <= 0:
+            return
+        if count % docker_prune_batch != 0:
+            return
+        
+        async with prune_lock:
+            await asyncio.to_thread(_prune_docker, console)
+
+    # Run with progress bar
+    results = []
+    try:
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            console=console,
+        ) as progress:
+            task_prog = progress.add_task("[cyan]Validating tasks...", total=len(task_dirs))
+
+            for coro in asyncio.as_completed([validate_one(d) for d in task_dirs]):
+                results.append(await coro)
+                progress.update(task_prog, advance=1)
+                
+                # Docker cleanup after batch (local docker only)
+                completed_count = len(results)
+                await maybe_prune_docker(completed_count)
+    finally:
+        if file_handle:
+            # Write summary at end
+            passed = sum(1 for r in results if r.passed and not r.error)
+            failed = sum(1 for r in results if not r.passed and not r.error)
+            errors = sum(1 for r in results if r.error)
+            file_handle.write(f"\n# Summary: {passed} passed, {failed} failed, {errors} errors\n")
+            file_handle.close()
+
+    return results
+
+
+def _format_result_line(result: ValidationResult, agent: str) -> str:
+    """Format a single result as a text line."""
+    parts = [result.task_id + ":"]
+
+    if agent in ("nop", "both"):
+        if result.nop_reward is not None:
+            parts.append(f"NOP={result.nop_reward}")
+        else:
+            parts.append("NOP=ERROR")
+
+    if agent in ("oracle", "both"):
+        if result.oracle_reward is not None:
+            parts.append(f"ORACLE={result.oracle_reward}")
+        else:
+            parts.append("ORACLE=ERROR")
+
+    if result.error:
+        parts.append(f"ERROR: {result.error}")
+    elif result.passed:
+        parts.append("PASS")
+    else:
+        parts.append("FAIL")
+
+    return " ".join(parts)
+
+
+def _check_passed(agent: str, nop_reward: float | None, oracle_reward: float | None) -> bool:
+    """Check if validation passed based on agent type and rewards."""
+    if agent == "both":
+        return nop_reward == 0 and oracle_reward == 1
+    elif agent == "nop":
+        return nop_reward == 0
+    elif agent == "oracle":
+        return oracle_reward == 1
+    return False
+
+
+def _print_results(
+    results: list[ValidationResult], agent: str, show_passed: bool, console: Console
+) -> None:
+    """Print results table (failures only by default) and summary."""
+    passed = [r for r in results if r.passed and not r.error]
+    failed = [r for r in results if not r.passed and not r.error]
+    errors = [r for r in results if r.error]
+
+    # Show table if there are failures/errors or if show_passed requested
+    if failed or errors or show_passed:
+        table = Table(
+            title="Validation Failures" if not show_passed else "Validation Results",
+            title_style="bold cyan",
+            show_lines=True,
+        )
+        table.add_column("Task ID", style="cyan")
+
+        if agent in ("nop", "both"):
+            table.add_column("NOP", justify="center")
+        if agent in ("oracle", "both"):
+            table.add_column("Oracle", justify="center")
+
+        table.add_column("Status", justify="center")
+        table.add_column("Notes")
+
+        # Show errors, then failures, then passed (if requested)
+        for result in sorted(
+            errors + failed + (passed if show_passed else []), key=lambda r: r.task_id
+        ):
+            _add_result_row(table, result, agent)
+
+        console.print("\n")
+        console.print(table)
+
+    # Always show summary
+    console.print("\n[bold]Summary:[/bold]")
+    console.print(f"  ✅ Passed: {len(passed)}")
+    console.print(f"  ❌ Failed: {len(failed)}")
+    console.print(f"  ⚠️  Errors: {len(errors)}")
+    console.print(f"  📊 Total: {len(results)}")
+
+    if not failed and not errors:
+        console.print(f"\n[bold green]🎉 All {len(passed)} task(s) passed validation![/bold green]")
+
+
+def _add_result_row(table: Table, result: ValidationResult, agent: str) -> None:
+    """Add a single result row to the table."""
+    row = [result.task_id]
+
+    if result.error:
+        # Error row
+        if agent in ("nop", "both"):
+            row.append("?")
+        if agent in ("oracle", "both"):
+            row.append("?")
+        row.extend(["❌ ERROR", result.error])
+        table.add_row(*row, style="red")
+        return
+
+    if result.passed:
+        # Passed row (only shown if show_passed=True)
+        if agent in ("nop", "both"):
+            row.append(f"✓ ({result.nop_reward})" if result.nop_reward is not None else "—")
+        if agent in ("oracle", "both"):
+            row.append(f"✓ ({result.oracle_reward})" if result.oracle_reward is not None else "—")
+        row.extend(["✅ PASS", ""])
+        table.add_row(*row, style="green")
+        return
+
+    # Failed row
+    notes = []
+
+    if agent in ("nop", "both"):
+        if result.nop_reward is not None:
+            row.append(f"{'✓' if result.nop_reward == 0 else '✗'} ({result.nop_reward})")
+            if result.nop_reward != 0:
+                notes.append(f"NOP expected 0, got {result.nop_reward}")
+        else:
+            row.append("—")
+
+    if agent in ("oracle", "both"):
+        if result.oracle_reward is not None:
+            row.append(f"{'✓' if result.oracle_reward == 1 else '✗'} ({result.oracle_reward})")
+            if result.oracle_reward != 1:
+                notes.append(f"Oracle expected 1, got {result.oracle_reward}")
+        else:
+            row.append("—")
+
+    row.extend(["❌ FAIL", "; ".join(notes)])
+    table.add_row(*row, style="red")
+
+
+def _prune_docker(console: Console) -> None:
+    """Run docker cleanup to free disk space."""
+    if shutil.which("docker") is None:
+        console.print(
+            "[yellow]Skipping docker prune (docker binary not found in PATH).[/yellow]"
+        )
+        return
+
+    console.print(
+        Panel(
+            f"Running docker cleanup: {DOCKER_CLEANUP_CMD}",
+            title="Disk cleanup",
+            border_style="yellow",
+        )
+    )
+
+    try:
+        result = subprocess.run(
+            DOCKER_CLEANUP_CMD,
+            shell=True,
+            capture_output=True,
+            text=True,
+            timeout=600,
+        )
+        if result.returncode == 0:
+            console.print("[green]Docker cleanup completed[/green]")
+        else:
+            console.print(f"[yellow]Docker cleanup returned code {result.returncode}[/yellow]")
+    except subprocess.TimeoutExpired:
+        console.print("[yellow]Docker cleanup timed out after 10 minutes[/yellow]")
+    except Exception as e:
+        console.print(f"[yellow]Docker cleanup failed: {e}[/yellow]")

swegen/tools/validate_utils.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from harbor.models.environment_type import EnvironmentType
+from harbor.models.task.task import Task
+
+from .harbor_runner import parse_harbor_outcome, run_harbor_agent
+
+
+class ValidationError(Exception):
+    """Raised when Harbor validation fails (NOP or Oracle)."""
+
+    pass
+
+
+def validate_task_structure(task_dir: Path) -> bool:
+    """Validate task structure using Harbor's Task model.
+
+    This ensures the generated task has all required files and valid structure
+    before running Harbor validation.
+
+    Args:
+        task_dir: Path to the task directory
+
+    Returns:
+        True if task is valid
+
+    Raises:
+        ValidationError: If task structure is invalid with details
+    """
+    logger = logging.getLogger("swegen")
+
+    try:
+        # Use Harbor's Task model to validate structure
+        task = Task(task_dir)
+
+        # Verify required attributes are present
+        if not task.instruction or len(task.instruction.strip()) < 10:
+            raise ValidationError("Invalid instruction: too short or empty")
+
+        if not task.config:
+            raise ValidationError("Missing or invalid task.toml")
+
+        # Verify required files exist
+        paths = task.paths
+        required_files = [
+            (paths.instruction_path, "instruction.md"),
+            (paths.config_path, "task.toml"),
+            (paths.solve_path, "solution/solve.sh"),
+            (paths.test_path, "tests/test.sh"),
+        ]
+
+        for file_path, name in required_files:
+            if not file_path.exists():
+                raise ValidationError(f"Missing required file: {name}")
+
+        logger.debug(f"✓ Task structure validated: {task.name}")
+        return True
+
+    except ValidationError:
+        raise
+    except Exception as e:
+        logger.error(f"Task validation failed: {e}")
+        raise ValidationError(f"Task structure validation failed: {e}") from e
+
+
+def run_nop_oracle(
+    task_id: str,
+    dataset_path: Path,
+    jobs_dir: Path,
+    timeout_multiplier: float | None = None,
+    environment: EnvironmentType = EnvironmentType.DOCKER,
+) -> tuple[int | None, int | None, dict[str, Path | None]]:
+    """Run both NOP and Oracle validations sequentially.
+
+    Validations are always run sequentially to avoid Docker conflict issues.
+    NOP keeps the Docker image so Oracle can reuse it (much faster).
+    Oracle deletes the image after running (cleanup).
+
+    Args:
+        task_id: Task identifier
+        dataset_path: Harbor dataset root path
+        jobs_dir: Jobs directory path
+        timeout_multiplier: Optional timeout multiplier
+        environment: Environment type (docker, daytona, e2b, modal, runloop, gke)
+
+    Returns:
+        Tuple of (nop_reward, oracle_reward, job_dirs) where:
+        - nop_reward: 0 if tests fail on buggy code (expected), None if error
+        - oracle_reward: 1 if tests pass after fix (expected), None if error
+        - job_dirs: Dict mapping "nop"/"oracle" to job result paths
+    """
+    job_dirs: dict[str, Path | None] = {"nop": None, "oracle": None}
+
+    # NOP: Keep image (delete_after=False) so Oracle can reuse it
+    _, nop_result = run_harbor_agent(
+        task_id=task_id,
+        dataset_path=dataset_path,
+        jobs_dir=jobs_dir,
+        agent="nop",
+        timeout_multiplier=timeout_multiplier,
+        capture_output=True,
+        delete_after=False,
+        environment=environment,
+    )
+    nop_reward = parse_harbor_outcome(nop_result).reward
+    job_dirs["nop"] = nop_result.parent if nop_result else None
+
+    # Oracle: Delete image after running (cleanup)
+    _, oracle_result = run_harbor_agent(
+        task_id=task_id,
+        dataset_path=dataset_path,
+        jobs_dir=jobs_dir,
+        agent="oracle",
+        timeout_multiplier=timeout_multiplier,
+        capture_output=True,
+        delete_after=True,
+        environment=environment,
+    )
+    oracle_reward = parse_harbor_outcome(oracle_result).reward
+    job_dirs["oracle"] = oracle_result.parent if oracle_result else None
+
+    return nop_reward, oracle_reward, job_dirs
+
+
+def check_validation_passed(nop_reward: int | None, oracle_reward: int | None) -> bool:
+    """Check if validation passed (NOP=0, Oracle=1)."""
+    return nop_reward == 0 and oracle_reward == 1
+
+
+# Re-export for convenience
+__all__ = [
+    "ValidationError",
+    "validate_task_structure",
+    "run_nop_oracle",
+    "check_validation_passed",
+    # Low-level (from harbor_runner)
+    "run_harbor_agent",
+    "parse_harbor_outcome",
+]