swegen 0.1.0__py3-none-any.whl

swegen/farm/farm_hand.py

@@ -0,0 +1,342 @@
+from __future__ import annotations
+
+import shutil
+import time
+import traceback
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from rich.console import Console
+from rich.panel import Panel
+
+from swegen.config import CreateConfig, FarmConfig
+from swegen.create import MissingIssueError, TrivialPRError, ValidationError
+from swegen.create.create import run_reversal
+from swegen.create.task_reference import TaskReferenceStore
+
+
+def _now_utc() -> datetime:
+    return datetime.now(UTC)
+
+
+def _slug(repo: str) -> str:
+    """Convert repo to slug using SWEBench convention: owner/repo -> owner__repo"""
+    return repo.replace("/", "__")
+
+
+def _task_id(repo: str, pr_number: int) -> str:
+    """Generate task ID using SWEBench convention: owner__repo-number"""
+    return f"{_slug(repo)}-{pr_number}"
+
+
+@dataclass
+class PRCandidate:
+    """A candidate PR for task generation."""
+
+    number: int
+    title: str
+    created_at: str
+    merged_at: str
+    author: str
+    files_changed: int
+    additions: int
+    deletions: int
+    url: str
+
+
+@dataclass
+class TaskResult:
+    """Result of processing a single PR into a task."""
+
+    repo: str
+    pr_number: int
+    task_id: str
+    status: str  # "success", "failed", or "dry-run"
+    message: str
+    duration_seconds: float
+    timestamp: str
+    category: str = None  # Category for detailed tracking
+
+
+def _cleanup_task(task_id: str, tasks_root: Path, console: Console) -> None:
+    removed_any = False
+    paths = [
+        tasks_root / task_id,
+        Path("trash") / task_id,
+    ]
+    for path in paths:
+        if path.exists():
+            shutil.rmtree(path, ignore_errors=True)
+            removed_any = True
+    if removed_any:
+        console.print(f"[dim]Cleaned up incomplete task directory: {task_id}[/dim]")
+
+
+def _classify_failure(stderr: str) -> tuple[str, str]:
+    """Classify failure reason and return (category, message).
+    
+    Categories:
+    - trivial: Trivial PR (too small/simple)
+    - no_issue: No linked issue
+    - no_tests: No tests detected
+    - validation_failed: Harbor validation failed
+    - already_exists: Task already exists
+    - rate_limit: GitHub API rate limit
+    - quota_exceeded: OpenAI quota exceeded
+    - timeout: Command timeout
+    - git_error: Git checkout/commit errors
+    - other: Unknown/other errors
+    """
+    lowered = stderr.lower()
+    if "trivial" in stderr:
+        return "trivial", "Trivial PR (skipped)"
+    if "no linked issue" in lowered or "missingissueerror" in lowered:
+        return "no_issue", "No linked issue (skipped)"
+    if "validation failed" in lowered or "harbor validation" in lowered:
+        return "validation_failed", "Validation failed (NOP or Oracle)"
+    if "task already exists" in lowered or "file exists" in lowered:
+        return "already_exists", "Task already exists (skipped)"
+    if "no test" in stderr:
+        return "no_tests", "No tests detected"
+    if "rate limit exceeded" in lowered and "github" in lowered:
+        return "rate_limit", "GitHub API rate limit exceeded (set GITHUB_TOKEN)"
+    if "insufficient_quota" in lowered or "exceeded your current quota" in lowered:
+        return "quota_exceeded", "OpenAI API quota exceeded (check billing)"
+    if "timed out" in lowered or "timeout" in lowered:
+        return "timeout", "Command timed out"
+    if "cannot checkout commit" in lowered or "force-pushed or deleted" in lowered:
+        return "git_error", "Git commit not found (may be force-pushed or deleted)"
+    if "git checkout" in lowered:
+        return "git_error", "Git checkout failed (repo cache may be corrupted)"
+    
+    message = (stderr or "Unknown error").replace("\n", " ")
+    return "other", message
+
+
+def _print_success(
+    console: Console,
+    pr: PRCandidate,
+    task_id: str,
+    harbor_dir: Path,
+) -> None:
+    console.print(
+        Panel.fit(
+            f"🎉 Successfully generated task\n[bold]{task_id}[/bold]\nHarbor: {harbor_dir}",
+            title=f"PR #{pr.number}",
+            border_style="green",
+        )
+    )
+
+
+def _gate_task(
+    task_id: str,
+    tasks_root: Path,
+) -> tuple[bool, str]:
+    """
+    Validate that the task directory exists.
+
+    Returns:
+        Tuple of (success, message)
+    """
+    task_dir = tasks_root / task_id
+    if not task_dir.exists():
+        return False, f"Task directory missing: {task_dir}"
+
+    return True, f"Task generated successfully at {task_dir}"
+
+
+def _run_reversal_for_pr(
+    pr: PRCandidate,
+    config: FarmConfig,
+    tasks_root: Path,
+    console: Console,
+) -> TaskResult:
+    start = time.time()
+    task_id = _task_id(config.repo, pr.number)
+    harbor_dir = tasks_root / task_id
+
+    # Wrap everything in try-except to catch unexpected errors
+    try:
+        return _run_reversal_for_pr_impl(
+            pr, config, tasks_root, console, task_id, harbor_dir, start
+        )
+    except Exception as e:
+        # Catch any unexpected exception and return proper error
+        error_msg = f"Unexpected error: {type(e).__name__}: {str(e)}"
+        console.print(f"[red]✗ PR #{pr.number}: {error_msg}[/red]")
+        console.print(f"[dim]{traceback.format_exc()}[/dim]")
+        _cleanup_task(task_id, tasks_root, console)
+        return TaskResult(
+            repo=config.repo,
+            pr_number=pr.number,
+            task_id=task_id,
+            status="failed",
+            message=error_msg,
+            duration_seconds=round(time.time() - start, 2),
+            timestamp=_now_utc().isoformat(),
+            category="other",
+        )
+
+
+def _run_reversal_for_pr_impl(
+    pr: PRCandidate,
+    config: FarmConfig,
+    tasks_root: Path,
+    console: Console,
+    task_id: str,
+    harbor_dir: Path,
+    start: float,
+) -> TaskResult:
+    if config.dry_run:
+        console.print(f"[cyan]DRY RUN[/cyan] would generate task for PR #{pr.number} -> {task_id}")
+        return TaskResult(
+            repo=config.repo,
+            pr_number=pr.number,
+            task_id=task_id,
+            status="dry-run",
+            message="Dry run (skipped actual execution)",
+            duration_seconds=0.0,
+            timestamp=_now_utc().isoformat(),
+            category=None,
+        )
+
+    # Build CreateConfig for run_reversal
+    create_config = CreateConfig(
+        repo=config.repo,
+        pr=pr.number,
+        output=config.output,
+        cc_timeout=config.cc_timeout,
+        validate=config.validate,  # Run Harbor validation if --validate flag is set
+        force=config.force,
+        state_dir=config.state_dir,
+        verbose=config.verbose,
+        quiet=False,
+        use_cache=not config.no_cache,
+        require_minimum_difficulty=config.require_minimum_difficulty,
+        min_source_files=config.min_source_files,
+        max_source_files=config.max_source_files,
+        require_issue=config.issue_only,
+        environment=config.environment,
+    )
+
+    # Capture any errors from the pipeline
+    success = False
+    error_msg = ""
+    error_category = None
+
+    try:
+        # Call the pipeline directly instead of using subprocess
+        run_reversal(create_config)
+        success = True
+    except TrivialPRError as e:
+        # Trivial PR - not an error, just skip it
+        error_msg = str(e)
+        error_category = "trivial"
+        success = False
+    except MissingIssueError as e:
+        # No linked issue - not an error, just skip it
+        error_msg = str(e)
+        error_category = "no_issue"
+        success = False
+    except ValidationError as e:
+        # Validation failed - not an error, just skip it
+        error_msg = str(e)
+        error_category = "validation_failed"
+        success = False
+    except FileExistsError as e:
+        # Task already exists - skip it
+        error_msg = f"Task already exists: {str(e)}"
+        error_category = "already_exists"
+        success = False
+    except Exception as e:
+        # Other errors
+        error_msg = f"{type(e).__name__}: {str(e)}"
+        if config.verbose:
+            console.print(f"[red]{traceback.format_exc()}[/red]")
+        # Classify the error
+        error_category, _ = _classify_failure(error_msg)
+        success = False
+
+    if success:
+        if not harbor_dir.exists():
+            # Check for trivial PR (should have been caught by TrivialPRError)
+            if "trivial" in error_msg.lower():
+                failure_reason = "Trivial PR (skipped)"
+                failure_category = "trivial"
+            else:
+                failure_reason = (
+                    "Pipeline reported success but Harbor task directory was not created."
+                )
+                failure_category = "other"
+            _cleanup_task(task_id, tasks_root, console)
+            console.print(f"[red]✗ PR #{pr.number}: {failure_reason}[/red]")
+            return TaskResult(
+                repo=config.repo,
+                pr_number=pr.number,
+                task_id=task_id,
+                status="failed",
+                message=failure_reason,
+                duration_seconds=round(time.time() - start, 2),
+                timestamp=_now_utc().isoformat(),
+                category=failure_category,
+            )
+
+        # Task is already in Harbor format (create now generates directly to Harbor)
+        duration = time.time() - start
+        gate_ok, gate_msg = _gate_task(task_id, tasks_root)
+        if gate_ok:
+            _print_success(console, pr, task_id, harbor_dir)
+
+            # Save task reference for future PRs
+            try:
+                reference_store = TaskReferenceStore()
+                reference_store.save(
+                    repo=config.repo,
+                    task_id=task_id,
+                    pr_number=pr.number,
+                )
+            except Exception as e:
+                console.print(f"[yellow]Warning: Could not save task reference: {e}[/yellow]")
+
+            return TaskResult(
+                repo=config.repo,
+                pr_number=pr.number,
+                task_id=task_id,
+                status="success",
+                message=gate_msg,
+                duration_seconds=round(duration, 2),
+                timestamp=_now_utc().isoformat(),
+                category=None,
+            )
+
+        # Gate failed
+        failure_reason = gate_msg
+        failure_category = "other"
+        _cleanup_task(task_id, tasks_root, console)
+        console.print(f"[red]✗ PR #{pr.number}: {failure_reason}[/red]")
+        return TaskResult(
+            repo=config.repo,
+            pr_number=pr.number,
+            task_id=task_id,
+            status="failed",
+            message=failure_reason,
+            duration_seconds=round(duration, 2),
+            timestamp=_now_utc().isoformat(),
+            category=failure_category,
+        )
+
+    # Pipeline failed
+    failure_category, failure_reason = _classify_failure(error_msg)
+    _cleanup_task(task_id, tasks_root, console)
+    console.print(f"[red]✗ PR #{pr.number}: {failure_reason}[/red]")
+    return TaskResult(
+        repo=config.repo,
+        pr_number=pr.number,
+        task_id=task_id,
+        status="failed",
+        message=failure_reason,
+        duration_seconds=round(time.time() - start, 2),
+        timestamp=_now_utc().isoformat(),
+        category=failure_category,
+    )

swegen/farm/fetcher.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import os
+import time
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import requests
+from rich.console import Console
+
+from swegen.create import is_test_file
+
+from .farm_hand import PRCandidate, _slug
+from .state import StreamState
+
+
+def load_skip_list(skip_list_file: Path, repo: str) -> set[int]:
+    """Load PR numbers from a skip list file for the given repository.
+
+    The file should contain task IDs like (SWEBench format):
+        owner__repo-123
+        owner__repo-456
+
+    This function extracts PR numbers matching the current repo.
+
+    Args:
+        skip_list_file: Path to the skip list file
+        repo: Repository in owner/repo format (e.g., "python/pillow")
+
+    Returns:
+        Set of PR numbers to skip
+    """
+    if not skip_list_file.exists():
+        return set()
+
+    # Create expected prefix from repo (e.g., "python/pillow" -> "python__pillow-")
+    repo_slug = _slug(repo)
+    prefix = f"{repo_slug}-"
+
+    skip_prs: set[int] = set()
+    try:
+        content = skip_list_file.read_text()
+        for line in content.strip().split("\n"):
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            # Check if this task ID matches our repo
+            if line.startswith(prefix):
+                # Extract PR number from task ID (e.g., "python__pillow-9272" -> 9272)
+                pr_part = line[len(prefix) :]
+                try:
+                    pr_number = int(pr_part)
+                    skip_prs.add(pr_number)
+                except ValueError:
+                    # Ignore malformed entries
+                    pass
+    except Exception:
+        # If file read fails, return empty set
+        pass
+
+    return skip_prs
+
+
+class StreamingPRFetcher:
+    """Fetches PRs from GitHub in a streaming fashion.
+
+    Yields PRs one at a time after filtering. Handles pagination,
+    rate limiting, and various filters (merged, has tests).
+
+    Attributes:
+        repo: Repository in "owner/repo" format
+        console: Rich console for output
+        state: StreamState for tracking processed PRs
+        min_files: Minimum total files changed (early approximate filter)
+        require_tests: Whether PRs must have test file changes
+        api_delay: Delay between API calls in seconds
+    """
+
+    def __init__(
+        self,
+        repo: str,
+        console: Console,
+        state: StreamState,
+        min_files: int = 3,
+        require_tests: bool = True,
+        api_delay: float = 0.5,
+    ):
+        self.repo = repo
+        self.console = console
+        self.state = state
+        self.min_files = min_files
+        self.require_tests = require_tests
+        self.api_delay = api_delay
+
+        # GitHub API setup
+        self.api_base = "https://api.github.com"
+        self.github_token = (
+            os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") or os.getenv("REPO_CREATION_TOKEN")
+        )
+        self.headers = {
+            "Accept": "application/vnd.github+json",
+            "User-Agent": "swegen-stream-farm",
+        }
+        if self.github_token:
+            self.headers["Authorization"] = f"token {self.github_token}"
+
+    def stream_prs(
+        self,
+        resume_from_time: str | None = None,
+    ) -> Iterator[PRCandidate]:
+        """Stream PRs from GitHub API, skipping already processed ones.
+
+        Yields PRs one at a time after validation. Fetches in pages
+        but yields immediately, allowing processing to happen concurrently.
+
+        Works backwards in time from present day (or resume point) by PR creation time.
+
+        Args:
+            resume_from_time: If specified, only process PRs created before this timestamp.
+                             Format: ISO 8601 string (e.g., "2024-01-15T23:59:59.999999+00:00")
+                             This allows resuming from a specific time and continuing backwards.
+
+        Yields:
+            PRCandidate instances for each PR that passes filters
+        """
+        yielded = 0
+        page = 1
+
+        # Fetch closed PRs sorted by created time descending
+        # This gives us all merged PRs in reverse chronological order (by creation)
+        params_base = {
+            "state": "closed",
+            "sort": "created",
+            "direction": "desc",
+            "per_page": 100,
+        }
+
+        self.console.print(f"[dim]Streaming PRs from {self.repo}...[/dim]")
+        if resume_from_time is not None:
+            resume_dt = datetime.fromisoformat(resume_from_time.replace("Z", "+00:00"))
+            self.console.print(
+                f"[yellow]Resuming from {resume_dt.strftime('%Y-%m-%d %H:%M:%S UTC')} "
+                f"(only processing PRs created before this time)[/yellow]"
+            )
+        elif self.state.total_processed > 0:
+            self.console.print(
+                f"[yellow]Resuming: {self.state.total_processed} PRs already processed "
+                f"({self.state.successful} successful, {self.state.failed} failed)[/yellow]"
+            )
+            if self.state.last_created_at:
+                last_dt = datetime.fromisoformat(self.state.last_created_at.replace("Z", "+00:00"))
+                self.console.print(
+                    f"[yellow]Last processed PR created at: {last_dt.strftime('%Y-%m-%d %H:%M:%S UTC')}[/yellow]"
+                )
+
+        skipped_stats = {
+            "already_processed": 0,
+            "in_skip_list": 0,
+            "not_merged": 0,
+            "too_few_changes": 0,
+            "no_tests": 0,
+            "api_error": 0,
+            "after_resume_time": 0,
+        }
+
+        while True:
+            # Fetch next page
+            url = f"{self.api_base}/repos/{self.repo}/pulls"
+            params: dict[str, Any] = {**params_base, "page": page}
+
+            try:
+                resp = requests.get(url, headers=self.headers, params=params, timeout=30)
+                resp.raise_for_status()
+            except requests.exceptions.RequestException as exc:
+                self.console.print(f"[red]API error on page {page}: {exc}[/red]")
+                skipped_stats["api_error"] += 1
+                break
+
+            prs = resp.json()
+            if not prs:
+                self.console.print("[dim]No more PRs available[/dim]")
+                break
+
+            # Check rate limiting
+            remaining = int(resp.headers.get("X-RateLimit-Remaining", 999))
+            if remaining < 10:
+                reset_time = int(resp.headers.get("X-RateLimit-Reset", 0))
+                wait_seconds = max(0, reset_time - time.time())
+                self.console.print(
+                    f"[yellow]Rate limit low ({remaining}), waiting {wait_seconds:.0f}s...[/yellow]"
+                )
+                time.sleep(wait_seconds + 1)
+
+            # Process PRs from this page
+            for pr_data in prs:
+                pr_number = pr_data["number"]
+
+                # Filter: must be merged
+                merged_at = pr_data.get("merged_at")
+                if not merged_at:
+                    skipped_stats["not_merged"] += 1
+                    continue
+
+                # Get creation time
+                created_at = pr_data.get("created_at")
+
+                # Skip if this PR was created after our resume time
+                # (we're working backwards, so we only want PRs created before the resume point)
+                if resume_from_time is not None and created_at:
+                    pr_created_dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+                    resume_dt = datetime.fromisoformat(resume_from_time.replace("Z", "+00:00"))
+                    if pr_created_dt >= resume_dt:
+                        skipped_stats["after_resume_time"] += 1
+                        continue
+
+                # Skip if already processed
+                if pr_number in self.state.processed_prs:
+                    skipped_stats["already_processed"] += 1
+                    continue
+
+                # Skip if in external skip list
+                if pr_number in self.state.skip_list_prs:
+                    skipped_stats["in_skip_list"] += 1
+                    continue
+
+                # Fetch full PR details
+                try:
+                    pr_url = f"{self.api_base}/repos/{self.repo}/pulls/{pr_number}"
+                    pr_resp = requests.get(pr_url, headers=self.headers, timeout=30)
+                    pr_resp.raise_for_status()
+                    pr_full = pr_resp.json()
+                    time.sleep(self.api_delay)
+                except requests.exceptions.RequestException:
+                    skipped_stats["api_error"] += 1
+                    continue
+
+                # Get file change count for metadata
+                files_changed = pr_full.get("changed_files", 0)
+
+                # Filter: minimum files changed (early approximate filter to save API calls)
+                # Note: This is total files (including tests/docs/CI)
+                # The accurate source-only check happens later in the pipeline
+                if files_changed < self.min_files:
+                    skipped_stats["too_few_changes"] += 1
+                    continue
+
+                # Filter: test file changes (if required)
+                if self.require_tests:
+                    try:
+                        has_tests = self._pr_has_test_changes(pr_number)
+                        time.sleep(self.api_delay)
+                        if not has_tests:
+                            skipped_stats["no_tests"] += 1
+                            continue
+                    except requests.exceptions.RequestException:
+                        skipped_stats["api_error"] += 1
+                        continue
+
+                # Passed all filters - yield this PR
+                candidate = PRCandidate(
+                    number=pr_number,
+                    title=pr_full.get("title", ""),
+                    created_at=pr_full.get("created_at", ""),
+                    merged_at=pr_full.get("merged_at", ""),
+                    author=pr_full.get("user", {}).get("login", "unknown"),
+                    files_changed=files_changed,
+                    additions=pr_full.get("additions", 0),
+                    deletions=pr_full.get("deletions", 0),
+                    url=pr_full.get("html_url", ""),
+                )
+
+                self.state.total_fetched += 1
+                yielded += 1
+
+                yield candidate
+
+            # Move to next page
+            page += 1
+
+            # Break if we got fewer results than expected (last page)
+            if len(prs) < 100:
+                self.console.print("[dim]Reached last page of PRs[/dim]")
+                break
+
+        # Final stats
+        self._print_stats(skipped_stats)
+        self.console.print(
+            f"[green]Stream complete: {yielded} PRs yielded, "
+            f"{self.state.total_processed} total processed[/green]"
+        )
+
+    def _pr_has_test_changes(self, pr_number: int) -> bool:
+        """Check if PR modifies test files.
+
+        Args:
+            pr_number: PR number to check
+
+        Returns:
+            True if PR has test file changes
+        """
+        files_url = f"{self.api_base}/repos/{self.repo}/pulls/{pr_number}/files"
+        page = 1
+
+        while True:
+            params = {"page": page, "per_page": 100}
+            resp = requests.get(files_url, headers=self.headers, params=params, timeout=30)
+            resp.raise_for_status()
+
+            files = resp.json()
+            if not files:
+                break
+
+            for file in files:
+                filename = file.get("filename", "")
+                # Use centralized test file detection (supports all languages)
+                if is_test_file(filename):
+                    return True
+
+            if len(files) < 100:
+                break
+            page += 1
+
+        return False
+
+    def _print_stats(self, skipped: dict) -> None:
+        """Print skipping statistics.
+
+        Args:
+            skipped: Dict of skip reasons to counts
+        """
+        total_skipped = sum(skipped.values())
+        if total_skipped == 0:
+            return
+
+        self.console.print("\n[dim]Skipped PRs:[/dim]")
+        for reason, count in skipped.items():
+            if count > 0:
+                self.console.print(f"  [dim]• {reason}: {count}[/dim]")