fc-data 0.2.0__py3-none-any.whl

datasmith/runners/render_problems.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from datasmith.runners.base import BaseRunner
+from datasmith.utils import get_client, get_logger
+
+logger = get_logger("runners.render_problems")
+
+
+class RenderProblemsRunner(BaseRunner):
+    """Scrape linked issues, run ProblemExtractor, and persist deconstructed context.
+
+    For each PR this runner:
+    1. BFS-scrapes linked GitHub issues via the GitHub API.
+    2. Runs :class:`~datasmith.agents.extractors.ProblemExtractor` (DSPy) once to
+       split the PR body into four structured fields.
+    3. Renders the full Jinja2 problem-statement template.
+    4. Upserts everything into the ``candidate_prs`` table (raw components + rendered).
+    5. Updates ``pull_requests.rendered_problem`` and the new
+       ``pull_requests.problem_description`` column (problem-only text).
+    """
+
+    def __init__(self, gh: Any, n_concurrent: int = 5) -> None:
+        super().__init__(name="render_problems", n_concurrent=n_concurrent)
+        self._gh = gh
+
+    async def _process_item(self, item: Any) -> None:
+        """Render the problem statement for a single PR dict."""
+        owner: str = item["owner"]
+        repo: str = item["repo"]
+        issue_number: int = item["issue_number"]
+        merge_commit_sha: str = item.get("merge_commit_sha", "") or ""
+        repo_description: str = item.get("repo_description", "") or ""
+
+        from datasmith.agents.extractors import ProblemExtractor
+        from datasmith.github.links import scrape_links
+        from datasmith.github.models import PR
+        from datasmith.github.render import render_problem_statement
+
+        pr = PR(
+            repository=f"{owner}/{repo}",
+            issue_number=issue_number,
+            title=item.get("title", ""),
+            body=item.get("body", ""),
+            created_at=item.get("created_at"),
+        )
+
+        # BFS-scrape linked issues (async GitHub API calls)
+        issues = await scrape_links(
+            pr,
+            self._gh.get_issue_expanded,
+            depth=2,
+            only_issues=True,
+            limit=6,
+        )
+
+        logger.info(
+            "Scraped %d linked issues for %s/%s#%d",
+            len(issues),
+            owner,
+            repo,
+            issue_number,
+        )
+
+        # Run ProblemExtractor once (DSPy LLM call — run in thread)
+        extraction = await asyncio.to_thread(
+            ProblemExtractor().extract_problem,
+            item.get("title", ""),
+            item.get("body", ""),
+        )
+        problem_description = extraction.to_problem_markdown()
+
+        # Render the full problem statement using the pre-extracted observations
+        # (pass initial_observations to avoid a second ProblemExtractor call)
+        rendered = await asyncio.to_thread(
+            render_problem_statement,
+            pr,
+            issues=issues,
+            repo_description=repo_description,
+            anonymize=True,
+            extract=False,
+            initial_observations=problem_description or getattr(pr, "body", ""),
+        )
+
+        # Serialize linked issues for storage (mode="json" converts datetime → ISO string)
+        issues_json = [issue.model_dump(mode="json") for issue in issues]
+
+        # Upsert all raw components + rendered output into candidate_prs
+        client = get_client()
+        client.table("candidate_prs").upsert({
+            "owner": owner,
+            "repo": repo,
+            "issue_number": issue_number,
+            "merge_commit_sha": merge_commit_sha,
+            "repo_description": repo_description,
+            "issues_json": issues_json,
+            "initial_observations": extraction.initial_observations,
+            "triage_attempts": extraction.triage_attempts,
+            "solution_overview": extraction.solution_overview,
+            "solution_observations": extraction.solution_observations,
+            "rendered_problem": rendered,
+        }).execute()
+
+        # Keep pull_requests in sync: rendered_problem (used by synthesize_images)
+        # and problem_description (problem-only extracted text)
+        client.table("pull_requests").update({
+            "rendered_problem": rendered,
+            "problem_description": problem_description,
+        }).eq("owner", owner).eq("repo", repo).eq("issue_number", issue_number).execute()
+
+        logger.info("Rendered problem context for %s/%s#%d", owner, repo, issue_number)

datasmith/runners/resolve_packages.py

@@ -0,0 +1,66 @@
+"""Runner for resolving Python dependencies for classified PRs."""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import json
+from typing import Any
+
+from datasmith.runners.base import BaseRunner
+from datasmith.utils import get_client, get_logger
+
+logger = get_logger("runners.resolve_packages")
+
+
+class ResolvePackagesRunner(BaseRunner):
+    """Resolve dependencies for classified PRs and persist to the packages table."""
+
+    def __init__(self, n_concurrent: int = 16) -> None:
+        super().__init__(name="resolve_packages", n_concurrent=n_concurrent)
+
+    async def _process_item(self, item: Any) -> None:
+        """Process an item dict with owner, repo, sha."""
+        owner = item["owner"]
+        repo = item["repo"]
+        sha = item["sha"]
+
+        from datasmith.resolution import analyze_commit
+
+        loop = asyncio.get_running_loop()
+        result = await loop.run_in_executor(None, functools.partial(analyze_commit, sha, f"{owner}/{repo}"))
+
+        client = get_client()
+
+        if result is None:
+            logger.info("Resolution returned None for %s/%s@%s", owner, repo, sha[:8])
+            return
+
+        env_payload = json.dumps(result.get("final_dependencies", []))
+
+        row = {
+            "owner": owner,
+            "repo": repo,
+            "sha": sha,
+            "package_name": result.get("package_name"),
+            "package_version": result.get("package_version"),
+            "python_version": result.get("python_version", ""),
+            "env_payload": env_payload,
+            "build_commands": result.get("build_command"),
+            "install_commands": result.get("install_command"),
+            "primary_root": result.get("primary_root"),
+            "resolution_strategy": result.get("resolution_strategy"),
+            "can_install": result.get("can_install", False),
+            "requires_python": None,
+        }
+
+        client.table("packages").upsert(row).execute()
+        logger.info(
+            "Resolved %s/%s@%s: python=%s can_install=%s deps=%d",
+            owner,
+            repo,
+            sha[:8],
+            result.get("python_version"),
+            result.get("can_install"),
+            len(result.get("final_dependencies", [])),
+        )

datasmith/runners/scrape_commits.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from datasmith.filters import symbolic_compliance
+from datasmith.runners.base import BaseRunner
+from datasmith.utils import get_client, get_logger
+from datasmith.utils.db import fetch_all
+
+logger = get_logger("runners.scrape_commits")
+
+
+def _parse_iso(value: str | None) -> datetime | None:
+    """Parse an ISO-8601 date or datetime string to a timezone-aware datetime."""
+    if not value:
+        return None
+    # Handle date-only strings like "2024-01-01"
+    if "T" not in value:
+        return datetime.fromisoformat(value).replace(tzinfo=timezone.utc)
+    # Handle full ISO datetime strings (with or without trailing Z)
+    cleaned = value.replace("Z", "+00:00")
+    return datetime.fromisoformat(cleaned)
+
+
+def _should_skip_pr(
+    pr_data: dict[str, Any],
+    since: datetime | None,
+    until: datetime | None,
+    seen_shas: set[str],
+) -> bool | str:
+    """Return False if the PR should be processed, 'skip' to skip, 'stop' to halt pagination."""
+    if since:
+        created = _parse_iso(pr_data.get("created_at"))
+        if created and created < since:
+            return "stop"
+
+    if not pr_data.get("merged_at"):
+        return "skip"
+
+    merged = _parse_iso(pr_data.get("merged_at"))
+    if merged:
+        if since and merged < since:
+            return "skip"
+        if until and merged >= until:
+            return "skip"
+
+    sha = pr_data.get("merge_commit_sha", "")
+    if sha:
+        if sha in seen_shas:
+            return "skip"
+        seen_shas.add(sha)
+
+    return False
+
+
+def _sanitize_text(value: str) -> str:
+    """Strip Postgres-illegal null bytes from text values."""
+    return value.replace("\u0000", "")
+
+
+async def _build_record(
+    gh: Any,
+    owner: str,
+    repo: str,
+    pr_data: dict[str, Any],
+) -> dict[str, Any]:
+    """Fetch diff/files and build the upsert record for a single PR."""
+    issue_number = pr_data["number"]
+    title = _sanitize_text(pr_data.get("title", ""))
+
+    diff = await gh.get_diff(owner, repo, issue_number)
+    if diff:
+        diff = _sanitize_text(diff)
+    files = await gh.get_files(owner, repo, issue_number)
+
+    file_changes: list[dict[str, Any]] | None = None
+    if files:
+        file_changes = [
+            {
+                "filename": f.get("filename", ""),
+                "additions": f.get("additions", 0),
+                "deletions": f.get("deletions", 0),
+            }
+            for f in files
+        ]
+
+    record: dict[str, Any] = {
+        "owner": owner,
+        "repo": repo,
+        "issue_number": issue_number,
+        "title": title,
+        "body": _sanitize_text(pr_data.get("body", "") or ""),
+        "state": pr_data.get("state", ""),
+        "created_at": pr_data.get("created_at"),
+        "merged_at": pr_data.get("merged_at"),
+        "closed_at": pr_data.get("closed_at"),
+        "merge_commit_sha": pr_data.get("merge_commit_sha", ""),
+        "base_sha": pr_data.get("base", {}).get("sha", ""),
+        "head_sha": pr_data.get("head", {}).get("sha", ""),
+        "labels": [label["name"] for label in pr_data.get("labels", [])],
+        "is_performance_commit_symbolic": symbolic_compliance(
+            title=title,
+            patch=diff or None,
+            file_changes=file_changes,
+        ),
+    }
+    if diff:
+        record["patch"] = diff
+    if file_changes:
+        record["file_changes"] = file_changes
+    return record
+
+
+class ScrapeCommitsRunner(BaseRunner):
+    """Scrape PRs for each repo, run compliance hooks, store in pull_requests table."""
+
+    def __init__(
+        self,
+        github_client: Any,
+        n_concurrent: int = 5,
+        since: str | None = None,
+        until: str | None = None,
+    ) -> None:
+        super().__init__(name="scrape_commits", n_concurrent=n_concurrent)
+        self._gh = github_client
+        self._since = _parse_iso(since)
+        self._until = _parse_iso(until)
+
+    async def _process_item(self, item: Any) -> None:
+        """Process a (owner, repo) tuple — scrape its merged PRs via GraphQL."""
+        owner, repo = item if isinstance(item, tuple) else item.split("/")
+
+        # Pre-fetch existing issue_numbers for this repo to avoid redundant API calls
+        existing_rows = fetch_all(
+            "pull_requests",
+            select="issue_number",
+            filters={"owner": owner, "repo": repo},
+        )
+        existing_issues: set[int] = {r["issue_number"] for r in existing_rows}
+
+        seen_shas: set[str] = set()
+        client = get_client()
+        count = 0
+
+        async for page in self._gh.paginate_merged_prs(owner, repo):
+            stop = False
+            for pr_data in page:
+                verdict = _should_skip_pr(pr_data, self._since, self._until, seen_shas)
+                if verdict == "stop":
+                    stop = True
+                    break
+                if verdict == "skip":
+                    continue
+
+                if pr_data["number"] in existing_issues:
+                    continue
+
+                record = await _build_record(self._gh, owner, repo, pr_data)
+                client.table("pull_requests").upsert(record).execute()
+                count += 1
+
+            if stop:
+                break
+
+        logger.info("Scraped %d merged PRs for %s/%s", count, owner, repo)

datasmith/runners/scrape_repos.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import Any
+
+from datasmith.runners.base import BaseRunner
+from datasmith.utils import get_client, get_logger
+
+logger = get_logger("runners.scrape_repos")
+
+
+class ScrapeReposRunner(BaseRunner):
+    """Scrape GitHub repos via search API and store in repositories table."""
+
+    def __init__(self, github_client: Any, n_concurrent: int = 5) -> None:
+        super().__init__(name="scrape_repos", n_concurrent=n_concurrent)
+        self._gh = github_client
+
+    async def _process_item(self, item: Any) -> None:
+        """Process a search query or repo identifier."""
+        owner, repo = item if isinstance(item, tuple) else item.split("/")
+
+        # Check if already exists
+        client = get_client()
+        resp = client.table("repositories").select("owner").eq("owner", owner).eq("repo", repo).execute()
+        if resp.data:
+            logger.debug("Skipping existing repo: %s/%s", owner, repo)
+            return
+
+        # Fetch repo info via GitHub API
+        resp_gh = await self._gh._request("GET", f"/repos/{owner}/{repo}")
+        if resp_gh is None:
+            return
+
+        data = resp_gh.json()
+        client.table("repositories").upsert({
+            "owner": owner,
+            "repo": repo,
+            "url": data.get("html_url", ""),
+            "language": data.get("language", ""),
+            "stars": data.get("stargazers_count", 0),
+            "topics": data.get("topics", []),
+            "description": data.get("description", ""),
+        }).execute()
+        logger.info("Stored repo: %s/%s", owner, repo)

datasmith/runners/synthesize_images.py

@@ -0,0 +1,310 @@
+from __future__ import annotations
+
+import asyncio
+import tempfile
+import threading
+from typing import Any
+
+from datasmith.agents.synthesizer import Synthesizer
+from datasmith.runners.base import BaseRunner
+from datasmith.utils import get_client, get_logger
+
+logger = get_logger("runners.synthesize_images")
+
+
+def _ensure_prerequisite_images(owner: str, repo: str, py_version: str = "") -> None:
+    """Build the base and repo Docker images if they don't exist locally.
+
+    The three-tier hierarchy (base → repo → PR) requires each parent image
+    to be present in the local daemon before the child can be built.
+    """
+    from datasmith.docker.images import ImageManager, get_base_image_name, get_repo_image_name
+
+    mgr = ImageManager()
+    base_tag = get_base_image_name()
+    repo_tag = get_repo_image_name(owner, repo)
+
+    if not mgr.image_exists(base_tag):
+        logger.info("Building missing base image: %s", base_tag)
+        mgr.build_base_image(py_version=py_version)
+
+    if not mgr.image_exists(repo_tag):
+        logger.info("Building missing repo image: %s", repo_tag)
+        mgr.build_repo_image(owner, repo, py_version=py_version)
+
+
+def _build_and_push_pr_image(
+    owner: str,
+    repo: str,
+    issue_number: int,
+    sha: str,
+    env_payload: str,
+    docker_context: Any | None = None,
+    python_version: str = "",
+) -> str:
+    """Build the final PR image from synthesized context and push to DockerHub.
+
+    The three-tier Dockerfiles (base/repo/pr) use the same shell scripts as
+    the synthesized single-Dockerfile flow, so the build result is identical.
+
+    Returns the pushed image tag.
+    """
+    from datasmith.docker.images import ImageManager, get_pr_image_name, get_repo_image_name
+    from datasmith.docker.publish import DockerHubPublisher
+
+    ctx = docker_context
+    mgr = ImageManager()
+    pr_tag = get_pr_image_name(owner, repo, issue_number)
+
+    if ctx is not None:
+        with tempfile.TemporaryDirectory(prefix="docker-ctx-") as tmpdir:
+            ctx.to_directory(tmpdir)
+            _fill_missing_scripts(tmpdir, base_commit=sha)
+            mgr.build_pr_image(
+                owner,
+                repo,
+                issue_number,
+                context=tmpdir,
+                commit_sha=sha or "HEAD",
+                env_payload=env_payload or "[]",
+                py_version=python_version,
+            )
+    else:
+        mgr.build_pr_image(
+            owner,
+            repo,
+            issue_number,
+            commit_sha=sha or "HEAD",
+            env_payload=env_payload or "[]",
+            py_version=python_version,
+        )
+
+    publisher = DockerHubPublisher()
+    repo_tag = get_repo_image_name(owner, repo)
+
+    try:
+        publisher.push(repo_tag)
+    except Exception:
+        logger.warning("Failed to push repo image %s (non-fatal)", repo_tag)
+
+    publisher.push(pr_tag)
+    logger.info("Pushed PR image: %s", pr_tag)
+    return pr_tag
+
+
+def _render_run_tests_sh(docker_templates: Any, base_commit: str) -> str:
+    """Render the run-tests.sh Jinja2 template with embedded scripts."""
+    from pathlib import Path
+
+    from jinja2 import Environment, FileSystemLoader
+
+    docker_templates = Path(docker_templates)
+    env = Environment(
+        loader=FileSystemLoader(str(docker_templates)),
+        keep_trailing_newline=True,
+        autoescape=False,
+    )
+    template = env.get_template("run-tests.sh")
+
+    pytest_runner = (docker_templates / "pytest_runner.py").read_text()
+    parser = (docker_templates / "parser.py").read_text()
+
+    return template.render(
+        base_commit=base_commit,
+        pytest_runner=pytest_runner,
+        parser=parser,
+        run_pytest=True,
+    )
+
+
+def _fill_missing_scripts(context_dir: str, base_commit: str = "") -> None:
+    """Copy any missing shell scripts and Dockerfile.pr from the templates directory.
+
+    Synthesized contexts may only contain a subset of the 9 expected files
+    (e.g. only ``build_pkg_sh``).  The Dockerfile.pr ``COPY`` directives
+    require every file to be present, so we backfill from the built-in
+    templates for anything the synthesizer didn't produce.
+
+    ``run-tests.sh`` is a Jinja2 template that requires rendering with
+    ``base_commit`` and embedded Python scripts before it can be used.
+    """
+    import os
+    import shutil
+    from pathlib import Path
+
+    templates = Path(__file__).parents[1] / "docker" / "templates"
+
+    # Every file that Dockerfile.pr references via COPY
+    required = [
+        "Dockerfile.pr",
+        "docker_build_env.sh",
+        "docker_build_pkg.sh",
+        "docker_build_run.sh",
+        "docker_build_final.sh",
+        "profile.sh",
+        "run-tests.sh",
+        "entrypoint.sh",
+    ]
+
+    for fname in required:
+        target = os.path.join(context_dir, fname)
+        if os.path.exists(target):
+            continue
+        if fname == "run-tests.sh":
+            # run-tests.sh is a Jinja2 template — render it instead of copying raw
+            rendered = _render_run_tests_sh(templates, base_commit=base_commit)
+            with open(target, "w") as f:
+                f.write(rendered)
+        else:
+            src = templates / fname
+            if src.exists():
+                shutil.copy2(str(src), target)
+
+
+# Lock to serialize prerequisite image builds (base + repo) across threads.
+# Building these is expensive and they're shared, so we avoid duplicate work.
+_prereq_lock = threading.Lock()
+# Track repos whose prerequisite images are confirmed present.
+_prereq_done: set[tuple[str, str]] = set()
+
+
+class SynthesizeImagesRunner(BaseRunner):
+    """Run Synthesizer for each PR to produce Docker build contexts."""
+
+    def __init__(
+        self,
+        synthesizer: Synthesizer,
+        gh: Any | None = None,
+        n_concurrent: int = 3,
+    ) -> None:
+        super().__init__(name="synthesize_images", n_concurrent=n_concurrent)
+        self._synthesizer = synthesizer
+        self._gh = gh  # GitHubClient, optional — needed for rendering problem statements
+
+    async def _render_problem(self, item: dict[str, Any]) -> str | None:
+        """Render the problem statement for a PR, scraping linked issues.
+
+        Returns the rendered markdown, or ``None`` if rendering is skipped
+        (no GitHubClient) or fails.
+        """
+        if self._gh is None:
+            return None
+
+        owner: str = item["owner"]
+        repo: str = item["repo"]
+        issue_number: int = item["issue_number"]
+
+        from datasmith.github.links import scrape_links
+        from datasmith.github.models import PR
+        from datasmith.github.render import render_problem_statement
+
+        # Build a PR object for scrape_links and render_problem_statement
+        pr = PR(
+            repository=f"{owner}/{repo}",
+            issue_number=issue_number,
+            title=item.get("title", ""),
+            body=item.get("body", ""),
+            created_at=item.get("created_at"),
+        )
+
+        # BFS-scrape linked issues (async GitHub API calls)
+        issues = await scrape_links(
+            pr,
+            self._gh.get_issue_expanded,
+            depth=2,
+            only_issues=True,
+            limit=6,
+        )
+
+        logger.info(
+            "Scraped %d linked issues for %s/%s#%d",
+            len(issues),
+            owner,
+            repo,
+            issue_number,
+        )
+
+        # Render the problem statement (may invoke ProblemExtractor LLM — run in thread)
+        repo_description: str = item.get("repo_description", "")
+        rendered = await asyncio.to_thread(
+            render_problem_statement,
+            pr,
+            issues=issues,
+            repo_description=repo_description,
+            anonymize=True,
+            extract=True,
+        )
+
+        # Persist to DB
+        client = get_client()
+        client.table("pull_requests").update({"rendered_problem": rendered}).eq("owner", owner).eq("repo", repo).eq(
+            "issue_number", issue_number
+        ).execute()
+
+        logger.info("Rendered problem statement for %s/%s#%d", owner, repo, issue_number)
+        return rendered
+
+    async def _process_item(self, item: Any) -> None:
+        """Process a PR dict with owner, repo, issue_number, pr_context."""
+        owner = item["owner"]
+        repo = item["repo"]
+        issue_number = item["issue_number"]
+        pr_context = item.get("pr_context", "")
+        py_version = item.get("python_version", "")
+
+        # Ensure base and repo images exist before synthesis needs them
+        await asyncio.to_thread(self._ensure_prereqs, owner, repo, py_version)
+
+        # Render the problem statement before synthesis (skip if already rendered)
+        if not pr_context:
+            rendered = await self._render_problem(item)
+            if rendered:
+                pr_context = rendered
+
+        sha = item.get("sha", "")
+        env_payload = item.get("env_payload", "")
+
+        from datasmith.docker.images import get_repo_image_name
+
+        repo_image = get_repo_image_name(owner, repo)
+
+        # Run synthesizer in thread (Docker operations are blocking)
+        ctx = await asyncio.to_thread(
+            self._synthesizer.run,
+            owner,
+            repo,
+            issue_number,
+            pr_context,
+            sha,
+            repo_image=repo_image,
+            env_payload=env_payload,
+            python_version=py_version,
+        )
+
+        if ctx is None:
+            raise RuntimeError(f"Synthesis failed for {owner}/{repo}#{issue_number}")
+
+        logger.info("Successfully synthesized image for %s/%s#%d", owner, repo, issue_number)
+
+        # Build the final PR image and push to DockerHub
+        pr_tag = await asyncio.to_thread(
+            _build_and_push_pr_image, owner, repo, issue_number, sha, env_payload, ctx, py_version
+        )
+
+        # Record the container name in Supabase
+        client = get_client()
+        client.table("pull_requests").update({"container_name": pr_tag}).eq("owner", owner).eq("repo", repo).eq(
+            "issue_number", issue_number
+        ).execute()
+
+    @staticmethod
+    def _ensure_prereqs(owner: str, repo: str, py_version: str) -> None:
+        """Build base/repo images if missing, with dedup across threads."""
+        key = (owner, repo)
+        if key in _prereq_done:
+            return
+        with _prereq_lock:
+            if key in _prereq_done:
+                return
+            _ensure_prerequisite_images(owner, repo, py_version)
+            _prereq_done.add(key)