fc-data 0.2.0__py3-none-any.whl

datasmith/agents/synthesizer.py

@@ -0,0 +1,439 @@
+from __future__ import annotations
+
+import datetime
+import enum
+import json
+from typing import Any, cast
+
+from datasmith.agents.sandbox import SandboxResult, verify_context
+from datasmith.docker.context import DockerContext
+from datasmith.utils import get_client, get_logger
+
+logger = get_logger("agents.synthesizer")
+
+
+class SynthesisState(str, enum.Enum):
+    CHECK_CACHE = "check_cache"
+    FIND_SIMILAR = "find_similar"
+    TRY_SIMILAR = "try_similar"
+    LLM_GENERATE = "llm_generate"
+    FAIL = "fail"
+
+
+class Synthesizer:
+    """State machine for synthesizing Docker build contexts."""
+
+    def __init__(
+        self,
+        max_attempts: int = 2,
+        dry_run: bool = False,
+        agent: str | None = None,
+        force: bool = False,
+    ) -> None:
+        self._max_attempts = max_attempts
+        self._dry_run = dry_run
+        self._agent = agent
+        self._force = force
+        self._trace: list[SynthesisState] = []
+
+    @property
+    def trace(self) -> list[SynthesisState]:
+        return list(self._trace)
+
+    def run(
+        self,
+        owner: str,
+        repo: str,
+        issue_number: int,
+        pr_context: str,
+        sha: str = "",
+        repo_image: str = "",
+        env_payload: str = "",
+        python_version: str = "",
+        force: bool = False,
+    ) -> DockerContext | None:
+        """Run the synthesis state machine. Returns DockerContext on success, None on failure."""
+        self._trace = []
+        force = force or self._force
+
+        # State: CHECK_CACHE
+        self._trace.append(SynthesisState.CHECK_CACHE)
+        cached = self._check_cache(owner, repo, sha)
+        if (not force) and (cached is not None):
+            logger.info("Cache hit for %s/%s@%s", owner, repo, sha[:12] if sha else "?")
+            return cached
+
+        # State: FIND_SIMILAR
+        self._trace.append(SynthesisState.FIND_SIMILAR)
+        similar_contexts = self._find_similar(owner, repo, issue_number)
+
+        # State: TRY_SIMILAR
+        failed_attempts: list[tuple[DockerContext, SandboxResult]] = []
+        if similar_contexts:
+            self._trace.append(SynthesisState.TRY_SIMILAR)
+            for ctx in similar_contexts:
+                result = verify_context(
+                    owner=owner,
+                    repo=repo,
+                    sha=sha,
+                    repo_image=repo_image,
+                    env_payload=env_payload,
+                    python_version=python_version,
+                    context=ctx,
+                )
+                if result.success:
+                    logger.info("Similar context passed for %s/%s#%d", owner, repo, issue_number)
+                    self._save_context(
+                        owner,
+                        repo,
+                        sha,
+                        issue_number,
+                        ctx,
+                        resource_metrics=result.resource_metrics,
+                    )
+                    return ctx
+                failed_attempts.append((ctx, result))
+
+        # State: LLM_GENERATE (sandbox-based)
+        # Skip LLM generation when using the "none" agent — rely only on similar contexts.
+        if self._agent == "none":
+            self._trace.append(SynthesisState.FAIL)
+            logger.info(
+                "Agent is 'none' — skipping LLM generation for %s/%s#%d",
+                owner,
+                repo,
+                issue_number,
+            )
+            return None
+
+        self._trace.append(SynthesisState.LLM_GENERATE)
+        prior_attempts = _format_prior_attempts(failed_attempts) if failed_attempts else ""
+        for attempt_idx in range(self._max_attempts):
+            generated, metrics = self._sandbox_generate(
+                owner=owner,
+                repo=repo,
+                sha=sha,
+                pr_context=pr_context,
+                repo_image=repo_image,
+                env_payload=env_payload,
+                python_version=python_version,
+                prior_attempts=prior_attempts,
+                issue_number=issue_number,
+                attempt_index=attempt_idx,
+            )
+            if generated is not None:
+                logger.info(
+                    "Sandbox synthesis succeeded for %s/%s#%d (attempt %d)",
+                    owner,
+                    repo,
+                    issue_number,
+                    attempt_idx + 1,
+                )
+                self._save_context(owner, repo, sha, issue_number, generated, resource_metrics=metrics)
+                return generated
+            logger.warning(
+                "Sandbox synthesis attempt %d failed for %s/%s#%d",
+                attempt_idx + 1,
+                owner,
+                repo,
+                issue_number,
+            )
+
+        # State: FAIL
+        self._trace.append(SynthesisState.FAIL)
+        logger.warning("All synthesis attempts failed for %s/%s#%d", owner, repo, issue_number)
+        return None
+
+    def _check_cache(self, owner: str, repo: str, sha: str) -> DockerContext | None:
+        if not sha:
+            return None
+        try:
+            client = get_client()
+            resp = (
+                client.table("candidate_containers")
+                .select("*")
+                .eq("owner", owner)
+                .eq("repo", repo)
+                .eq("sha", sha)
+                .execute()
+            )
+            if resp.data:
+                row = cast(dict[str, Any], resp.data[0])
+                return DockerContext(
+                    dockerfile=row.get("dockerfile", ""),
+                    build_base_sh=row.get("build_base_sh", ""),
+                    build_env_sh=row.get("build_env_sh", ""),
+                    build_pkg_sh=row.get("build_pkg_sh", ""),
+                    build_run_sh=row.get("build_run_sh", ""),
+                    build_final_sh=row.get("build_final_sh", ""),
+                    profile_sh=row.get("profile_sh", ""),
+                    run_tests_sh=row.get("run_tests_sh", ""),
+                    entrypoint_sh=row.get("entrypoint_sh", ""),
+                )
+        except Exception:
+            logger.debug("Cache check failed, proceeding")
+        return None
+
+    def _find_similar(self, owner: str, repo: str, issue_number: int) -> list[DockerContext]:
+        """Find previously successful build contexts for the same repository.
+
+        Results are ordered by chronological proximity to the given PR so that
+        the most temporally adjacent contexts — most likely to share the same
+        dependency environment — are tried first.
+        """
+        try:
+            client = get_client()
+
+            # Step 1: look up the current PR's creation date.
+            pr_resp = (
+                client.table("pull_requests")
+                .select("created_at")
+                .eq("owner", owner)
+                .eq("repo", repo)
+                .eq("issue_number", issue_number)
+                .execute()
+            )
+            current_date: datetime.datetime | None = None
+            if pr_resp.data:
+                raw = cast(dict[str, Any], pr_resp.data[0]).get("created_at")
+                if raw:
+                    current_date = _parse_ts(raw)
+
+            # Step 2: fetch all non-empty contexts for this repo.
+            ctx_resp = (
+                client.table("candidate_containers")
+                .select("issue_number,build_pkg_sh,build_run_sh")
+                .eq("owner", owner)
+                .eq("repo", repo)
+                .execute()
+            )
+            rows = cast(list[dict[str, Any]], ctx_resp.data)
+            rows = [r for r in rows if r.get("build_pkg_sh")]
+
+            if not rows:
+                return []
+
+            # Step 3: if we have a reference date, sort by proximity.
+            if current_date is not None:
+                context_issue_numbers = [r["issue_number"] for r in rows if r.get("issue_number") is not None]
+                pr_dates: dict[int, datetime.datetime] = {}
+                if context_issue_numbers:
+                    dates_resp = (
+                        client.table("pull_requests")
+                        .select("issue_number,created_at")
+                        .eq("owner", owner)
+                        .eq("repo", repo)
+                        .in_("issue_number", context_issue_numbers)
+                        .execute()
+                    )
+                    for p in cast(list[dict[str, Any]], dates_resp.data):
+                        iss = p.get("issue_number")
+                        raw_date = p.get("created_at")
+                        if iss is not None and raw_date:
+                            pr_dates[iss] = _parse_ts(raw_date)
+
+                _sentinel = datetime.timedelta.max
+
+                def _proximity(row: dict[str, Any]) -> datetime.timedelta:
+                    iss = row.get("issue_number")
+                    d = pr_dates.get(iss) if iss is not None else None
+                    return abs(d - current_date) if d is not None else _sentinel
+
+                rows.sort(key=_proximity)
+
+            rows = rows[:5]
+            return [
+                DockerContext(
+                    build_pkg_sh=r.get("build_pkg_sh", ""),
+                    build_run_sh=r.get("build_run_sh", ""),
+                )
+                for r in rows
+            ]
+        except Exception:
+            logger.debug("Similar context lookup failed")
+            return []
+
+    def _sandbox_generate(
+        self,
+        owner: str,
+        repo: str,
+        sha: str,
+        pr_context: str,
+        repo_image: str,
+        env_payload: str,
+        python_version: str,
+        prior_attempts: str = "",
+        issue_number: int = 0,
+        attempt_index: int = 0,
+    ) -> tuple[DockerContext | None, dict]:
+        from datasmith.agents.sandbox import SandboxRunner
+
+        runner = SandboxRunner(agent=self._agent)
+        result = runner.run(
+            owner=owner,
+            repo=repo,
+            sha=sha,
+            repo_image=repo_image,
+            env_payload=env_payload,
+            python_version=python_version,
+            pr_context=pr_context,
+            prior_attempts=prior_attempts,
+            dry_run=self._dry_run,
+        )
+        self._log_attempt(
+            owner=owner,
+            repo=repo,
+            sha=sha,
+            issue_number=issue_number,
+            attempt_index=attempt_index,
+            result=result,
+        )
+        ctx = result.docker_context if result.success else None
+        return ctx, result.resource_metrics
+
+    def _log_attempt(
+        self,
+        owner: str,
+        repo: str,
+        sha: str,
+        issue_number: int,
+        attempt_index: int,
+        result: SandboxResult,
+    ) -> None:
+        """Persist agent output to the ``error_logs`` Supabase table."""
+        timestamp = datetime.datetime.now(tz=datetime.timezone.utc).isoformat()
+
+        failure = result.failure_json or {}
+        # Cap raw output at 100 KB for Supabase storage
+        raw_output = result.raw_agent_output
+        if len(raw_output) > 100_000:
+            raw_output = raw_output[-100_000:]
+
+        row = {
+            "owner": owner,
+            "repo": repo,
+            "sha": sha,
+            "issue_number": issue_number,
+            "attempt_index": attempt_index,
+            "agent_name": result.agent_name,
+            "success": result.success,
+            "duration_s": result.duration_s,
+            "failure_stage": failure.get("stage") or None,
+            "failure_return_code": failure.get("return_code") or None,
+            "error_message": (failure.get("error_message") or "")[-10_000:] or None,
+            "agent_output": raw_output or None,
+            "files_changed": json.dumps(result.files_changed),
+            "resource_metrics": result.resource_metrics or None,
+            "created_at": timestamp,
+        }
+        try:
+            client = get_client()
+            client.table("error_logs").insert(row).execute()
+            logger.info(
+                "Logged synthesis attempt to error_logs for %s/%s@%s attempt %d", owner, repo, sha[:12], attempt_index
+            )
+        except Exception:
+            logger.debug("Failed to log synthesis attempt to Supabase", exc_info=True)
+
+    def _save_context(
+        self,
+        owner: str,
+        repo: str,
+        sha: str,
+        issue_number: int,
+        ctx: DockerContext,
+        resource_metrics: dict | None = None,
+    ) -> None:
+        """Persist the agent-edited scripts to the ``candidate_containers`` table.
+
+        Only ``build_pkg_sh`` and ``build_run_sh`` are saved — the other
+        fields come from templates and don't need to be persisted.
+        """
+        if not sha:
+            return
+        try:
+            client = get_client()
+            row: dict = {
+                "owner": owner,
+                "repo": repo,
+                "sha": sha,
+                "issue_number": issue_number,
+                "build_pkg_sh": ctx.build_pkg_sh,
+                "build_run_sh": ctx.build_run_sh,
+            }
+            if resource_metrics:
+                row["resource_metrics"] = resource_metrics
+            client.table("candidate_containers").upsert(row).execute()
+            logger.info("Saved context for %s/%s@%s", owner, repo, sha[:12])
+        except Exception:
+            logger.warning("Failed to save context for %s/%s@%s", owner, repo, sha[:12])
+
+
+def _parse_ts(ts: str) -> datetime.datetime:
+    """Parse an ISO-8601 timestamp string to a timezone-aware datetime.
+
+    Handles both ``Z`` and ``+HH:MM`` UTC offset suffixes, which is
+    necessary for Python 3.9/3.10 compatibility where ``fromisoformat``
+    does not accept the trailing ``Z``.
+    """
+    return datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
+
+
+def _format_prior_attempts(attempts: list[tuple[DockerContext, SandboxResult]]) -> str:
+    """Format failed TRY_SIMILAR attempts into context for the LLM agent."""
+    lines = [
+        "# Prior Attempts",
+        "",
+        "The following build contexts were tried and failed.",
+        "Use these failures to inform your approach — avoid repeating the same mistakes.",
+        "",
+    ]
+    for i, (ctx, result) in enumerate(attempts, 1):
+        failure = result.failure_json or {}
+        stage = failure.get("stage", "unknown")
+        rc = failure.get("return_code", 1)
+        error = failure.get("error_message", "")
+
+        lines.append(f"## Attempt {i}")
+        lines.append("")
+        lines.append(f"**Stage**: {stage}")
+        lines.append(f"**Return code**: {rc}")
+        lines.append("")
+
+        lines.append("### docker_build_pkg.sh")
+        lines.append("```bash")
+        pkg = ctx.build_pkg_sh
+        if pkg and len(pkg) > 3000:
+            lines.append(pkg[:3000])
+            lines.append("# ... (truncated)")
+        else:
+            lines.append(pkg or "(empty)")
+        lines.append("```")
+        lines.append("")
+
+        lines.append("### docker_build_run.sh")
+        lines.append("```bash")
+        run = ctx.build_run_sh
+        if run and len(run) > 3000:
+            lines.append(run[:3000])
+            lines.append("# ... (truncated)")
+        else:
+            lines.append(run or "(empty)")
+        lines.append("```")
+        lines.append("")
+
+        if error:
+            lines.append("### Error output")
+            lines.append("```")
+            lines.append(error[-3000:])
+            lines.append("```")
+            lines.append("")
+
+        if result.agent_output:
+            stdout_tail = result.agent_output[-3000:]
+            lines.append("### Build output (last 3000 chars)")
+            lines.append("```")
+            lines.append(stdout_tail)
+            lines.append("```")
+            lines.append("")
+    return "\n".join(lines)

datasmith/agents/templates/AGENTS.md.j2

@@ -0,0 +1,150 @@
+# FormulaCode Docker Build Verification Guide
+
+You are fixing a Docker build context for the FormulaCode dataset. Your goal is to iteratively
+fix the build until `sandbox_verify.py` succeeds.
+
+## Task Info
+
+- **Repository**: {{ owner }}/{{ repo }}
+- **Commit SHA**: {{ sha }}
+- **Python version**: {{ python_version }}
+
+## Quick Start
+
+```bash
+python3 sandbox_verify.py
+```
+
+This runs the full verification pipeline: build → tests.
+On failure it writes `task/failure.json`. On success it writes `task/verification_success.json`.
+Both stages (build, tests) must pass for verification to succeed.
+
+## PR Context
+
+{{ pr_context }}
+
+## Prior Attempts
+
+If `prior_attempts.md` exists in this workspace, it contains build scripts that were already
+tried and failed. **Read it first** to understand what went wrong and avoid repeating the same
+mistakes.
+
+## Workflow
+
+Follow this iterative cycle:
+
+1. Run `python3 sandbox_verify.py` (A good timeout is 60 minutes, but adjust as needed)
+2. If it fails, read `task/failure.json` for error details
+3. Edit `task/docker_build_pkg.sh` and/or `task/docker_build_run.sh`
+4. Re-run `python3 sandbox_verify.py`
+5. Repeat until `task/verification_success.json` is created
+
+**IMPORTANT**: Only modify `task/docker_build_pkg.sh` and `task/docker_build_run.sh`.
+Do **not** modify the Dockerfile, `task.txt`, or any other scripts.
+
+## File Constraints
+
+### Files you CAN edit
+- `task/docker_build_pkg.sh` — Primary build script. Install the package, add build deps.
+- `task/docker_build_run.sh` — Runtime setup. Add test/benchmark deps, repo-specific config.
+
+### Files you MUST NOT edit
+- `task/Dockerfile.pr` — Multi-stage build definition (base and repo stages are pre-built)
+- `task/task.txt` — Task configuration (owner, repo, sha, deps)
+- `task/docker_build_base.sh` — Base system setup (template)
+- `task/docker_build_env.sh` — Python environment creation (template)
+- `task/docker_build_final.sh` — Final image setup (template)
+- `task/profile.sh` — ASV benchmark runner
+- `task/run-tests.sh` — Test runner
+- `task/entrypoint.sh` — Container entrypoint
+
+## Understanding failure.json
+
+When verification fails, `task/failure.json` contains:
+
+```json
+{
+  "stage": "build|tests",
+  "return_code": 1,
+  "stderr": "Full error output from the failed stage",
+  "stdout": "Full standard output from the failed stage",
+  "error_message": "Human-readable error description"
+}
+```
+
+- **build** failures: Usually missing dependencies or compilation errors → fix in `docker_build_pkg.sh`
+- **tests** failures: Missing test deps, import errors, or benchmark setup issues → fix in `docker_build_run.sh` or `docker_build_pkg.sh`
+
+## Common Fixes for docker_build_pkg.sh
+
+### Add build dependency (conda)
+```bash
+micromamba install -y -n "$ENV_NAME" -c conda-forge some-build-tool
+```
+
+### Add build dependency (pip)
+```bash
+micromamba run -n "$ENV_NAME" pip install build-dependency
+```
+
+### Set compiler flags
+```bash
+export CFLAGS="${CFLAGS:--Wno-error=incompatible-pointer-types}"
+export CXXFLAGS="${CXXFLAGS:--Wno-error}"
+```
+
+### Initialize git submodules
+```bash
+git -C "$REPO_ROOT" submodule update --init --recursive
+```
+
+### Build Cython extensions
+```bash
+micromamba install -y -n "$ENV_NAME" -c conda-forge cython
+micromamba run -n "$ENV_NAME" python setup.py build_ext --inplace
+```
+
+### Patch source files
+```bash
+sed -i 's/problematic_code/fixed_code/' path/to/file.py
+```
+
+## Common Fixes for docker_build_run.sh
+
+### Add runtime/test dependencies
+```bash
+source /etc/profile.d/asv_utils.sh
+source /etc/profile.d/asv_build_vars.sh
+micromamba run -n "$ENV_NAME" pip install pytest-xdist pytest-timeout
+```
+
+## Environment Variables Available
+
+These are set by earlier build stages and available in both scripts:
+
+- `REPO_ROOT` — Repository root path (usually `/workspace/repo`)
+- `ENV_NAME` — Current environment name (e.g., `asv_3.10`)
+- `ASV_PY_VERSIONS` — Space-separated Python versions
+- `CONF_NAME` — Path to ASV config file
+- `ALL_EXTRAS` — Extras to install (if any)
+- `MAMBA_ROOT_PREFIX` — Micromamba root (`/opt/conda`)
+
+## Docker Build Stages
+
+The Dockerfile.pr has 4 stages: env → pkg → run → final (base and repo images are pre-built).
+Only the `pkg` and `run` stages use your editable scripts.
+
+- **pkg** stage: Runs `docker_build_pkg.sh` to install the package
+- **run** stage: Runs `docker_build_run.sh` to prepare for tests/benchmarks
+
+Changes to `docker_build_pkg.sh` rebuild from the pkg stage onward (fast iteration).
+Changes to `docker_build_run.sh` rebuild only the run stage.
+
+## Tips
+
+- Read the full error in `failure.json` before making changes
+- Start with the simplest fix (add a missing dep) before trying complex patches
+- The package install command in `docker_build_pkg.sh` is usually `pip install --no-build-isolation -v -e .`
+- If the build times out, look for ways to simplify or skip expensive steps
+- Docker layer caching means earlier stages are cached — only your changed stage rebuilds
+- A lot of bookkeeping code is used in downstream stages; avoid removing code unless you understand its purpose.