fc-data 0.2.0__py3-none-any.whl

datasmith/agents/templates/sandbox_verify.py

@@ -0,0 +1,428 @@
+#!/usr/bin/env python3
+"""Simplified verification script for sandbox-based synthesis.
+
+Builds a Docker image from the task directory, then runs profile.sh and
+run-tests.sh inside the container. Writes failure.json or
+verification_success.json to the task directory.
+
+This file is self-contained — no datasmith imports. It is copied into the
+Codex sandbox workspace and executed by the agent.
+
+Usage:
+    python sandbox_verify.py                # verify task/ directory
+    python sandbox_verify.py --task /path    # verify custom task directory
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import subprocess as sp
+import sys
+import threading
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+from python_on_whales import DockerClient
+from python_on_whales.exceptions import DockerException
+
+# Files the agent must NOT modify.  Hashes are checked before every build.
+_IMMUTABLE_FILES = (
+    "Dockerfile.pr",
+    "docker_build_base.sh",
+    "docker_build_env.sh",
+    "docker_build_final.sh",
+    "profile.sh",
+    "run-tests.sh",
+    "entrypoint.sh",
+    "task.txt",
+)
+
+
+@dataclass(frozen=True)
+class Task:
+    """Mirrors the Task dataclass used in task.txt files."""
+
+    owner: str = ""
+    repo: str = ""
+    sha: str | None = None
+    commit_date: float = 0.0
+    env_payload: str = ""
+    python_version: str = ""
+    tag: str = "pkg"
+    benchmarks: str = ""
+    repo_image: str = ""
+
+
+def _parse_task(task_dir: Path) -> Task:
+    text = (task_dir / "task.txt").read_text().strip()
+    return eval(text, {"__builtins__": {}}, {"Task": Task})  # noqa: S307
+
+
+def _image_tag(task: Task, stage: str = "run") -> str:
+    sha_short = (task.sha or "unknown")[:12]
+    owner_repo = f"{task.owner}-{task.repo}".lower()
+    return f"formulacode/{owner_repo}:{sha_short}-{stage}"
+
+
+def _write_failure(
+    task_dir: Path,
+    stage: str,
+    *,
+    stdout: str = "",
+    stderr: str = "",
+    rc: int = 1,
+    metrics: dict | None = None,
+) -> None:
+    # Truncate output to keep failure.json readable by CLI agents.
+    # The Read tool rejects files larger than ~25K tokens, causing cascading
+    # failures when the agent tries to read failure.json alongside other files.
+    _MAX_OUTPUT = 20_000  # chars — keeps total file well under the token limit
+    failure: dict = {
+        "stage": stage,
+        "return_code": rc,
+        "error_message": f"Verification failed during '{stage}' stage (rc={rc}).",
+        "stdout": stdout[-_MAX_OUTPUT:] if len(stdout) > _MAX_OUTPUT else stdout,
+        "stderr": stderr[-_MAX_OUTPUT:] if len(stderr) > _MAX_OUTPUT else stderr,
+    }
+    if metrics:
+        failure["resource_metrics"] = metrics
+    (task_dir / "failure.json").write_text(json.dumps(failure, indent=2))
+    # Also remove stale success file if present
+    success = task_dir / "verification_success.json"
+    if success.exists():
+        success.unlink()
+
+
+def _write_success(task_dir: Path, image_tag: str, metrics: dict | None = None) -> None:
+    info: dict = {"local_image": image_tag, "verified_at": time.time()}
+    if metrics:
+        info["resource_metrics"] = metrics
+    (task_dir / "verification_success.json").write_text(json.dumps(info, indent=2))
+    # Remove stale failure file if present
+    failure = task_dir / "failure.json"
+    if failure.exists():
+        failure.unlink()
+
+
+def build_image(
+    docker: DockerClient,
+    task_dir: Path,
+    task: Task,
+    target: str = "run",
+    metrics: dict | None = None,
+) -> str:
+    tag = _image_tag(task, target)
+
+    build_args = {
+        "REPO_IMAGE": task.repo_image,
+        "COMMIT_SHA": task.sha or "",
+        "ENV_PAYLOAD": task.env_payload,
+    }
+    if task.python_version:
+        build_args["PY_VERSION"] = task.python_version
+    if task.benchmarks:
+        build_args["BENCHMARKS"] = task.benchmarks
+
+    print(f"Building {tag} (target={target}) from {task_dir}")
+    log_lines: list[str] = []
+    start = time.time()
+    try:
+        for line in docker.build(
+            str(task_dir),
+            tags=[tag],
+            target=target,
+            build_args=build_args,
+            file=str(task_dir / "Dockerfile.pr"),
+            stream_logs=True,
+        ):
+            log_lines.append(line)
+    except DockerException as e:
+        stdout = "".join(log_lines)
+        if metrics is not None:
+            metrics["build_duration_s"] = round(time.time() - start, 2)
+        raise BuildError(
+            f"Docker build failed (rc={e.return_code})",
+            stdout=stdout,
+            stderr=e.stderr or "",
+            rc=e.return_code,
+        ) from e
+
+    if metrics is not None:
+        metrics["build_duration_s"] = round(time.time() - start, 2)
+        try:
+            img = docker.image.inspect(tag)
+            metrics["image_size_bytes"] = img.size
+        except Exception:
+            pass
+
+    return tag
+
+
+class BuildError(Exception):
+    """Raised when a Docker build fails, carrying captured output."""
+
+    def __init__(self, message: str, stdout: str, stderr: str, rc: int) -> None:
+        super().__init__(message)
+        self.stdout = stdout
+        self.stderr = stderr
+        self.rc = rc
+
+
+_MEM_UNITS = {"B": 1, "KIB": 1024, "MIB": 1024**2, "GIB": 1024**3, "TIB": 1024**4}
+_MEM_RE = re.compile(r"([\d.]+)\s*((?:[KMGT]i)?B)", re.IGNORECASE)
+
+
+def _parse_mem_usage(text: str) -> int:
+    """Parse a Docker memory string like ``'123.4MiB / 16GiB'`` → bytes (first value only)."""
+    m = _MEM_RE.search(text)
+    if not m:
+        return 0
+    value, unit = float(m.group(1)), m.group(2).upper()
+    return int(value * _MEM_UNITS.get(unit, 1))
+
+
+def _run_container_with_timeout(
+    image_tag: str,
+    command: list[str],
+    timeout: int,
+    metrics: dict | None = None,
+) -> tuple[bool, str, str, int]:
+    """Run a Docker container with a host-side timeout.
+
+    Returns (timed_out, stdout, stderr, returncode).
+
+    When *metrics* is provided, records ``test_duration_s`` and
+    ``peak_memory_bytes`` by polling ``docker stats`` in a background thread.
+    """
+    name = f"fc-{uuid.uuid4().hex[:8]}"
+    cmd = ["docker", "run", "--name", name, "--pull", "never", image_tag, *command]
+    # No --rm: we clean up manually after collecting metrics.
+
+    peak_mem: list[int] = [0]
+    stop_event = threading.Event()
+
+    def _poll_stats() -> None:
+        """Background thread: poll ``docker stats`` every 2 s to track peak memory."""
+        while not stop_event.is_set():
+            try:
+                r = sp.run(  # noqa: S603, S607
+                    ["docker", "stats", name, "--no-stream", "--format", "{{.MemUsage}}"],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+                if r.returncode == 0 and r.stdout.strip():
+                    current = _parse_mem_usage(r.stdout.strip())
+                    if current > peak_mem[0]:
+                        peak_mem[0] = current
+            except Exception:
+                pass
+            stop_event.wait(2.0)
+
+    poller = threading.Thread(target=_poll_stats, daemon=True)
+    poller.start()
+
+    start = time.time()
+    try:
+        result = sp.run(cmd, capture_output=True, text=True, timeout=timeout)  # noqa: S603, S607
+        timed_out = False
+        stdout, stderr, rc = result.stdout, result.stderr, result.returncode
+    except (sp.TimeoutExpired, KeyboardInterrupt) as exc:
+        stdout = getattr(exc, "stdout", None) or ""
+        stderr = getattr(exc, "stderr", None) or ""
+        if isinstance(stdout, bytes):
+            stdout = stdout.decode(errors="replace")
+        if isinstance(stderr, bytes):
+            stderr = stderr.decode(errors="replace")
+        sp.run(["docker", "kill", name], capture_output=True)  # noqa: S603, S607
+        if isinstance(exc, KeyboardInterrupt):
+            stop_event.set()
+            poller.join(timeout=5)
+            sp.run(["docker", "rm", "-f", name], capture_output=True)  # noqa: S603, S607
+            raise
+        timed_out, rc = True, -1
+    finally:
+        stop_event.set()
+        poller.join(timeout=5)
+
+    wall_time = time.time() - start
+
+    if metrics is not None:
+        metrics["test_duration_s"] = round(wall_time, 2)
+        if peak_mem[0] > 0:
+            metrics["peak_memory_bytes"] = peak_mem[0]
+
+    # Cleanup container
+    sp.run(["docker", "rm", "-f", name], capture_output=True)  # noqa: S603, S607
+    return timed_out, stdout, stderr, rc
+
+
+def _parse_test_summary(stdout: str) -> dict | None:
+    """Extract the JSON summary between FORMULACODE_TESTS_START/END markers."""
+    start = "FORMULACODE_TESTS_START"
+    end = "FORMULACODE_TESTS_END"
+    s = stdout.find(start)
+    e = stdout.find(end)
+    if s == -1 or e == -1:
+        return None
+    payload = stdout[s + len(start) : e].strip()
+    try:
+        return json.loads(payload)  # type: ignore[no-any-return]
+    except json.JSONDecodeError:
+        return None
+
+
+def _parse_snapshot_summary(stdout: str) -> dict | None:
+    """Extract the JSON summary between FORMULACODE_SNAPSHOT_START/END markers."""
+    start = "FORMULACODE_SNAPSHOT_START"
+    end = "FORMULACODE_SNAPSHOT_END"
+    s = stdout.find(start)
+    e = stdout.find(end)
+    if s == -1 or e == -1:
+        return None
+    payload = stdout[s + len(start) : e].strip()
+    try:
+        return json.loads(payload)
+    except json.JSONDecodeError:
+        return None
+
+
+def run_tests(image_tag: str, timeout: int = 720, metrics: dict | None = None) -> tuple[bool, str, str, int]:
+    timed_out, stdout, stderr, rc = _run_container_with_timeout(
+        image_tag, ["/run-tests.sh", "--all"], timeout, metrics=metrics
+    )
+    if timed_out:
+        return True, stdout, f"Tests timed out after {timeout}s (treated as success)", rc
+
+    # Exit code 78 (EX_CONFIG) or sentinel = no benchmarks discovered
+    if rc == 78 or "FORMULACODE_NO_BENCHMARKS" in stdout:
+        return False, stdout, "No ASV benchmarks discovered — task cannot be used in FormulaCode", rc
+
+    # Non-timeout failures
+    if rc != 0:
+        return False, stdout, stderr, rc
+
+    # run-tests.sh may exit 0 even on collection errors — check structured output
+    summary = _parse_test_summary(stdout)
+    if summary is not None:
+        if summary.get("total", 0) == 0 or summary.get("error", 0) > 0:
+            return False, stdout, stderr, rc
+
+    # Check that benchmarks were actually discovered
+    snapshot = _parse_snapshot_summary(stdout)
+    if snapshot is not None and snapshot.get("total", 0) == 0:
+        return False, stdout, "No ASV benchmarks discovered (snapshot total=0)", rc
+
+    return True, stdout, stderr, rc
+
+
+def _check_file_integrity(task_dir: Path) -> str | None:
+    """Verify that immutable files haven't been modified since workspace setup.
+
+    Returns ``None`` if all files are intact, or an error description.
+    """
+    hashes_file = task_dir.parent / ".immutable_hashes.json"
+    if not hashes_file.exists():
+        return None  # no hash manifest = no enforcement (e.g. manual runs)
+
+    expected: dict[str, str] = json.loads(hashes_file.read_text())
+    modified: list[str] = []
+    deleted: list[str] = []
+
+    for fname, expected_hash in expected.items():
+        fp = task_dir / fname
+        if not fp.exists():
+            deleted.append(fname)
+        elif hashlib.md5(fp.read_bytes()).hexdigest() != expected_hash:  # noqa: S324
+            modified.append(fname)
+
+    if not modified and not deleted:
+        return None
+
+    parts = [
+        "File integrity violation — only docker_build_pkg.sh and "
+        "docker_build_run.sh may be edited.",
+    ]
+    if modified:
+        parts.append(f"Modified: {', '.join(modified)}")
+    if deleted:
+        parts.append(f"Deleted: {', '.join(deleted)}")
+    parts.append("Revert your changes to these files and try again.")
+    return "\n".join(parts)
+
+
+def verify(task_dir: Path) -> bool:
+    docker = DockerClient()
+    metrics: dict = {}
+
+    # Check file integrity before anything else
+    integrity_error = _check_file_integrity(task_dir)
+    if integrity_error:
+        print(f"INTEGRITY ERROR:\n{integrity_error}")
+        _write_failure(task_dir, "integrity", stderr=integrity_error, metrics=metrics)
+        return False
+
+    # Parse task
+    try:
+        task = _parse_task(task_dir)
+    except Exception as e:
+        print(f"Failed to parse task in {task_dir}: {e}")
+        _write_failure(task_dir, "parse", stderr=str(e), metrics=metrics)
+        return False
+
+    if not task.sha:
+        print(f"Task in {task_dir} has no SHA")
+        _write_failure(task_dir, "parse", stderr="Task.sha is None", metrics=metrics)
+        return False
+
+    # Build
+    try:
+        tag = build_image(docker, task_dir, task, target="run", metrics=metrics)
+        print(f"Build succeeded: {tag}")
+    except BuildError as e:
+        print(f"Build failed: {e}")
+        _write_failure(task_dir, "build", stdout=e.stdout, stderr=e.stderr, rc=e.rc, metrics=metrics)
+        return False
+    except Exception as e:
+        print(f"Build failed: {str(e)[:200]}")
+        _write_failure(task_dir, "build", stderr=str(e), metrics=metrics)
+        return False
+
+    # Tests — mandatory, no skip option (profile.sh runs inside run-tests.sh)
+    ok, stdout, stderr, rc = run_tests(tag, metrics=metrics)
+    if not ok:
+        print(f"Tests failed for {task_dir.name}")
+        _write_failure(task_dir, "tests", stdout=stdout, stderr=stderr, rc=rc, metrics=metrics)
+        return False
+    print(f"Tests passed for {task_dir.name}")
+
+    _write_success(task_dir, tag, metrics=metrics)
+    print(f"SUCCESS: {task_dir.name}")
+    return True
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Verify Docker build context")
+    parser.add_argument(
+        "--task",
+        type=Path,
+        default=Path("task"),
+        help="Task directory (default: task/)",
+    )
+    args = parser.parse_args()
+
+    if not args.task.exists():
+        print(f"Task directory not found: {args.task}")
+        sys.exit(1)
+
+    ok = verify(args.task)
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

datasmith/docker/__init__.py

@@ -0,0 +1,31 @@
+"""ds.docker — Docker image lifecycle, build contexts, verifiers, publishing."""
+
+from datasmith.docker.context import DockerContext
+from datasmith.docker.images import (
+    ImageManager,
+    get_base_image_name,
+    get_pr_image_name,
+    get_repo_image_name,
+)
+from datasmith.docker.publish import DockerHubPublisher
+from datasmith.docker.verifiers import (
+    MultiObjVerifier,
+    ProfileVerifier,
+    PytestVerifier,
+    SmokeVerifier,
+    VerifyResult,
+)
+
+__all__ = [
+    "DockerContext",
+    "DockerHubPublisher",
+    "ImageManager",
+    "MultiObjVerifier",
+    "ProfileVerifier",
+    "PytestVerifier",
+    "SmokeVerifier",
+    "VerifyResult",
+    "get_base_image_name",
+    "get_pr_image_name",
+    "get_repo_image_name",
+]

datasmith/docker/context.py

@@ -0,0 +1,112 @@
+"""Docker build context model."""
+
+from __future__ import annotations
+
+import io
+import os
+import tarfile
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+
+class DockerContext(BaseModel):
+    model_config = ConfigDict(frozen=False)
+
+    dockerfile: str = ""
+    build_base_sh: str = ""
+    build_env_sh: str = ""
+    build_pkg_sh: str = ""
+    build_run_sh: str = ""
+    build_final_sh: str = ""
+    profile_sh: str = ""
+    run_tests_sh: str = ""
+    entrypoint_sh: str = ""
+
+    _FILE_MAP: ClassVar[dict[str, str]] = {
+        "Dockerfile": "dockerfile",
+        "docker_build_base.sh": "build_base_sh",
+        "docker_build_env.sh": "build_env_sh",
+        "docker_build_pkg.sh": "build_pkg_sh",
+        "docker_build_run.sh": "build_run_sh",
+        "docker_build_final.sh": "build_final_sh",
+        "profile.sh": "profile_sh",
+        "run-tests.sh": "run_tests_sh",
+        "entrypoint.sh": "entrypoint_sh",
+    }
+
+    _LEGACY_MAP: ClassVar[dict[str, str]] = {
+        "dockerfile_data": "dockerfile",
+        "base_building_data": "build_base_sh",
+        "env_building_data": "build_env_sh",
+        "building_data": "build_pkg_sh",
+        "run_building_data": "build_run_sh",
+        "final_building_data": "build_final_sh",
+        "profile_data": "profile_sh",
+        "run_tests_data": "run_tests_sh",
+        "entrypoint_data": "entrypoint_sh",
+    }
+
+    def to_tar_bytes(self) -> bytes:
+        """Serialize context to in-memory tar for Docker build."""
+        buf = io.BytesIO()
+        with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+            for filename, field in sorted(self._FILE_MAP.items()):
+                content = getattr(self, field)
+                if not content:
+                    continue
+                data = content.encode("utf-8")
+                info = tarfile.TarInfo(name=filename)
+                info.size = len(data)
+                info.mtime = 0
+                info.uid = 0
+                info.gid = 0
+                info.uname = ""
+                info.gname = ""
+                tar.addfile(info, io.BytesIO(data))
+        return buf.getvalue()
+
+    def to_directory(self, path: str) -> None:
+        """Write all context files to a directory on disk."""
+        os.makedirs(path, exist_ok=True)
+        for filename, field in self._FILE_MAP.items():
+            content = getattr(self, field)
+            if content:
+                with open(os.path.join(path, filename), "w") as f:
+                    f.write(content)
+
+    # Fallback filenames for backward compatibility (old name -> new name).
+    _COMPAT_MAP: ClassVar[dict[str, str]] = {
+        "run_tests.sh": "run-tests.sh",
+    }
+
+    @classmethod
+    def from_directory(cls, path: str) -> DockerContext:
+        """Load a DockerContext from a task directory."""
+
+        def _read(name: str) -> str:
+            fp = os.path.join(path, name)
+            if os.path.exists(fp):
+                with open(fp) as f:
+                    return f.read()
+            # Try legacy filename
+            legacy = {v: k for k, v in cls._COMPAT_MAP.items()}.get(name)
+            if legacy:
+                fp = os.path.join(path, legacy)
+                if os.path.exists(fp):
+                    with open(fp) as f:
+                        return f.read()
+            return ""
+
+        kwargs = {field: _read(filename) for filename, field in cls._FILE_MAP.items()}
+        return cls(**kwargs)
+
+    @classmethod
+    def from_legacy_dict(cls, data: dict[str, Any]) -> DockerContext:
+        """Create a DockerContext from the old context registry format."""
+        kwargs = {}
+        for legacy_key, field in cls._LEGACY_MAP.items():
+            value = data.get(legacy_key, "")
+            if value:
+                kwargs[field] = value
+        return cls(**kwargs)