synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/sdk/learning/validators.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from synth_ai.sdk.learning.sft import SFTDataError, parse_jsonl_line
+
+
+def validate_training_jsonl(path: str | Path, *, sample_lines: int = 50) -> None:
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(str(p))
+
+    max_samples = max(1, sample_lines)
+    non_empty_lines = 0
+
+    with p.open("r", encoding="utf-8") as fh:
+        for lineno, raw_line in enumerate(fh, start=1):
+            stripped = raw_line.strip()
+            if not stripped:
+                continue
+            non_empty_lines += 1
+            if non_empty_lines > max_samples:
+                break
+            try:
+                parse_jsonl_line(stripped, min_messages=2)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"invalid json on line {lineno}: {exc}") from exc
+            except SFTDataError as exc:
+                raise ValueError(f"line {lineno}: {exc}") from exc
+
+    if non_empty_lines == 0:
+        raise ValueError("empty JSONL")
+
+
+def validate_task_app_url(url: str, *, name: str = "TASK_APP_BASE_URL") -> None:
+    from synth_ai.sdk.task.validators import validate_task_app_url as _vt
+
+    try:
+        _vt(url)
+    except ValueError as exc:
+        raise ValueError(f"{name}: {exc}") from exc
+
+
+def validate_trainer_cfg_rl(trainer: dict[str, Any]) -> None:
+    bs = int(trainer.get("batch_size", 1))
+    gs = int(trainer.get("group_size", 2))
+    if bs < 1:
+        raise ValueError("trainer.batch_size must be >= 1")
+    if gs < 2:
+        raise ValueError("trainer.group_size must be >= 2")

synth_ai/sdk/research_agent/__init__.py

@@ -0,0 +1,34 @@
+from synth_ai.sdk.research_agent.container_builder import (
+    ContainerBackend,
+    DockerBackend,
+    ModalBackend,
+    get_backend,
+)
+from synth_ai.sdk.research_agent.container_spec import ContainerSpec
+from synth_ai.sdk.research_agent.defaults import (
+    DEFAULT_BACKEND,
+    DEFAULT_BASE_IMAGE,
+    DEFAULT_INSTRUCTIONS,
+    DEFAULT_PACKAGES,
+    DEFAULT_PYTHON_VERSION,
+    DEFAULT_REASONING_EFFORT,
+    DEFAULT_RESULT_PATTERNS,
+)
+from synth_ai.sdk.research_agent.results_collector import ResultsCollector
+
+__all__ = [
+    "ContainerBackend",
+    "ContainerSpec",
+    "DockerBackend",
+    "ModalBackend",
+    "ResultsCollector",
+    "get_backend",
+    "DEFAULT_BACKEND",
+    "DEFAULT_BASE_IMAGE",
+    "DEFAULT_INSTRUCTIONS",
+    "DEFAULT_PACKAGES",
+    "DEFAULT_PYTHON_VERSION",
+    "DEFAULT_REASONING_EFFORT",
+    "DEFAULT_RESULT_PATTERNS",
+]
+

synth_ai/sdk/research_agent/container_builder.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import contextlib
+import fnmatch
+import io
+import tarfile
+import tempfile
+import time
+import uuid
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Dict, Iterable, Tuple
+
+from synth_ai.sdk.research_agent.container_spec import ContainerSpec
+from synth_ai.sdk.research_agent.defaults import DEFAULT_BACKEND
+
+
+class ContainerBackend(ABC):
+    """Abstract base for container execution backends."""
+
+    @abstractmethod
+    async def provision(self, spec: ContainerSpec) -> str:
+        """Provision a new container and return its id/handle."""
+
+    @abstractmethod
+    async def execute(
+        self,
+        container_id: str,
+        command: str,
+        *,
+        env: Dict[str, str] | None = None,
+        workdir: Path | None = None,
+    ) -> Dict[str, str | int]:
+        """Execute a command in the container."""
+
+    @abstractmethod
+    async def collect_artifacts(self, container_id: str, patterns: Iterable[str]) -> Dict[str, bytes]:
+        """Pull artifacts that match any of the glob patterns."""
+
+    @abstractmethod
+    async def destroy(self, container_id: str) -> None:
+        """Tear down container resources."""
+
+
+class DockerBackend(ContainerBackend):
+    """Docker implementation using docker-py."""
+
+    def __init__(self, *, client=None):
+        self._client = client
+        self._containers: Dict[str, Tuple[object, ContainerSpec]] = {}
+
+    def _ensure_client(self):
+        if self._client is None:
+            try:
+                import docker  # type: ignore
+            except ImportError as exc:
+                raise RuntimeError("docker SDK is not installed. Add docker>=7.0.0 to dependencies.") from exc
+            self._client = docker.from_env()
+        return self._client
+
+    async def provision(self, spec: ContainerSpec) -> str:
+        spec.validate()
+        client = self._ensure_client()
+        context_bytes = spec.build_context()
+        image_tag = f"research-agent:{int(time.time())}"
+
+        def _build():
+            return client.images.build(
+                fileobj=io.BytesIO(context_bytes),
+                custom_context=True,
+                rm=True,
+                nocache=True,
+                tag=image_tag,
+                buildargs=spec.build_args,
+            )
+
+        loop = asyncio.get_running_loop()
+        image, _ = await loop.run_in_executor(None, _build)
+
+        container = client.containers.create(
+            image=image.id,
+            command="sleep infinity",
+            environment={**spec.env_vars, **spec.secrets},
+            tty=True,
+            detach=True,
+            working_dir=str(spec.workdir),
+        )
+        container.start()
+        self._containers[container.id] = (container, spec)
+        return container.id
+
+    async def execute(
+        self,
+        container_id: str,
+        command: str,
+        *,
+        env: Dict[str, str] | None = None,
+        workdir: Path | None = None,
+    ) -> Dict[str, str | int]:
+        container, spec = self._containers[container_id]
+        if env is None:
+            env = {}
+        exec_env = {**spec.env_vars, **spec.secrets, **env}
+        workdir_str = str(workdir or spec.workdir)
+
+        def _run():
+            result = container.exec_run(  # type: ignore[attr-defined]
+                cmd=["bash", "-lc", command],
+                environment=exec_env,
+                workdir=workdir_str,
+                demux=True,
+            )
+            stdout, stderr = result.output
+            return {
+                "exit_code": result.exit_code,
+                "stdout": (stdout or b"").decode(),
+                "stderr": (stderr or b"").decode(),
+            }
+
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, _run)
+
+    async def collect_artifacts(self, container_id: str, patterns: Iterable[str]) -> Dict[str, bytes]:
+        container, spec = self._containers[container_id]
+        loop = asyncio.get_running_loop()
+
+        def _pull():
+            try:
+                stream, _ = container.get_archive(str(spec.artifacts_dir))  # type: ignore[attr-defined]
+            except Exception:
+                return {}
+            tar_bytes = b"".join(stream)
+            collected: Dict[str, bytes] = {}
+            with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+                for member in tar.getmembers():
+                    if not member.isfile():
+                        continue
+                    relative_name = str(Path(member.name).name)
+                    if not any(fnmatch.fnmatch(relative_name, pat) for pat in patterns):
+                        continue
+                    file_obj = tar.extractfile(member)
+                    if file_obj:
+                        collected[relative_name] = file_obj.read()
+            return collected
+
+        return await loop.run_in_executor(None, _pull)
+
+    async def destroy(self, container_id: str) -> None:
+        container, _ = self._containers.pop(container_id, (None, None))
+        if container is None:
+            return
+
+        def _stop():
+            with contextlib.suppress(Exception):
+                container.kill()  # type: ignore[attr-defined]
+            with contextlib.suppress(Exception):
+                container.remove(force=True)  # type: ignore[attr-defined]
+
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, _stop)
+
+
+class ModalBackend(ContainerBackend):
+    """Modal implementation using modal SDK. Returns artifacts inline from execute()."""
+
+    def __init__(self):
+        self._runs: Dict[str, Dict[str, object]] = {}
+
+    def _write_build_context(self, spec: ContainerSpec) -> tempfile.TemporaryDirectory:
+        """Materialize Dockerfile + overlay files to a temp dir for Modal build."""
+        temp_dir = tempfile.TemporaryDirectory()
+        ctx = Path(temp_dir.name)
+        (ctx / "Dockerfile").write_text(spec.to_dockerfile())
+
+        overlay_root = ctx / "overlay_files"
+        for rel_path, content in spec.rendered_overlay_files().items():
+            target = overlay_root / rel_path
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(content)
+
+        for rel_path, content in spec.files.items():
+            if not str(rel_path).startswith("/"):
+                continue
+            data = content.encode() if isinstance(content, str) else content
+            target = ctx / str(rel_path).lstrip("/")
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(data)
+
+        return temp_dir
+
+    async def provision(self, spec: ContainerSpec) -> str:
+        spec.validate()
+        try:
+            import modal  # type: ignore
+        except ImportError as exc:  # pragma: no cover - runtime import guard
+            raise RuntimeError("modal SDK is not installed. Add modal>=1.1.1 to dependencies.") from exc
+
+        ctx_dir = self._write_build_context(spec)
+        loop = asyncio.get_running_loop()
+
+        def _build_image():
+            return modal.Image.from_dockerfile(
+                path=ctx_dir.name,
+                build_args=spec.build_args,
+                force_build=True,
+            )
+
+        image = await loop.run_in_executor(None, _build_image)
+
+        # Combine env_vars and secrets into a Modal Secret
+        # Modal function decorator doesn't accept 'env' parameter directly
+        # Environment variables must be passed via secrets
+        combined_env: dict[str, str | None] = {**spec.env_vars, **spec.secrets}
+        secret_obj = None
+        if combined_env:
+            secret_obj = modal.Secret.from_dict(combined_env)
+
+        app = modal.App(f"oneshot-research-{int(time.time())}")
+
+        workdir_str = str(spec.workdir)
+        
+        @app.function(
+            image=image,
+            timeout=60 * 60,
+            secrets=[secret_obj] if secret_obj else [],
+        )
+        def run_task(command: str, patterns: list[str], artifacts_dir: str = "/app/artifacts") -> Dict:
+            """Execute the agent and pull artifacts matching patterns."""
+            import glob
+            import os
+            import subprocess
+
+            result = subprocess.run(
+                ["bash", "-lc", command],
+                capture_output=True,
+                text=True,
+                cwd=workdir_str,
+            )
+
+            artifacts: Dict[str, str] = {}
+            for pat in patterns:
+                for path in glob.glob(os.path.join(artifacts_dir, pat)):
+                    if not os.path.isfile(path):
+                        continue
+                    name = os.path.basename(path)
+                    with open(path, "rb") as f:
+                        artifacts[name] = base64.b64encode(f.read()).decode()
+
+            return {
+                "exit_code": result.returncode,
+                "stdout": result.stdout,
+                "stderr": result.stderr,
+                "artifacts": artifacts,
+            }
+
+        container_id = str(uuid.uuid4())
+        self._runs[container_id] = {
+            "app": app,
+            "function": run_task,
+            "result": None,
+            "ctx_dir": ctx_dir,
+            "patterns": tuple(spec.result_matchers()),
+        }
+        return container_id
+
+    async def execute(
+        self,
+        container_id: str,
+        command: str,
+        *,
+        env: Dict[str, str] | None = None,
+        workdir: Path | None = None,
+    ) -> Dict[str, str | int]:
+        run_info = self._runs.get(container_id)
+        if not run_info:
+            raise ValueError(f"Unknown container_id: {container_id}")
+        app = run_info["app"]
+        run_fn = run_info["function"]
+        patterns = list(run_info["patterns"])
+
+        loop = asyncio.get_running_loop()
+
+        def _call():
+            with app.run():
+                return run_fn.call(command, patterns)
+
+        result = await loop.run_in_executor(None, _call)
+        run_info["result"] = result
+        return {
+            "exit_code": result.get("exit_code", -1),
+            "stdout": result.get("stdout", ""),
+            "stderr": result.get("stderr", ""),
+        }
+
+    async def collect_artifacts(self, container_id: str, patterns: Iterable[str]) -> Dict[str, bytes]:
+        run_info = self._runs.get(container_id)
+        if not run_info:
+            return {}
+        result = run_info.get("result") or {}
+        artifacts: Dict[str, bytes] = {}
+        encoded = result.get("artifacts") or {}  # type: ignore[misc]
+        for name, b64 in encoded.items():
+            try:
+                artifacts[name] = base64.b64decode(b64)
+            except Exception:
+                continue
+        return artifacts
+
+    async def destroy(self, container_id: str) -> None:
+        info = self._runs.pop(container_id, None)
+        if not info:
+            return
+        ctx_dir = info.get("ctx_dir")
+        if ctx_dir and hasattr(ctx_dir, "cleanup"):
+            with contextlib.suppress(Exception):
+                ctx_dir.cleanup()  # type: ignore[call-arg]
+
+
+def get_backend(name: str = DEFAULT_BACKEND) -> ContainerBackend:
+    """Resolve backend by name."""
+    normalized = (name or DEFAULT_BACKEND).lower()
+    if normalized == "docker":
+        return DockerBackend()
+    if normalized == "modal":
+        return ModalBackend()
+    raise ValueError(f"Unsupported container backend: {name}")

synth_ai/sdk/research_agent/container_spec.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import io
+import textwrap
+import time
+from dataclasses import dataclass, field
+from importlib import import_module
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional
+
+from synth_ai.sdk.research_agent.defaults import (
+    DEFAULT_BACKEND,
+    DEFAULT_BASE_IMAGE,
+    DEFAULT_INSTRUCTIONS,
+    DEFAULT_PACKAGES,
+    DEFAULT_PYTHON_VERSION,
+    DEFAULT_RESULT_PATTERNS,
+    DEFAULT_SYNTH_PIP_SPEC,
+)
+
+
+def _load_box_bootstrap() -> str:
+    """Load the canonical box_bootstrap.sh content from regenerate_bootstrap."""
+    try:
+        mod = import_module("scripts.regenerate_bootstrap")
+        content = getattr(mod, "bootstrap_content", None)
+        if content:
+            return content
+    except Exception:
+        pass
+    return "#!/bin/bash\nset -euo pipefail\necho 'box_bootstrap.sh missing; ensure regenerate_bootstrap.py is available.'\nexit 1\n"
+
+
+@dataclass
+class ContainerSpec:
+    """Declarative container configuration for running the research agent."""
+
+    repo_url: str
+    repo_branch: str = "main"
+    repo_commit: Optional[str] = None
+
+    agent_instructions: str = DEFAULT_INSTRUCTIONS
+    base_image: str = DEFAULT_BASE_IMAGE
+    python_version: str = DEFAULT_PYTHON_VERSION
+    apt_packages: List[str] = field(default_factory=lambda: list(DEFAULT_PACKAGES))
+
+    env_vars: Dict[str, str] = field(default_factory=dict)
+    secrets: Dict[str, str] = field(default_factory=dict)
+    files: Dict[str, str | bytes] = field(default_factory=dict)
+    preflight_script: Optional[str] = None
+    postflight_script: Optional[str] = None
+
+    artifacts_dir: Path = Path("/app/artifacts")
+    result_patterns: List[str] = field(default_factory=lambda: list(DEFAULT_RESULT_PATTERNS))
+    workdir: Path = Path("/app/repo")
+
+    backend: str = DEFAULT_BACKEND
+    overlay_dir: Optional[Path] = None
+    bootstrap_content: str = field(default_factory=_load_box_bootstrap)
+    synth_pip_spec: str = DEFAULT_SYNTH_PIP_SPEC
+
+    def validate(self) -> None:
+        """Lightweight validation before provisioning."""
+        if not self.repo_url:
+            raise ValueError("repo_url is required")
+        if not self.agent_instructions:
+            raise ValueError("agent_instructions is required")
+        if not self.base_image:
+            raise ValueError("base_image is required")
+        if not self.artifacts_dir.is_absolute():
+            raise ValueError("artifacts_dir must be absolute")
+
+    @property
+    def build_args(self) -> Dict[str, str]:
+        """Build args passed to Docker/Modal image creation."""
+        return {
+            "GIT_URL": self.repo_url,
+            "GIT_BRANCH": self.repo_branch,
+            "GIT_COMMIT": self.repo_commit or "",
+            "PYTHON_VERSION": self.python_version,
+            "SYNTH_PIP_SPEC": self.synth_pip_spec or "",
+        }
+
+    def to_dockerfile(self) -> str:
+        """Render a Dockerfile that mirrors the existing OneShot bootstrap."""
+        package_line = " ".join(sorted(set(self.apt_packages)))
+        return textwrap.dedent(
+            f"""
+            FROM {self.base_image}
+
+            ARG GIT_URL
+            ARG GIT_BRANCH
+            ARG GIT_COMMIT
+            ARG PYTHON_VERSION="{self.python_version}"
+
+            ENV DEBIAN_FRONTEND=noninteractive
+            ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+            ENV PIP_NO_PYTHON_VERSION_WARNING=1
+            ENV PIP_BREAK_SYSTEM_PACKAGES=1
+            ENV PYTHONWARNINGS=ignore
+
+            RUN apt-get update && \\
+                apt-get install -y --no-install-recommends {package_line} && \\
+                ln -sf /usr/bin/python3 /usr/bin/python && \\
+                mkdir -p {self.artifacts_dir} /app/overlay_files && \\
+                apt-get clean
+
+            # Install uv for fast Python installs
+            RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \\
+                ln -sf /root/.local/bin/uv /usr/local/bin/uv || true
+
+            WORKDIR /app
+            RUN git clone --branch "$GIT_BRANCH" "$GIT_URL" repo && \\
+                cd repo && if [ -n "$GIT_COMMIT" ]; then git checkout "$GIT_COMMIT"; fi && \\
+                python3 -m venv /app/repo/.venv && \\
+                . /app/repo/.venv/bin/activate && \\
+                pip install --no-cache-dir --upgrade pip && \\
+                if [ -n "$SYNTH_PIP_SPEC" ]; then \\
+                  pip install --no-cache-dir "$SYNTH_PIP_SPEC"; \\
+                else \\
+                  pip install --no-cache-dir -e .; \\
+                fi
+            ENV VIRTUAL_ENV="/app/repo/.venv"
+            ENV PATH="/app/repo/.venv/bin:${{PATH}}"
+
+            COPY overlay_files/ /app/
+
+            WORKDIR {self.workdir}
+            """
+        ).strip() + "\n"
+
+    def rendered_overlay_files(self) -> Dict[str, bytes]:
+        """Overlay files placed into the build context / runtime container."""
+        files: Dict[str, bytes] = {}
+
+        if self.overlay_dir and self.overlay_dir.exists():
+            for path in self.overlay_dir.rglob("*"):
+                if path.is_dir():
+                    continue
+                rel_path = path.relative_to(self.overlay_dir)
+                files.setdefault(str(rel_path), path.read_bytes())
+
+        if self.agent_instructions:
+            files["LM_INSTRUCTIONS.md"] = self.agent_instructions.encode()
+
+        if self.preflight_script:
+            files["overlay_hidden_pre/preflight.sh"] = self.preflight_script.encode()
+            files["pre_flight.sh"] = self.preflight_script.encode()
+        if self.postflight_script:
+            files["overlay_hidden_post/postflight.sh"] = self.postflight_script.encode()
+
+        for relative_path, content in self.files.items():
+            data = content.encode() if isinstance(content, str) else content
+            files[str(relative_path)] = data
+
+        files.setdefault("box_bootstrap.sh", self.bootstrap_content.encode())
+
+        return files
+
+    def build_context(self) -> bytes:
+        """Create a tar build context for Docker builds."""
+        import tarfile
+
+        buf = io.BytesIO()
+        with tarfile.open(fileobj=buf, mode="w") as tar:
+            dockerfile_bytes = self.to_dockerfile().encode()
+            dockerfile_info = tarfile.TarInfo("Dockerfile")
+            dockerfile_info.size = len(dockerfile_bytes)
+            dockerfile_info.mtime = int(time.time())
+            tar.addfile(dockerfile_info, io.BytesIO(dockerfile_bytes))
+
+            for rel_path, content in self.rendered_overlay_files().items():
+                overlay_path = Path("overlay_files") / rel_path
+                executable = overlay_path.name.endswith(".sh")
+                info = tarfile.TarInfo(str(overlay_path))
+                info.size = len(content)
+                info.mtime = int(time.time())
+                if executable:
+                    info.mode = 0o755
+                tar.addfile(info, io.BytesIO(content))
+
+            for rel_path, content in self.files.items():
+                if str(rel_path).startswith("/"):
+                    data = content.encode() if isinstance(content, str) else content
+                    target = Path(str(rel_path).lstrip("/"))
+                    info = tarfile.TarInfo(str(target))
+                    info.size = len(data)
+                    info.mtime = int(time.time())
+                    if target.name.endswith(".sh"):
+                        info.mode = 0o755
+                    tar.addfile(info, io.BytesIO(data))
+
+        buf.seek(0)
+        return buf.read()
+
+    def result_matchers(self) -> Iterable[str]:
+        """Expose patterns for callers that only need read-only access."""
+        return tuple(self.result_patterns)

synth_ai/sdk/research_agent/defaults.py

@@ -0,0 +1,34 @@
+"""Defaults and shared constants for the research agent (library side)."""
+
+from __future__ import annotations
+
+DEFAULT_INSTRUCTIONS = "Run baseline, then optimize prompt with configured optimizer."
+DEFAULT_MODEL = "gpt-4o-mini"
+DEFAULT_REASONING_EFFORT = "medium"
+
+DEFAULT_BACKEND = "docker"
+DEFAULT_BASE_IMAGE = "ubuntu:24.04"
+DEFAULT_PYTHON_VERSION = "3.11"
+DEFAULT_PACKAGES = (
+    "git",
+    "curl",
+    "build-essential",
+    "cmake",
+    "ninja-build",
+    "pkg-config",
+    "python3",
+    "python3-venv",
+    "python3-pip",
+    "ca-certificates",
+    "jq",
+)
+
+DEFAULT_RESULT_PATTERNS = (
+    "*.json",
+    "*.log",
+    "*.md",
+    "*.toml",
+    "diff.patch",
+)
+
+DEFAULT_SYNTH_PIP_SPEC = "synth-ai==0.2.26.dev1"