inspect-sandboxes 0.1.1.dev1__tar.gz → 0.2.0__tar.gz

{inspect_sandboxes-0.1.1.dev1 → inspect_sandboxes-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inspect_sandboxes
-Version: 0.1.1.dev1
+Version: 0.2.0
 Summary: Collection of sandboxes for Inspect AI
 Project-URL: Source Code, https://github.com/meridianlabs-ai/inspect_sandboxes
 Project-URL: Issue Tracker, https://github.com/meridianlabs-ai/inspect_sandboxes/issues

{inspect_sandboxes-0.1.1.dev1 → inspect_sandboxes-0.2.0}/pyproject.toml

@@ -102,6 +102,9 @@ ignore = [
     "D415", # first line should end with a period, question mark, or exclamation point
 ]
 
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["D103"]  # missing docstring in public function
+
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 
@@ -111,6 +114,7 @@ relative_files = true
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 testpaths = ["tests"]
+addopts = "-m 'not integration'"
 markers = [
-    "integration: marks tests that require real Modal resources (deselect with '-m \"not integration\"')",
+    "integration: marks tests that require real Modal/Daytona resources (run with '-m integration')",
 ]

inspect_sandboxes-0.2.0/src/inspect_sandboxes/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
+
+__commit_id__ = commit_id = None

inspect_sandboxes-0.2.0/src/inspect_sandboxes/daytona/README.md

@@ -0,0 +1,167 @@
+# Daytona Sandbox
+
+Cloud development environment sandbox for [Inspect AI](https://inspect.ai-safety-institute.org.uk/) using [Daytona](https://www.daytona.io/).
+
+## Setup
+
+Create a Daytona account and set your API key:
+
+```bash
+export DAYTONA_API_KEY=your_api_key
+```
+
+## Usage
+
+### Default snapshot
+
+```python
+from inspect_ai import Task, eval
+from inspect_ai.solver import generate, system_message
+
+task = Task(
+    dataset=[{"input": "What is 2+2?", "target": "4"}],
+    solver=[
+        system_message("You are a helpful assistant."),
+        generate(),
+    ],
+    sandbox="daytona",  # Uses Daytona's default snapshot
+)
+
+eval(task)
+```
+
+### Dockerfile
+
+```python
+task = Task(
+    dataset=[...],
+    solver=[...],
+    sandbox=("daytona", "path/to/Dockerfile"),
+)
+```
+
+### Single-service Docker Compose
+
+```yaml
+# compose.yaml
+services:
+  default:
+    image: python:3.12
+    environment:
+      - MY_VAR=hello
+    deploy:
+      resources:
+        limits:
+          cpus: "2.0"
+          memory: 4g
+        reservations:
+          devices:
+            - capabilities: [gpu]
+              count: 1
+```
+
+```python
+task = Task(
+    dataset=[...],
+    solver=[...],
+    sandbox=("daytona", "compose.yaml"),
+)
+```
+
+### Multi-service Docker Compose (DinD)
+
+When a compose file defines more than one service, the provider automatically
+uses Docker-in-Docker: a single Daytona sandbox runs a Docker daemon, and
+services are brought up via `docker compose` inside it. Each service is
+exposed as a separate `SandboxEnvironment`.
+
+```yaml
+# compose.yaml
+services:
+  default:
+    image: python:3.12
+    x-default: true
+  helper:
+    image: redis:7
+```
+
+```python
+task = Task(
+    dataset=[...],
+    solver=[...],
+    sandbox=("daytona", "compose.yaml"),
+)
+
+# In a solver, access services by name:
+default_env = sandbox()          # the x-default service
+helper_env = sandbox("helper")
+```
+
+**Default service selection** (priority): `x-default: true` > service named
+`"default"` or `"main"` > first service in the file.
+
+**Resources**: Per-service resources are summed across all services plus 1 CPU
+and 1 GiB overhead for the Docker daemon. Ensure the total fits within your
+Daytona per-sandbox limits.
+
+**DinD image**: The DinD sandbox uses `docker:28.3.3-dind` as the base image.
+
+**DinD snapshots**: The provider auto-creates a Daytona snapshot for the DinD
+base image. You can also provide a pre-created snapshot:
+
+```yaml
+x-daytona:
+  snapshot: "my-dind-snapshot"
+```
+
+**Image caching**: Each DinD sandbox gets a fresh Docker daemon with no image
+cache. `docker compose build` rebuilds from scratch every sample. For faster
+startup, pre-build your images and push them to a registry, then use `image:`
+instead of `build:` in your compose file:
+
+```yaml
+services:
+  default:
+    image: myregistry/myapp:latest   # pulled, not built
+  verifier:
+    image: myregistry/verifier:latest
+```
+
+*Daytona does not support snapshotting a running sandbox (to capture Docker's
+layer cache) or shared volumes across sandboxes, so registry-based caching is
+the recommended approach.*
+
+### x-daytona extensions
+
+```yaml
+x-daytona:
+  auto_stop_interval: 10       # Minutes of inactivity before auto-stop (0 = disabled)
+  auto_archive_interval: 60    # Minutes before stopped sandbox auto-archives
+  auto_delete_interval: 1440   # Minutes before stopped sandbox auto-deletes
+  network_block_all: false     # Block all network access
+  network_allow_list: "10.0.0.0/8,192.168.0.0/16"  # Comma-separated CIDR allowlist
+  language: "python"           # Hint for language-aware features
+  os_user: "ubuntu"            # OS user for commands (overrides service-level user)
+  public: false                # Publicly accessible sandbox
+  ephemeral: true              # Auto-delete when stopped
+  timeout: 60.0                # Seconds to wait for sandbox creation
+  snapshot: "my-snapshot"      # Pre-created snapshot (skips image build)
+  resources:                   # Sandbox-level resource overrides
+    cpu: 4                     #   (DinD: overrides per-service aggregation)
+    memory: 8
+    gpu: 1
+  env_vars:                    # Extra env vars (single-service: merged with service
+    EXTRA_VAR: "value"         #   environment; DinD: set on the VM, not compose services)
+  labels:                      # Custom labels, merged with inspect's own labels
+    team: "research"
+```
+
+## Notes and Limitations
+
+- **Default user**: The default sandbox user is `daytona` (not root), with passwordless `sudo`.
+- **user**: The `user` parameter to `exec()` is supported via `sudo -u` in single-service mode and `docker compose exec --user` in DinD mode. Numeric UIDs are supported. Requires `sudo` with passwordless access (configured by default).
+- **stdin**: The `input` parameter to `exec()` is supported via input redirection from a temp file. POSIX-compatible.
+- **stderr**: The Daytona API returns combined stdout+stderr; `stderr` is always empty.
+- **Architecture**: Daytona runners use `linux/amd64`. arm64-only images are not supported.
+- **Network**: Outbound internet depends on your Daytona subscription tier. Tiers 1-2 restrict to essential services (Docker Hub, npm, PyPI, GitHub, AI providers). Tiers 3-4 have full internet. Docker Compose `network_mode` is translated to Daytona's `network_block_all`; `x-daytona.network_block_all` takes precedence. For DinD, the VM always has network enabled (for `docker pull`); service-level isolation is via compose `network_mode`.
+- **DinD startup latency**: Docker daemon boot + image pulls + compose up can take 30s+. Use pre-built images and snapshots to reduce this.

inspect_sandboxes-0.2.0/src/inspect_sandboxes/daytona/__init__.py

@@ -0,0 +1 @@
+"""Daytona sandbox implementation for Inspect AI"""

{inspect_sandboxes-0.1.1.dev1 → inspect_sandboxes-0.2.0}/src/inspect_sandboxes/daytona/_compose.py

@@ -2,7 +2,12 @@ import math
 from pathlib import Path
 from typing import Any
 
-from daytona_sdk import Image, Resources
+from daytona_sdk import (
+    CreateSandboxFromImageParams,
+    CreateSandboxFromSnapshotParams,
+    Image,
+    Resources,
+)
 from inspect_ai.util import ComposeConfig, ComposeService
 
 from inspect_sandboxes._util.compose import (
@@ -12,25 +17,20 @@ from inspect_sandboxes._util.compose import (
 )
 
 
-def convert_compose_to_daytona_params(
-    config: ComposeConfig, compose_path: str | None
-) -> tuple[str | Image, Resources | None, dict[str, Any]]:
-    """Convert a ComposeConfig to Daytona sandbox creation parameters.
+def create_single_service_params(
+    config: ComposeConfig,
+    compose_path: str | None,
+    labels: dict[str, str],
+) -> CreateSandboxFromImageParams | CreateSandboxFromSnapshotParams:
+    """Create Daytona sandbox params from a single-service compose config.
 
     Args:
         config: Parsed compose configuration.
         compose_path: Path to the compose file for resolving relative paths.
             Pass None when using a ComposeConfig object directly.
-
-    Returns:
-        Tuple of (image, resources, sandbox_params) where:
-            - image: Docker image name or Image object
-            - resources: Resources config, or None if not specified
-            - sandbox_params: Extra sandbox params (env_vars, network settings, etc.)
+        labels: Labels to apply (merged with x-daytona labels).
     """
-    service = next((svc for svc in config.services.values() if svc.x_default), None)
-    if service is None:
-        service = config.services.get("default") or next(iter(config.services.values()))
+    _, service = find_default_service(config)
 
     compose_dir = Path(compose_path).parent if compose_path else Path.cwd()
 
@@ -45,7 +45,7 @@ def convert_compose_to_daytona_params(
     else:
         raise ValueError("Compose service must specify either 'image' or 'build'")
 
-    # Resources
+    # Resources (from service, may be overridden by x-daytona.resources below)
     resources = _service_to_resources(service)
 
     # Sandbox-level params
@@ -57,12 +57,87 @@ def convert_compose_to_daytona_params(
     if service.user:
         sandbox_params["os_user"] = service.user
 
-    _apply_daytona_extensions(sandbox_params, config.extensions)
+    # Translate Docker network_mode to Daytona network_block_all.
+    # Only set as a default; x-daytona extensions can override below.
+    if service.network_mode is not None and "network_block_all" not in sandbox_params:
+        sandbox_params["network_block_all"] = service.network_mode == "none"
+
+    apply_daytona_extensions(sandbox_params, config.extensions)
+
+    sandbox_params.setdefault("auto_stop_interval", 0)
+    x_labels = sandbox_params.pop("labels", {})
+    merged_labels = {**x_labels, **labels}
+
+    # x-daytona.resources overrides service-level resources
+    resources_override = sandbox_params.pop("resources", None)
+    if resources_override:
+        resources = Resources(
+            cpu=resources_override.get("cpu"),
+            memory=resources_override.get("memory"),
+            gpu=resources_override.get("gpu"),
+        )
+
+    # x-daytona.snapshot: use pre-built snapshot instead of building from image
+    snapshot = sandbox_params.pop("snapshot", None)
+    if snapshot:
+        return CreateSandboxFromSnapshotParams(
+            snapshot=snapshot,
+            labels=merged_labels,
+            **sandbox_params,
+        )
+
+    return CreateSandboxFromImageParams(
+        image=image,
+        resources=resources,
+        labels=merged_labels,
+        **sandbox_params,
+    )
+
+
+def find_default_service(config: ComposeConfig) -> tuple[str, ComposeService]:
+    """Find the default service in a compose config.
+
+    Priority: x-default: true -> service named "default" or "main" -> first service.
+
+    Returns:
+        Tuple of (service_name, service_config).
+    """
+    for name, svc in config.services.items():
+        if svc.x_default:
+            return name, svc
+    for candidate in ("default", "main"):
+        if candidate in config.services:
+            return candidate, config.services[candidate]
+    name = next(iter(config.services))
+    return name, config.services[name]
+
+
+def aggregate_resources(config: ComposeConfig) -> Resources | None:
+    """Sum per-service resources across all services + (Docker daemon) overhead."""
+    total_cpu = 0
+    total_memory = 0
+    total_gpu = 0
+    has_any = False
+
+    for svc in config.services.values():
+        r = _service_to_resources(svc)
+        if r:
+            has_any = True
+            total_cpu += r.cpu or 0
+            total_memory += r.memory or 0
+            total_gpu += r.gpu or 0
+
+    if not has_any:
+        return None
 
-    return image, resources, sandbox_params
+    return Resources(
+        cpu=total_cpu + 1,
+        memory=total_memory + 1,
+        gpu=total_gpu or None,
+    )
 
 
-def _apply_daytona_extensions(
+def apply_daytona_extensions(
     params: dict[str, Any], extensions: dict[str, Any]
 ) -> None:
     """Apply Daytona-specific extensions from x-daytona compose key.
@@ -83,6 +158,10 @@ def _apply_daytona_extensions(
             x-daytona values take precedence over service-level environment.
         - labels (dict): Custom labels. Merged by the caller with inspect's own labels,
             which take precedence.
+        - snapshot (str): Pre-created Daytona snapshot name. For single-service,
+            skips image building. For DinD, uses as the DinD VM snapshot.
+        - resources (dict): Sandbox-level resource overrides (cpu, memory, gpu).
+            For DinD, overrides the per-service aggregation.
         - volumes: Not yet supported.
 
     Args:
@@ -103,6 +182,8 @@ def _apply_daytona_extensions(
         "ephemeral",
         "timeout",
         "labels",
+        "snapshot",
+        "resources",
     ]
 
     for key in simple_keys:

inspect_sandboxes-0.2.0/src/inspect_sandboxes/daytona/_daytona.py

@@ -0,0 +1,314 @@
+"""Daytona sandbox provider"""
+
+from __future__ import annotations
+
+import os
+import sys
+import uuid
+from contextvars import ContextVar
+from logging import getLogger
+
+from daytona_sdk import (
+    AsyncDaytona,
+    CreateSandboxFromImageParams,
+    CreateSandboxFromSnapshotParams,
+    DaytonaNotFoundError,
+    Image,
+)
+from inspect_ai.util import (
+    ComposeConfig,
+    SandboxEnvironment,
+    SandboxEnvironmentConfigType,
+    is_compose_yaml,
+    is_dockerfile,
+    parse_compose_yaml,
+    sandboxenv,
+    trace_message,
+)
+from rich import box, print
+from rich.prompt import Confirm
+from rich.table import Table
+from typing_extensions import override
+
+from ._compose import create_single_service_params
+from ._dind_env import DaytonaDinDServiceEnvironment
+from ._sandbox_utils import (
+    close_client,
+    create_sandbox,
+    delete_sandbox,
+    list_sandboxes,
+)
+from ._single_env import DaytonaSingleServiceEnvironment
+
+logger = getLogger(__name__)
+
+INSPECT_SANDBOX_LABEL = {"created_by": "inspect-ai"}
+
+_daytona_client: ContextVar[AsyncDaytona | None] = ContextVar(
+    "daytona_client", default=None
+)
+_running_sandboxes: ContextVar[list[str]] = ContextVar("daytona_running_sandboxes")
+_run_id: ContextVar[str] = ContextVar("daytona_run_id")
+
+
+def _init_context() -> None:
+    _running_sandboxes.set([])
+    _run_id.set(uuid.uuid4().hex)
+
+
+def _run_labels() -> dict[str, str]:
+    return {**INSPECT_SANDBOX_LABEL, "inspect_run_id": _run_id.get()}
+
+
+@sandboxenv(name="daytona")
+class DaytonaSandboxEnvironment(SandboxEnvironment):
+    """Daytona sandbox provider
+
+    Owns all lifecycle class methods. sample_init returns instances of
+    DaytonaSingleServiceEnvironment or DaytonaDinDServiceEnvironment.
+    """
+
+    @classmethod
+    def config_files(cls) -> list[str]:
+        return [
+            "compose.yaml",
+            "compose.yml",
+            "docker-compose.yaml",
+            "docker-compose.yml",
+            "Dockerfile",
+        ]
+
+    @classmethod
+    def is_docker_compatible(cls) -> bool:
+        return True
+
+    @override
+    @classmethod
+    async def task_init(
+        cls,
+        task_name: str,
+        config: SandboxEnvironmentConfigType | None,
+    ) -> None:
+        _init_context()
+        client = AsyncDaytona()
+        _daytona_client.set(client)
+
+    @override
+    @classmethod
+    async def sample_init(
+        cls,
+        task_name: str,
+        config: SandboxEnvironmentConfigType | None,
+        metadata: dict[str, str],
+    ) -> dict[str, SandboxEnvironment]:
+        client = _daytona_client.get()
+        if client is None:
+            raise RuntimeError(
+                "Daytona client not initialized. task_init must be called first."
+            )
+
+        params: CreateSandboxFromSnapshotParams | CreateSandboxFromImageParams
+
+        if config is None:
+            params = CreateSandboxFromSnapshotParams(
+                snapshot=None,
+                labels=_run_labels(),
+                auto_stop_interval=0,
+            )
+        elif is_dockerfile(config):
+            image = Image.from_dockerfile(config)
+            params = CreateSandboxFromImageParams(
+                image=image,
+                labels=_run_labels(),
+                auto_stop_interval=0,
+            )
+        elif is_compose_yaml(config) or isinstance(config, ComposeConfig):
+            compose_config: ComposeConfig
+            compose_file: str | None
+            if is_compose_yaml(config):
+                compose_config = parse_compose_yaml(config, multiple_services=True)
+                compose_file = str(config)
+            else:
+                assert isinstance(config, ComposeConfig)
+                compose_config = config
+                compose_file = None
+
+            if len(compose_config.services) > 1:
+                envs = await DaytonaDinDServiceEnvironment.sample_init_dind(
+                    client, compose_config, compose_file, _run_labels()
+                )
+                any_env = next(iter(envs.values())).as_type(
+                    DaytonaDinDServiceEnvironment
+                )
+                _running_sandboxes.get().append(any_env.project.sandbox.id)
+                return envs
+            params = create_single_service_params(
+                compose_config, compose_file, _run_labels()
+            )
+        else:
+            raise ValueError(
+                f"Unrecognized config: {config}. "
+                "Expected a compose file (*.yaml/*.yml), Dockerfile, "
+                "ComposeConfig object, or None."
+            )
+
+        sandbox = await create_sandbox(client, params)
+        _running_sandboxes.get().append(sandbox.id)
+
+        return {"default": DaytonaSingleServiceEnvironment(sandbox)}
+
+    @override
+    @classmethod
+    async def sample_cleanup(
+        cls,
+        task_name: str,
+        config: SandboxEnvironmentConfigType | None,
+        environments: dict[str, SandboxEnvironment],
+        interrupted: bool,
+    ) -> None:
+        if not environments:
+            return
+
+        any_env = next(iter(environments.values()))
+        if isinstance(any_env, DaytonaDinDServiceEnvironment):
+            await DaytonaDinDServiceEnvironment.sample_cleanup(
+                task_name, config, environments, interrupted
+            )
+        else:
+            await DaytonaSingleServiceEnvironment.sample_cleanup(
+                task_name, config, environments, interrupted
+            )
+
+    @override
+    @classmethod
+    async def task_cleanup(
+        cls,
+        task_name: str,
+        config: SandboxEnvironmentConfigType | None,
+        cleanup: bool,
+    ) -> None:
+        if not cleanup:
+            return
+
+        client = _daytona_client.get()
+        if client is None:
+            return
+
+        failed_ids: list[str] = []
+        deleted_ids: set[str] = set()
+
+        for sandbox_id in _running_sandboxes.get().copy():
+            try:
+                sandbox = await client.get(sandbox_id)
+                await delete_sandbox(client, sandbox)
+                deleted_ids.add(sandbox_id)
+                trace_message(logger, "daytona", f"Deleted sandbox {sandbox_id}")
+            except DaytonaNotFoundError:
+                # Already deleted (e.g. by sample_cleanup) — nothing to do.
+                deleted_ids.add(sandbox_id)
+                trace_message(
+                    logger,
+                    "daytona",
+                    f"Sandbox {sandbox_id} already deleted, skipping.",
+                )
+            except Exception as e:
+                failed_ids.append(sandbox_id)
+                logger.error(f"Failed to delete sandbox {sandbox_id}: {e}")
+
+        # Second pass: find orphaned sandboxes by run label.
+        # This catches sandboxes where creation failed (e.g. build_failed)
+        # and the ID was never tracked in _running_sandboxes.
+        run_id = _run_id.get()
+        if run_id:
+            try:
+                orphans = await list_sandboxes(client, {"inspect_run_id": run_id})
+                for sb in orphans:
+                    if sb.id in deleted_ids:
+                        continue
+                    try:
+                        await delete_sandbox(client, sb)
+                        trace_message(
+                            logger,
+                            "daytona",
+                            f"Deleted orphaned sandbox {sb.id} (state={sb.state})",
+                        )
+                    except Exception as e:
+                        failed_ids.append(sb.id)
+                        logger.error(f"Failed to delete orphaned sandbox {sb.id}: {e}")
+            except Exception as e:
+                logger.warning(f"Failed to list sandboxes for cleanup: {e}")
+
+        if failed_ids:
+            logger.warning(
+                f"Failed to cleanup {len(failed_ids)} sandbox(es). "
+                f"Failed IDs: {', '.join(failed_ids)}"
+            )
+
+        _running_sandboxes.get().clear()
+
+        await close_client(client)
+        _daytona_client.set(None)
+
+    @override
+    @classmethod
+    async def cli_cleanup(cls, id: str | None) -> None:
+        client = AsyncDaytona()
+        try:
+            if id is not None:
+                # Single sandbox cleanup
+                try:
+                    sandbox = await client.get(id)
+                    await delete_sandbox(client, sandbox)
+                    print(f"Successfully deleted sandbox {id}")
+                except Exception as e:
+                    print(f"[red]Error deleting sandbox {id}: {e}[/red]")
+                    sys.exit(1)
+            else:
+                # Bulk cleanup
+                sandboxes = await list_sandboxes(client, INSPECT_SANDBOX_LABEL)
+
+                if not sandboxes:
+                    print("No Daytona sandboxes found to clean up.")
+                    return
+
+                table = Table(
+                    box=box.SQUARE,
+                    show_lines=False,
+                    title_style="bold",
+                    title_justify="left",
+                )
+                table.add_column("Sandbox ID")
+                for sb in sandboxes:
+                    table.add_row(sb.id)
+                print(table)
+
+                is_interactive = sys.stdin.isatty()
+                is_ci = "CI" in os.environ
+                is_pytest = "PYTEST_CURRENT_TEST" in os.environ
+
+                if is_interactive and not is_ci and not is_pytest:
+                    if not Confirm.ask(
+                        f"Are you sure you want to delete ALL {len(sandboxes)} sandbox(es) above?"
+                    ):
+                        print("Cancelled.")
+                        return
+
+                success_count = 0
+                failure_count = 0
+
+                for sb in sandboxes:
+                    try:
+                        await delete_sandbox(client, sb)
+                        success_count += 1
+                    except Exception as e:
+                        print(f"[yellow]Error deleting sandbox {sb.id}: {e}[/yellow]")
+                        failure_count += 1
+
+                print(f"\n[green]Successfully deleted: {success_count}[/green]")
+                if failure_count > 0:
+                    print(f"[red]Failed to delete: {failure_count}[/red]")
+                    sys.exit(1)
+                else:
+                    print("Complete.")
+        finally:
+            await close_client(client)