boxd-inspect 0.1.0__py3-none-any.whl

boxd_inspect/__init__.py

@@ -0,0 +1,7 @@
+"""Run Inspect AI sandboxes on boxd machines."""
+
+from ._config import BoxdSandboxEnvironmentConfig
+from ._environment import BoxdSandboxEnvironment
+from ._version import __version__
+
+__all__ = ["BoxdSandboxEnvironment", "BoxdSandboxEnvironmentConfig", "__version__"]

boxd_inspect/_commands.py

@@ -0,0 +1,203 @@
+"""Shell lines the machine runs for each sandbox operation.
+
+Every exec on a boxd machine runs one ``/bin/sh`` line as the machine's user
+``boxd``. These functions build those lines; they do no I/O, so the unit tests
+check them directly.
+"""
+
+from __future__ import annotations
+
+import re
+import shlex
+from dataclasses import dataclass
+
+SHELL_PATH = "/bin/sh"
+SYSTEM_PATH = "/usr/sbin:/usr/bin:/sbin:/bin"
+
+SUDO = "/usr/bin/sudo"
+TIMEOUT = "/usr/bin/timeout"
+ENV = "/usr/bin/env"
+TIMEOUT_KILL_AFTER = 5
+"""Seconds between the TERM and the KILL a timed-out command gets."""
+
+_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+# cd to $0, then run "$@": the directory and the argv never pass through a
+# shell's word splitting.
+_CD_AND_RUN = 'cd -- "$0" && exec "$@"'
+
+
+def check_env(env: dict[str, str] | None) -> None:
+    """Raise ``ValueError`` for a variable name a shell cannot assign."""
+    for name in env or {}:
+        if not _ENV_NAME.fullmatch(name):
+            raise ValueError(f"{name!r} is not an environment variable name")
+
+
+@dataclass(frozen=True)
+class Compose:
+    """A compose project inside the machine, and one of its services."""
+
+    project: str
+    directory: str
+    file: str
+    service: str
+
+    def base(self) -> list[str]:
+        return [
+            "docker",
+            "compose",
+            "--project-name",
+            self.project,
+            "--project-directory",
+            self.directory,
+            "--file",
+            self.file,
+        ]
+
+
+def exec_argv(
+    cmd: list[str],
+    *,
+    cwd: str,
+    env: dict[str, str] | None,
+    user: str | None,
+    timeout: int | None,
+    compose: Compose | None,
+) -> list[str]:
+    """Build the argv, run by the machine's shell, that runs ``cmd`` as Inspect asks.
+
+    On the machine: ``timeout`` and ``sudo`` by absolute path, then a shell that
+    changes to ``cwd`` (as ``user``, so a directory only they can enter works)
+    and ``exec``s ``cmd``. ``env`` goes through ``/usr/bin/env`` so it survives
+    ``sudo``. In a compose service: ``docker compose exec`` with ``--workdir``,
+    ``--user`` and ``--env``, and the same ``timeout`` inside the container so
+    the command's whole process tree is stopped when it runs out.
+    """
+    if not cmd:
+        raise ValueError("cmd must not be empty")
+    check_env(env)
+    limit = [TIMEOUT, "-k", f"{TIMEOUT_KILL_AFTER}s", f"{timeout}s"] if timeout is not None else []
+    if compose is not None:
+        argv = [*compose.base(), "exec", "-T", "--workdir", cwd]
+        if user is not None:
+            argv += ["--user", user]
+        for name, value in (env or {}).items():
+            argv += ["--env", f"{name}={value}"]
+        return [*argv, compose.service, *limit, *cmd]
+
+    run = [SHELL_PATH, "-c", _CD_AND_RUN, cwd]
+    if env:
+        run += [ENV, *(f"{name}={value}" for name, value in env.items())]
+    run += cmd
+    if user is not None:
+        return [SUDO, "-n", "-H", "-u", _sudo_user(user), "--", *limit, *run]
+    return [*limit, *run]
+
+
+def privileged_argv(
+    script: str,
+    *args: str,
+    user: str | None,
+    compose: Compose | None,
+) -> list[str]:
+    """Run a POSIX ``script`` (``$1``... are ``args``) as ``user`` with a pinned PATH.
+
+    For the file operations: their helper programs come from the system
+    directories, never from a PATH the sandbox's user can change.
+    """
+    inner = pinned_shell_command(script, *args)
+    if compose is not None:
+        argv = [*compose.base(), "exec", "-T"]
+        if user is not None:
+            argv += ["--user", user]
+        return [*argv, compose.service, *inner]
+    if user is not None:
+        return [SUDO, "-n", "-H", "-u", _sudo_user(user), "--", *inner]
+    return inner
+
+
+def shell_line(
+    argv: list[str],
+    *,
+    stdin: str | None = None,
+    stdout: str | None = None,
+    remove: list[str] | None = None,
+) -> str:
+    """One ``/bin/sh`` line: ``argv`` with redirections, then remove temp files.
+
+    The exit status is ``argv``'s own. ``remove`` lists temp files the line
+    deletes after the command, whatever its result.
+    """
+    line = shlex.join(argv)
+    if stdin is not None:
+        line += f" < {shlex.quote(stdin)}"
+    if stdout is not None:
+        line += f" > {shlex.quote(stdout)}"
+    if remove:
+        line += f"; rc=$?; /bin/rm -f -- {shlex.join(remove)}; exit $rc"
+    return line
+
+
+def script_line(script_path: str, *, remove: list[str] | None = None) -> str:
+    """Run an uploaded script (for commands too long for one argument)."""
+    line = f"{SHELL_PATH} {shlex.quote(script_path)}"
+    if remove:
+        line += f"; rc=$?; /bin/rm -f -- {shlex.join(remove)}; exit $rc"
+    return line
+
+
+def pinned_shell_command(script: str, *args: str) -> list[str]:
+    """``/bin/sh`` running ``script`` with PATH set to the system directories.
+
+    The same pin Inspect puts on its own privileged commands: what the script
+    runs never comes from a directory the sandbox's user can write to.
+    """
+    pin = f"PATH={SYSTEM_PATH}\nexport PATH\nunset CDPATH\n"
+    return [SHELL_PATH, "-c", pin + script, "sh", *args]
+
+
+def _sudo_user(user: str) -> str:
+    """``sudo -u`` takes a uid as ``#<uid>``."""
+    return f"#{user}" if user.isdigit() else user
+
+
+READ_SCRIPT = """\
+if [ -d "$1" ]; then echo "$1: Is a directory" >&2; exit 21; fi
+if [ -e "$1" ] && [ -r "$1" ]; then
+  size=$(stat -L -c %s -- "$1") || exit 1
+  if [ "$size" -gt "$2" ]; then echo "$1: larger than $2 bytes" >&2; exit 27; fi
+fi
+exec cat -- "$1"
+"""
+"""Print a file (``$1``) of at most ``$2`` bytes. A directory fails with
+``EISDIR`` (21), a larger file with ``EFBIG`` (27), before anything is sent."""
+
+WRITE_SCRIPT = """\
+parent=$(dirname -- "$1") || exit 1
+mkdir -p -- "$parent" || exit 1
+if [ -d "$1" ]; then echo "$1: Is a directory" >&2; exit 21; fi
+exec cat > "$1"
+"""
+"""Write stdin to a file, making its directory first. Writes through the file
+(and a symlink to it), so its owner and mode stay and its permissions apply."""
+
+PWD_SCRIPT = "exec pwd"
+
+__all__ = [
+    "ENV",
+    "PWD_SCRIPT",
+    "READ_SCRIPT",
+    "SHELL_PATH",
+    "SUDO",
+    "SYSTEM_PATH",
+    "TIMEOUT",
+    "WRITE_SCRIPT",
+    "Compose",
+    "check_env",
+    "exec_argv",
+    "pinned_shell_command",
+    "privileged_argv",
+    "script_line",
+    "shell_line",
+]

boxd_inspect/_compose.py

@@ -0,0 +1,157 @@
+"""A compose project to run with Docker inside the machine.
+
+The compose file (or a ``Dockerfile``, wrapped in the same compose file Inspect's
+Docker sandbox makes for one) goes to the machine with its directory, so build
+contexts next to it work as they do locally.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import tarfile
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+from inspect_ai.util import ComposeConfig, is_dockerfile, parse_compose_yaml
+
+GENERATED_FILE = ".boxd-inspect-compose.yaml"
+"""The compose file this package writes next to a ``Dockerfile``."""
+
+MAX_CONTEXT_BYTES = 512 * 1024 * 1024
+"""The largest directory (packed) sent to each machine."""
+
+DOCKERFILE_COMPOSE = """\
+services:
+  default:
+    build:
+      context: "."
+      dockerfile: "{dockerfile}"
+    command: "tail -f /dev/null"
+    init: true
+    network_mode: none
+    stop_grace_period: 1s
+"""
+"""What Inspect's Docker sandbox runs for a lone ``Dockerfile``."""
+
+
+@dataclass(frozen=True)
+class ComposeSource:
+    """What goes to the machine: a packed directory and the compose file in it."""
+
+    archive: bytes
+    file: str
+    services: list[str]
+    default_service: str
+
+
+_cache: dict[tuple[str, float], ComposeSource] = {}
+
+
+def compose_source(compose: str | ComposeConfig) -> ComposeSource:
+    """Pack the compose project once per file version; samples reuse the result."""
+    if isinstance(compose, ComposeConfig):
+        return _from_config(compose)
+    path = Path(compose).resolve()
+    key = (str(path), _directory_mtime(path.parent))
+    cached = _cache.get(key)
+    if cached is None:
+        cached = _from_file(path)
+        _cache[key] = cached
+    return cached
+
+
+def _from_file(path: Path) -> ComposeSource:
+    if not path.is_file():
+        raise FileNotFoundError(f"{path} does not exist")
+    extra: dict[str, bytes] = {}
+    if is_dockerfile(str(path)):
+        text = DOCKERFILE_COMPOSE.format(dockerfile=path.name)
+        extra[GENERATED_FILE] = text.encode()
+        config = ComposeConfig.model_validate(yaml.safe_load(text))
+        file = GENERATED_FILE
+    else:
+        config = parse_compose_yaml(str(path), multiple_services=True)
+        file = path.name
+    services = list(config.services)
+    return ComposeSource(
+        archive=_pack(path.parent, extra),
+        file=file,
+        services=services,
+        default_service=default_service(config),
+    )
+
+
+def _from_config(config: ComposeConfig) -> ComposeSource:
+    text = yaml.safe_dump(
+        config.model_dump(mode="json", by_alias=True, exclude_none=True), sort_keys=False
+    )
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w:gz") as tar:
+        _add_bytes(tar, GENERATED_FILE, text.encode())
+    return ComposeSource(
+        archive=buffer.getvalue(),
+        file=GENERATED_FILE,
+        services=list(config.services),
+        default_service=default_service(config),
+    )
+
+
+def default_service(config: ComposeConfig) -> str:
+    """Pick the service Inspect calls ``default``: ``x-default: true``, else the one named so."""
+    for name, service in config.services.items():
+        extra = service.model_extra or {}
+        if getattr(service, "x_default", None) is True or extra.get("x-default") is True:
+            return name
+    if "default" in config.services:
+        return "default"
+    raise ValueError(
+        "No 'default' service in the compose file. Name a service 'default' "
+        "or add 'x-default: true' to one."
+    )
+
+
+def _pack(directory: Path, extra: dict[str, bytes]) -> bytes:
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w:gz") as tar:
+        tar.add(directory, arcname=".", filter=_portable)
+        for name, data in extra.items():
+            _add_bytes(tar, name, data)
+    data = buffer.getvalue()
+    if len(data) > MAX_CONTEXT_BYTES:
+        raise ValueError(
+            f"{directory} packs to {len(data) // (1024 * 1024)} MiB, more than the "
+            f"{MAX_CONTEXT_BYTES // (1024 * 1024)} MiB sent to each machine. Move the "
+            "compose file to a smaller directory, or start from a snapshot."
+        )
+    return data
+
+
+def _portable(info: tarfile.TarInfo) -> tarfile.TarInfo | None:
+    """Own every entry by the machine's user; skip devices and version-control data."""
+    if info.isdev() or "/.git/" in f"/{info.name}/":
+        return None
+    info.uid = info.gid = 1000
+    info.uname = info.gname = "boxd"
+    return info
+
+
+def _add_bytes(tar: tarfile.TarFile, name: str, data: bytes) -> None:
+    info = tarfile.TarInfo(name)
+    info.size = len(data)
+    info.mode = 0o644
+    tar.addfile(info, io.BytesIO(data))
+
+
+def _directory_mtime(directory: Path) -> float:
+    """Find the newest change in a directory tree, so an edited build context is re-packed."""
+    newest = directory.stat().st_mtime
+    for root, dirs, files in os.walk(directory):
+        dirs[:] = [d for d in dirs if d != ".git"]
+        for name in files:
+            try:
+                newest = max(newest, os.stat(os.path.join(root, name)).st_mtime)
+            except OSError:
+                continue
+    return newest

boxd_inspect/_config.py

@@ -0,0 +1,148 @@
+"""The configuration of a ``boxd`` sandbox."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from inspect_ai.util import ComposeConfig, is_compose_yaml, is_dockerfile
+from pydantic import BaseModel, ConfigDict, Field
+
+REGIONS = ("fra", "nyc")
+SIZES = {1: "4G", 2: "8G", 4: "16G", 8: "32G"}
+
+_PREFIX = re.compile(r"[a-z][a-z0-9-]{0,19}")
+_LABEL_KEY = re.compile(r"[a-z0-9]([a-z0-9._/-]{0,61}[a-z0-9])?")
+
+
+class BoxdSandboxEnvironmentConfig(BaseModel):
+    """How each sample's boxd machine is made.
+
+    The API key is never part of the configuration (it would end up in the eval
+    log). Set ``BOXD_API_KEY``.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    compose: str | None = None
+    """A ``compose.yaml`` or ``Dockerfile`` to run with Docker inside the machine.
+
+    Without it, commands run on the machine itself (Ubuntu 24.04, user ``boxd``,
+    passwordless ``sudo``, Docker, Python and Node preinstalled).
+    """
+
+    region: str | None = None
+    """``fra`` or ``nyc``. Default: ``BOXD_REGION``, then your account's region."""
+
+    org: str | None = None
+    """The org to create machines in. Default: ``BOXD_ORG``, then the key's org."""
+
+    vcpu: int | None = None
+    """1, 2, 4 or 8. Memory follows (4, 8, 16 or 32 GiB) unless ``memory`` is set."""
+
+    memory: str | None = None
+    """``4G``, ``8G``, ``16G`` or ``32G``. Default: the org's machine size."""
+
+    from_snapshot: str | None = None
+    """Start each sample from this snapshot (for example one with images already pulled)."""
+
+    isolated: bool = True
+    """Run the eval's untrusted commands on an isolated machine: no peers on your
+    org's private network, and none of the in-machine boxd credentials, CLI or
+    connected integrations a non-isolated machine of yours gets. Outbound
+    internet still works. Turn it off only for code you trust."""
+
+    working_dir: str | None = None
+    """Where commands run and relative paths resolve. Default: ``/home/boxd`` on
+    the machine, the container's working directory with ``compose``."""
+
+    name_prefix: str = "inspect"
+    """Machine names start with this, then the task, the sample and a random suffix."""
+
+    labels: dict[str, str] = Field(default_factory=dict)
+    """Extra labels on every machine, next to ``inspect-run`` and ``inspect-task``."""
+
+    delete_after: int | None = None
+    """Seconds after its creation at which boxd deletes a machine, whatever it is
+    doing: a safety net for machines an eval that was killed never cleaned up.
+    Keep it above your longest sample."""
+
+    ready_timeout: float = 180
+    """Seconds to wait for a new machine to answer."""
+
+    compose_timeout: float = 900
+    """Seconds ``docker compose up`` may take (pulls and builds included)."""
+
+    def __hash__(self) -> int:
+        # Inspect keeps configurations in dicts; the dict field is not hashable.
+        return hash(self.model_dump_json())
+
+    def check(self) -> None:
+        """Raise ``ValueError`` for a value boxd would refuse, before any request."""
+        if self.region is not None and self.region not in REGIONS:
+            raise ValueError(f"region is one of {', '.join(REGIONS)}, not {self.region!r}")
+        if self.vcpu is not None and self.vcpu not in SIZES:
+            raise ValueError(f"vcpu is one of 1, 2, 4 or 8, not {self.vcpu}")
+        if self.memory is not None and self.memory not in SIZES.values():
+            raise ValueError(f"memory is one of 4G, 8G, 16G or 32G, not {self.memory!r}")
+        if not _PREFIX.fullmatch(self.name_prefix):
+            raise ValueError(
+                "name_prefix is 1 to 20 lowercase letters, digits or '-', starting with a letter"
+            )
+        if self.working_dir is not None and not self.working_dir.startswith("/"):
+            raise ValueError("working_dir is an absolute path")
+        for key, value in self.labels.items():
+            if not _LABEL_KEY.fullmatch(key) or key.startswith("boxd/"):
+                raise ValueError(
+                    f"label key {key!r} is 1 to 63 of a-z, 0-9, '.', '_', '/', '-', "
+                    "starting and ending with a letter or digit, and not under 'boxd/'"
+                )
+            if len(value) > 256 or any(ord(c) < 32 or ord(c) == 127 for c in value):
+                raise ValueError(
+                    f"label {key} has a value over 256 characters or a control character"
+                )
+        if self.compose is not None and not (
+            is_compose_yaml(self.compose) or is_dockerfile(self.compose)
+        ):
+            raise ValueError(f"compose is a compose.yaml or a Dockerfile, not {self.compose!r}")
+
+    def machine_memory(self) -> str | None:
+        """``memory``, or the size that goes with ``vcpu``."""
+        if self.memory is not None:
+            return self.memory
+        return SIZES[self.vcpu] if self.vcpu is not None else None
+
+
+Resolved = tuple[BoxdSandboxEnvironmentConfig, "str | ComposeConfig | None"]
+
+
+def resolve_config(config: object) -> Resolved:
+    """Turn what Inspect passes as ``config`` into our settings and a compose source.
+
+    Inspect passes nothing, our object, a ``ComposeConfig``, or a path: a
+    ``compose.yaml`` or ``Dockerfile`` (run with Docker in the machine) or a
+    JSON file holding our settings.
+    """
+    compose: str | ComposeConfig | None = None
+    if config is None:
+        resolved = BoxdSandboxEnvironmentConfig()
+    elif isinstance(config, BoxdSandboxEnvironmentConfig):
+        resolved = config
+    elif isinstance(config, ComposeConfig) or (
+        isinstance(config, str) and (is_compose_yaml(config) or is_dockerfile(config))
+    ):
+        resolved = BoxdSandboxEnvironmentConfig()
+        compose = config
+    elif isinstance(config, str) and config.endswith(".json"):
+        resolved = BoxdSandboxEnvironmentConfig.model_validate(json.loads(Path(config).read_text()))
+    else:
+        raise TypeError(
+            "The boxd sandbox takes a BoxdSandboxEnvironmentConfig, a ComposeConfig, "
+            "or the path of a compose.yaml, a Dockerfile or a JSON file, "
+            f"not {config!r}"
+        )
+    resolved.check()
+    if compose is None and resolved.compose is not None:
+        compose = resolved.compose
+    return resolved, compose