inspect-openshell-sandbox 0.1.0__py3-none-any.whl

inspect_openshell/__init__.py

@@ -0,0 +1,10 @@
+"""OpenShell sandbox environment for Inspect AI.
+
+Use ``sandbox="openshell"`` in a Task, with an ``openshell.yaml`` next to the
+task file to choose the image, working directory and policy.
+"""
+
+from ._sandbox import OpenShellSandboxEnvironment
+
+__all__ = ["OpenShellSandboxEnvironment"]
+__version__ = "0.1.0"

inspect_openshell/_sandbox.py

@@ -0,0 +1,266 @@
+"""Inspect sandbox provider backed by NVIDIA OpenShell.
+
+Each Inspect sample gets its own OpenShell sandbox, created from an image and
+governed by an OpenShell policy. Commands run through ``openshell sandbox exec``
+and files move through the same channel, so the kernel-level policy applies to
+everything the agent does, including the scorer's reads.
+
+Config (``openshell.yaml`` beside the task file, all keys optional):
+
+    image: python:3.12-slim   # passed to `openshell sandbox create --from`
+    build: images/python      # optional: docker build this dir (relative to the
+                              # config file) and tag it as `image` at task_init
+    workdir: /sandbox         # where sample files land and where exec runs.
+                              # Must exist in the image and be writable by the
+                              # sandbox user (UID 1000); /sandbox always is. On
+                              # the Docker driver the image's WORKDIR becomes the
+                              # workspace, so set both to the same path.
+    policy: policy.yaml       # passed to `openshell sandbox create --policy`
+    create_args: []           # extra flags for `openshell sandbox create`
+    exec_args: []             # extra flags for every `openshell sandbox exec`
+    hide_env: [OPENSHELL_SANDBOX]  # unset these in the agent's shell so the
+                              # environment does not announce itself; [] to keep
+
+Verified against the OpenShell CLI documentation (sandbox overview):
+``create --from <image>``, ``exec -n <name> [--env K=V] [--no-login-shell] -- cmd``
+with stdin piped and the command's exit status returned, ``delete <name>``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import os
+import shlex
+import shutil
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, Union, overload
+
+import yaml
+from inspect_ai.util import (
+    ExecResult,
+    SandboxConnection,
+    SandboxEnvironment,
+    sandboxenv,
+)
+
+CLI = "openshell"
+STDIN_LIMIT = 4 * 1024 * 1024  # the CLI caps piped stdin at 4 MiB
+
+
+@dataclass
+class OpenShellConfig:
+    image: str = "python:3.12-slim"
+    build: str | None = None
+    workdir: str = "/sandbox"
+    policy: str | None = None
+    create_args: list[str] = field(default_factory=list)
+    exec_args: list[str] = field(default_factory=list)
+    hide_env: list[str] = field(default_factory=lambda: ["OPENSHELL_SANDBOX"])
+
+    @classmethod
+    def load(cls, config: str | None) -> "OpenShellConfig":
+        if config is None:
+            return cls()
+        path = Path(config)
+        data = yaml.safe_load(path.read_text()) or {}
+        for key in ("policy", "build"):
+            if data.get(key) and not os.path.isabs(data[key]):
+                data[key] = str((path.parent / data[key]).resolve())
+        return cls(**data)
+
+
+async def _run(args: list[str], *, input: bytes | None = None, timeout: float | None = None) -> ExecResult[str]:
+    """Run a local `openshell ...` command and capture its result."""
+    proc = await asyncio.create_subprocess_exec(
+        *args,
+        stdin=asyncio.subprocess.PIPE if input is not None else asyncio.subprocess.DEVNULL,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    try:
+        out, err = await asyncio.wait_for(proc.communicate(input), timeout=timeout)
+    except asyncio.TimeoutError:
+        proc.kill()
+        await proc.wait()
+        raise TimeoutError(f"timed out after {timeout}s: {' '.join(args[:6])} ...")
+    return ExecResult(
+        success=proc.returncode == 0,
+        returncode=proc.returncode or 0,
+        stdout=out.decode("utf-8", errors="replace"),
+        stderr=err.decode("utf-8", errors="replace"),
+    )
+
+
+@sandboxenv(name="openshell")
+class OpenShellSandboxEnvironment(SandboxEnvironment):
+    """One OpenShell sandbox per sample."""
+
+    def __init__(self, name: str, config: OpenShellConfig) -> None:
+        super().__init__()
+        self.name = name
+        self.config = config
+
+    # ---- lifecycle -------------------------------------------------------
+
+    @classmethod
+    def config_files(cls) -> list[str]:
+        return ["openshell.yaml", "openshell.yml"]
+
+    @classmethod
+    def default_concurrency(cls) -> int | None:
+        return 4  # sandbox creation is heavier than docker; tune per gateway
+
+    @classmethod
+    async def task_init(cls, task_name: str, config: str | None) -> None:
+        if shutil.which(CLI) is None:
+            raise RuntimeError(
+                "the `openshell` CLI is not on PATH; install it from "
+                "https://github.com/NVIDIA/OpenShell and start a gateway"
+            )
+        cfg = OpenShellConfig.load(config)
+        if cfg.policy and not Path(cfg.policy).exists():
+            raise FileNotFoundError(f"openshell policy file not found: {cfg.policy}")
+        if cfg.build:
+            if not Path(cfg.build, "Dockerfile").exists():
+                raise FileNotFoundError(f"no Dockerfile in openshell build dir: {cfg.build}")
+            if shutil.which("docker") is None:
+                raise RuntimeError("`build:` is set but the docker CLI is not on PATH")
+            result = await _run(["docker", "build", "-t", cfg.image, cfg.build], timeout=1800)
+            if not result.success:
+                raise RuntimeError(f"docker build of {cfg.image} failed:\n{result.stderr[-2000:]}")
+
+    @classmethod
+    async def sample_init(
+        cls, task_name: str, config: str | None, metadata: dict[str, str]
+    ) -> dict[str, SandboxEnvironment]:
+        cfg = OpenShellConfig.load(config)
+        # The gateway caps sandbox names at 19 characters, so the task name
+        # cannot be part of it; "insp-" + 12 hex = 17.
+        name = f"insp-{uuid.uuid4().hex[:12]}"
+        args = [CLI, "sandbox", "create", "--name", name, "--from", cfg.image]
+        if cfg.policy:
+            args += ["--policy", cfg.policy]
+        args += cfg.create_args
+        result = await _run(args, timeout=600)
+        if not result.success:
+            raise RuntimeError(f"openshell sandbox create failed for {name}:\n{result.stderr or result.stdout}")
+        env = cls(name, cfg)
+        made = await env._exec_raw(["mkdir", "-p", cfg.workdir], timeout=60)
+        if not made.success:
+            await _run([CLI, "sandbox", "delete", name], timeout=120)
+            raise RuntimeError(
+                f"cannot create workdir {cfg.workdir!r} in the sandbox (runs as an unprivileged user; "
+                f"/sandbox is writable): {made.stderr.strip()}"
+            )
+        return {"default": env}
+
+    @classmethod
+    async def sample_cleanup(
+        cls,
+        task_name: str,
+        config: str | None,
+        environments: dict[str, SandboxEnvironment],
+        interrupted: bool,
+    ) -> None:
+        for env in environments.values():
+            if isinstance(env, OpenShellSandboxEnvironment):
+                await _run([CLI, "sandbox", "delete", env.name], timeout=300)
+
+    @classmethod
+    async def task_cleanup(cls, task_name: str, config: str | None, cleanup: bool) -> None:
+        return None
+
+    # ---- exec ------------------------------------------------------------
+
+    def _resolve(self, path: str) -> str:
+        return path if os.path.isabs(path) else os.path.join(self.config.workdir, path)
+
+    async def _exec_raw(
+        self,
+        cmd: list[str],
+        *,
+        input: bytes | None = None,
+        env: dict[str, str] | None = None,
+        timeout: float | None = None,
+    ) -> ExecResult[str]:
+        args = [CLI, "sandbox", "exec", "-n", self.name, "--no-login-shell"]
+        for k, v in (env or {}).items():
+            args += ["--env", f"{k}={v}"]
+        args += self.config.exec_args
+        args += ["--", *cmd]
+        return await _run(args, input=input, timeout=timeout)
+
+    async def exec(
+        self,
+        cmd: list[str],
+        input: str | bytes | None = None,
+        cwd: str | None = None,
+        env: dict[str, str] | None = None,
+        user: str | None = None,
+        timeout: int | None = None,
+        timeout_retry: bool = True,
+        concurrency: bool = True,
+    ) -> ExecResult[str]:
+        if user is not None:
+            raise NotImplementedError("openshell sandbox: per-command `user` is not supported")
+        workdir = self._resolve(cwd) if cwd else self.config.workdir
+        # Run inside a shell so cwd applies; the CLI itself already wraps in bash -c.
+        # OPENSHELL_* variables cannot be overridden via --env (reserved prefix), so
+        # the ones in hide_env are unset here, for the agent's process only.
+        unset = f"unset {' '.join(shlex.quote(v) for v in self.config.hide_env)} && " if self.config.hide_env else ""
+        script = f"{unset}cd {shlex.quote(workdir)} && {shlex.join(cmd)}"
+        data = input.encode() if isinstance(input, str) else input
+        if data is not None and len(data) > STDIN_LIMIT:
+            raise ValueError("openshell sandbox: piped stdin is limited to 4 MiB")
+        return await self._exec_raw(["bash", "-c", script], input=data, env=env, timeout=timeout)
+
+    # ---- files -----------------------------------------------------------
+
+    async def write_file(self, file: str, contents: str | bytes) -> None:
+        path = self._resolve(file)
+        data = contents.encode() if isinstance(contents, str) else contents
+        # base64 over stdin keeps binary safe and avoids any upload path semantics.
+        b64 = base64.b64encode(data)
+        if len(b64) > STDIN_LIMIT:
+            raise ValueError("openshell sandbox: write_file over 3 MiB is not supported yet; use upload")
+        script = f"mkdir -p {shlex.quote(os.path.dirname(path) or '/')} && base64 -d > {shlex.quote(path)}"
+        result = await self._exec_raw(["bash", "-c", script], input=b64, timeout=120)
+        if not result.success:
+            if "Permission denied" in result.stderr:
+                raise PermissionError(f"{file}: {result.stderr.strip()}")
+            if "Is a directory" in result.stderr:
+                raise IsADirectoryError(file)
+            raise RuntimeError(f"write_file {file} failed: {result.stderr.strip()}")
+
+    @overload
+    async def read_file(self, file: str, text: Literal[True] = True) -> str: ...
+    @overload
+    async def read_file(self, file: str, text: Literal[False]) -> bytes: ...
+
+    async def read_file(self, file: str, text: bool = True) -> Union[str, bytes]:
+        path = self._resolve(file)
+        result = await self._exec_raw(["bash", "-c", f"base64 < {shlex.quote(path)}"], timeout=120)
+        if not result.success:
+            err = result.stderr
+            if "No such file" in err:
+                raise FileNotFoundError(file)
+            if "Is a directory" in err:
+                raise IsADirectoryError(file)
+            if "Permission denied" in err:
+                raise PermissionError(file)
+            raise RuntimeError(f"read_file {file} failed: {err.strip()}")
+        data = base64.b64decode(result.stdout)
+        if text:
+            return data.decode("utf-8")
+        return data
+
+    # ---- interactive -----------------------------------------------------
+
+    async def connection(self, *, user: str | None = None) -> SandboxConnection:
+        return SandboxConnection(
+            type="openshell",
+            command=f"{CLI} sandbox exec -n {self.name} --tty -- /bin/bash",
+        )

inspect_openshell_sandbox-0.1.0.dist-info/METADATA

@@ -0,0 +1,121 @@
+Metadata-Version: 2.5
+Name: inspect-openshell-sandbox
+Version: 0.1.0
+Summary: An NVIDIA OpenShell sandbox environment for Inspect AI
+Project-URL: Homepage, https://github.com/32bitsret/inspect-openshell-sandbox
+Project-URL: Repository, https://github.com/32bitsret/inspect-openshell-sandbox
+Project-URL: Issues, https://github.com/32bitsret/inspect-openshell-sandbox/issues
+Project-URL: Changelog, https://github.com/32bitsret/inspect-openshell-sandbox/blob/main/CHANGELOG.md
+Author: Retnan Daser
+License: MIT
+License-File: LICENSE
+Keywords: agents,evals,inspect-ai,openshell,sandbox
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: inspect-ai>=0.3.236
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# inspect-openshell-sandbox
+
+An [NVIDIA OpenShell](https://github.com/NVIDIA/OpenShell) sandbox environment
+for [Inspect AI](https://inspect.aisi.org.uk/). Each sample runs in its own
+OpenShell sandbox: kernel-enforced file, syscall and network policy, with
+credentials that only work at approved endpoints. Inspect tasks need no change
+beyond `sandbox="openshell"`.
+
+## Status
+
+0.1.0, alpha. Tested with OpenShell 0.1.x on Docker Desktop (macOS, Apple
+Silicon) with the `grpc_endpoint` workaround for host networking, and Inspect
+AI 0.3.266. Linux hosts and the Podman, Kubernetes and VM drivers are
+untested; the CLI mapping should hold but defaults may differ.
+
+## Install
+
+```bash
+# OpenShell CLI + local gateway (see the OpenShell README for options)
+curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh
+
+# this provider, into the venv that runs inspect
+uv pip install inspect-openshell-sandbox        # from PyPI
+# or, from a checkout of this repo:
+uv pip install -e .
+```
+
+## Use
+
+Put an `openshell.yaml` next to the task file:
+
+```yaml
+image: spec-games-openshell:python   # openshell sandbox create --from
+build: ./images/python               # optional: docker build this dir + tag as `image` at task_init
+workdir: /space                      # must exist in the image, writable by UID 1000
+# policy: policy.yaml               # optional openshell policy for this task
+```
+
+Without `build:` and `workdir:` the defaults are `python:3.12-slim` and
+`/sandbox`, the unprivileged user's home. To use any other directory, the
+image must create it and `chown 1000:1000` it, and set it as `WORKDIR`: on the
+Docker driver OpenShell adopts the image's `WORKDIR` as the workspace. The
+bundled `images/python/Dockerfile` does this for `/space`.
+
+and in the task:
+
+```python
+Task(..., sandbox="openshell")
+```
+
+Inspect finds `openshell.yaml` automatically, the same way it finds
+`compose.yaml` for docker.
+
+## What it does
+
+| Inspect call | OpenShell |
+|---|---|
+| `sample_init` | `openshell sandbox create --name inspect-<task>-<id> --from <image> [--policy ...]` |
+| `exec(cmd, cwd, env, input, timeout)` | `openshell sandbox exec -n <name> --no-login-shell [--env K=V] -- bash -c "cd <cwd> && <cmd>"` |
+| `write_file` | base64 over stdin, `base64 -d > file` |
+| `read_file` | `base64 < file`, decoded locally |
+| `sample_cleanup` | `openshell sandbox delete <name>` |
+
+Files and commands all go through `exec`, so the sandbox policy governs the
+scorer's reads as well as the agent's writes.
+
+## Limits, first version
+
+- Sandboxes run as an unprivileged user (UID 1000); only `/sandbox` and below is writable unless the image says otherwise. A `workdir` outside it fails at `sample_init` with a clear error.
+- Sandbox names are capped at 19 characters by the gateway, so the task name is not part of the name.
+- The sandbox exposes `OPENSHELL_SANDBOX` in the environment. Since a model that can see it knows it is being sandboxed, the provider unsets it in the agent's shell by default (`hide_env`). `HOME` is the workdir. OpenShell's own directories under `/run/openshell` and similar are still present.
+- Per-command `user` is not supported.
+- `write_file` is capped at about 3 MiB (the CLI's 4 MiB stdin limit after
+  base64). Larger files should use `openshell sandbox upload`; not wired yet.
+- Sandbox creation is slower than docker. `default_concurrency` is 4.
+- No policy is applied unless `policy:` is set. The interesting use is a policy
+  that makes the test file read-only, so a spec edit is refused by the kernel
+  instead of detected afterwards.
+
+## Smoke test without a model
+
+```bash
+python scripts/smoke.py
+```
+
+Creates a sandbox from `python:3.12-slim`, writes a file, runs a command,
+reads the file back, deletes the sandbox, and prints each step.
+
+## Origin
+
+Built for [spec-games](https://github.com/32bitsret/spec-games), where it runs a
+contradictory-spec task with the test file locked read-only by an OpenShell
+policy. That task's `policy.yaml` and `Dockerfile` are a worked example.

inspect_openshell_sandbox-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+inspect_openshell/__init__.py,sha256=6HDtrK4C6O2UdE_7HbJjDXyeRBX0EXRgXT-0XUz0Hkk,309
+inspect_openshell/_sandbox.py,sha256=R-qor8_AB_Nm9EAOHHZ6loYjSwd6K1pkFrWTgO2e9Lo,11445
+inspect_openshell_sandbox-0.1.0.dist-info/METADATA,sha256=f9Q2U12zRhPz4-1893_cSh8XhV0bIkpHhx-tS_uEWI0,5256
+inspect_openshell_sandbox-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+inspect_openshell_sandbox-0.1.0.dist-info/entry_points.txt,sha256=8TjuobMAPIMUupX2POViMVO5Ugm2nwfhbJcBSnr4DOw,60
+inspect_openshell_sandbox-0.1.0.dist-info/licenses/LICENSE,sha256=PZfQ2kFRaVXfTfFFLfXflXy21OVtYekhkdjRxKM6yz8,1069
+inspect_openshell_sandbox-0.1.0.dist-info/RECORD,,

inspect_openshell_sandbox-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

inspect_openshell_sandbox-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[inspect_ai]
+openshell-sandbox = inspect_openshell._sandbox

inspect_openshell_sandbox-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Retnan Daser
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.