agentdocker-lite 0.0.1__tar.gz

agentdocker_lite-0.0.1/.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build
+      - run: python -m build
+      - uses: pypa/gh-action-pypi-publish@release/v1

agentdocker_lite-0.0.1/.gitignore

@@ -0,0 +1 @@
+__pycache__/

agentdocker_lite-0.0.1/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: agentdocker-lite
+Version: 0.0.1
+Summary: Lightweight Linux namespace sandbox with persistent shell and instant reset
+License-Expression: MIT
+Requires-Python: >=3.10

agentdocker_lite-0.0.1/README.md

@@ -0,0 +1,115 @@
+# agentdocker-lite
+
+Lightweight Linux namespace sandbox with persistent shell and instant filesystem reset.
+
+**20x faster lifecycle** than Docker. Designed for high-frequency workloads like RL training where environments are created, reset, and destroyed thousands of times.
+
+## Key features
+
+- **Persistent shell**: ~42ms per command (vs ~330ms with fork/exec/chroot per command)
+- **Instant reset**: overlayfs ~27ms, btrfs ~28ms -- clear filesystem without recreating the sandbox
+- **Fast lifecycle**: ~4ms create, ~6ms delete (overlayfs)
+- **Signal-pipe protocol**: uses a separate fd for command completion signaling -- no sentinel collision with command output
+- **CoW filesystem backends**: overlayfs (default) or btrfs snapshots
+- **cgroup v2**: optional CPU, memory, PID limits
+- **Auto rootfs**: pass a Docker image name, rootfs is auto-prepared and cached
+
+## Requirements
+
+- Linux with kernel supporting overlayfs (or btrfs)
+- Root or `CAP_SYS_ADMIN` (for mount/cgroup)
+- `util-linux` (`unshare`)
+- Docker (only for auto-preparing rootfs from image names)
+- Python >= 3.10
+
+## Install
+
+```bash
+cd agentdocker-lite
+pip install -e .
+```
+
+## Quick start
+
+```python
+from agentdocker_lite import Sandbox, SandboxConfig
+
+config = SandboxConfig(
+    image="ubuntu:22.04",       # Docker image or path to rootfs dir
+    working_dir="/workspace",
+)
+
+sb = Sandbox(config, name="worker-0")
+
+# Run commands (~42ms each)
+output, ec = sb.run("echo hello world")
+
+# Direct file I/O (bypasses shell)
+sb.write_file("/workspace/test.txt", "content")
+content = sb.read_file("/workspace/test.txt")
+
+# Reset filesystem to initial state (~27ms)
+sb.reset()
+
+# Cleanup
+sb.delete()
+```
+
+## Configuration
+
+```python
+SandboxConfig(
+    image="ubuntu:22.04",           # Docker image or rootfs path
+    working_dir="/workspace",       # Initial cwd inside sandbox
+    environment={"FOO": "bar"},     # Extra env vars
+    volumes=["/host/path:/container/path:ro"],  # Bind mounts
+    fs_backend="overlayfs",         # "overlayfs" or "btrfs"
+    env_base_dir="/tmp/agentdocker_lite",
+    rootfs_cache_dir="/tmp/agentdocker_lite_rootfs_cache",
+    cpu_max="50000 100000",         # cgroup cpu.max
+    memory_max="536870912",         # cgroup memory.max (bytes)
+    pids_max="256",                 # cgroup pids.max
+)
+```
+
+## API
+
+| Method | Description |
+|--------|-------------|
+| `sb.run(cmd, timeout=None)` | Run command, returns `(output, exit_code)` |
+| `sb.reset()` | Reset filesystem to initial state |
+| `sb.delete()` | Full cleanup (unmount, remove cgroup, delete files) |
+| `sb.copy_to(local, container)` | Copy file into sandbox |
+| `sb.copy_from(container, local)` | Copy file out of sandbox |
+| `sb.read_file(path)` | Read file content |
+| `sb.write_file(path, content)` | Write file content |
+| `sb.rootfs` | Host path to sandbox rootfs |
+
+## Examples
+
+```bash
+# Basic usage
+sudo python examples/basic_usage.py
+
+# 32-worker concurrent benchmark
+sudo python examples/concurrent_sandboxes.py
+```
+
+## Architecture
+
+```
+Host kernel (shared)
+  |
+  +-- Sandbox "worker-0"
+  |     +-- PID namespace (unshare --pid)
+  |     +-- Mount namespace (unshare --mount)
+  |     +-- chroot into overlayfs rootfs
+  |     |     +-- lowerdir: shared base image (read-only)
+  |     |     +-- upperdir: per-sandbox changes (cleared on reset)
+  |     +-- Persistent bash process (stdin/stdout pipes + signal fd)
+  |     +-- cgroup v2 limits
+  |
+  +-- Sandbox "worker-1"
+  |     +-- (same structure, independent namespaces)
+  ...
+```

agentdocker_lite-0.0.1/examples/basic_usage.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Basic usage example for agentdocker-lite.
+
+Must be run as root (requires mount/cgroup operations).
+Requires Docker to auto-prepare rootfs from image names.
+"""
+
+from agentdocker_lite import Sandbox, SandboxConfig
+
+
+def main():
+    # Create a sandbox from a Docker image (auto-exports to rootfs on first use).
+    # Or pass a path to an existing rootfs directory.
+    config = SandboxConfig(
+        image="ubuntu:22.04",
+        working_dir="/workspace",
+        cpu_max="50000 100000",  # 50% of one core
+        memory_max="536870912",  # 512 MB
+        pids_max="256",
+    )
+
+    sb = Sandbox(config, name="demo")
+
+    # Run commands (~42ms per command via persistent shell).
+    output, ec = sb.run("echo hello from sandbox")
+    print(f"[exit={ec}] {output.strip()}")
+
+    output, ec = sb.run("cat /etc/os-release | head -2")
+    print(f"[exit={ec}] {output.strip()}")
+
+    # Write and read files directly (bypasses shell, even faster).
+    sb.write_file("/workspace/test.txt", "hello world\n")
+    content = sb.read_file("/workspace/test.txt")
+    print(f"File content: {content.strip()}")
+
+    # Reset filesystem to initial state (~27ms).
+    sb.reset()
+
+    # File is gone after reset.
+    output, ec = sb.run("cat /workspace/test.txt 2>&1")
+    print(f"After reset [exit={ec}]: {output.strip()}")
+
+    # Clean up.
+    sb.delete()
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()

agentdocker_lite-0.0.1/examples/concurrent_sandboxes.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Concurrent sandbox example: create N sandboxes, run commands, reset, destroy."""
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from agentdocker_lite import Sandbox, SandboxConfig
+
+
+def worker(worker_id: int) -> dict:
+    config = SandboxConfig(
+        image="ubuntu:22.04",
+        working_dir="/workspace",
+        pids_max="128",
+    )
+
+    t0 = time.monotonic()
+    sb = Sandbox(config, name=f"worker-{worker_id}")
+    create_ms = (time.monotonic() - t0) * 1000
+
+    t0 = time.monotonic()
+    output, ec = sb.run("echo hello && uname -r")
+    run_ms = (time.monotonic() - t0) * 1000
+
+    t0 = time.monotonic()
+    sb.reset()
+    reset_ms = (time.monotonic() - t0) * 1000
+
+    t0 = time.monotonic()
+    sb.delete()
+    delete_ms = (time.monotonic() - t0) * 1000
+
+    return {
+        "worker": worker_id,
+        "create_ms": create_ms,
+        "run_ms": run_ms,
+        "reset_ms": reset_ms,
+        "delete_ms": delete_ms,
+    }
+
+
+def main():
+    n_workers = 32
+    print(f"Launching {n_workers} sandboxes concurrently...")
+
+    t0 = time.monotonic()
+    with ThreadPoolExecutor(max_workers=n_workers) as pool:
+        results = list(pool.map(worker, range(n_workers)))
+    wall_ms = (time.monotonic() - t0) * 1000
+
+    print(f"\n{'worker':>8} {'create':>10} {'run':>10} {'reset':>10} {'delete':>10}")
+    for r in results:
+        print(
+            f"{r['worker']:>8} {r['create_ms']:>9.1f}ms {r['run_ms']:>9.1f}ms "
+            f"{r['reset_ms']:>9.1f}ms {r['delete_ms']:>9.1f}ms"
+        )
+
+    avg_create = sum(r["create_ms"] for r in results) / len(results)
+    avg_reset = sum(r["reset_ms"] for r in results) / len(results)
+    print(f"\nAvg create: {avg_create:.1f}ms  Avg reset: {avg_reset:.1f}ms")
+    print(f"Wall clock: {wall_ms:.0f}ms for {n_workers} workers")
+
+
+if __name__ == "__main__":
+    main()

agentdocker_lite-0.0.1/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "agentdocker-lite"
+version = "0.0.1"
+description = "Lightweight Linux namespace sandbox with persistent shell and instant reset"
+requires-python = ">=3.10"
+license = "MIT"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agentdocker_lite"]

agentdocker_lite-0.0.1/src/agentdocker_lite/__init__.py

@@ -0,0 +1,6 @@
+"""agentdocker-lite: Lightweight Linux namespace sandbox for high-frequency workloads."""
+
+from agentdocker_lite.sandbox import Sandbox, SandboxConfig
+
+__all__ = ["Sandbox", "SandboxConfig"]
+__version__ = "0.0.1"

agentdocker_lite-0.0.1/src/agentdocker_lite/rootfs.py

@@ -0,0 +1,242 @@
+"""Utilities for preparing base rootfs directories from Docker images."""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def prepare_rootfs_from_docker(
+    image_name: str,
+    output_dir: str | Path,
+    pull: bool = True,
+) -> Path:
+    """Export a Docker image as a rootfs directory.
+
+    Equivalent to::
+
+        docker pull <image_name>
+        docker export $(docker create <image_name>) | tar -C <output_dir> -xf -
+
+    Args:
+        image_name: Docker image (e.g. ``"ubuntu:22.04"``).
+        output_dir: Target directory for the extracted rootfs.
+        pull: Pull the image first (set ``False`` if already local).
+
+    Returns:
+        Path to *output_dir*.
+
+    Raises:
+        RuntimeError: If any Docker/tar command fails.
+    """
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if pull:
+        logger.info("Pulling image: %s", image_name)
+        result = subprocess.run(
+            ["docker", "pull", image_name],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"docker pull failed: {result.stderr.strip()}")
+
+    logger.info("Creating temporary container from %s", image_name)
+    create = subprocess.run(
+        ["docker", "create", image_name],
+        capture_output=True,
+        text=True,
+    )
+    if create.returncode != 0:
+        raise RuntimeError(f"docker create failed: {create.stderr.strip()}")
+    container_id = create.stdout.strip()
+
+    try:
+        logger.info("Exporting %s -> %s", image_name, output_dir)
+        export_proc = subprocess.Popen(
+            ["docker", "export", container_id],
+            stdout=subprocess.PIPE,
+        )
+        tar_proc = subprocess.Popen(
+            ["tar", "-C", str(output_dir), "-xf", "-"],
+            stdin=export_proc.stdout,
+        )
+        if export_proc.stdout is not None:
+            export_proc.stdout.close()
+        tar_proc.communicate()
+
+        if tar_proc.returncode != 0:
+            raise RuntimeError(
+                f"tar extraction failed for {image_name} (exit {tar_proc.returncode})"
+            )
+    finally:
+        subprocess.run(["docker", "rm", "-f", container_id], capture_output=True)
+
+    subprocess.run(["docker", "rmi", "-f", image_name], capture_output=True)
+
+    logger.info("Rootfs ready: %s", output_dir)
+    return output_dir
+
+
+def prepare_rootfs_without_docker(
+    image_ref: str,
+    output_dir: str | Path,
+) -> Path:
+    """Export an OCI image as a rootfs without requiring a Docker daemon.
+
+    Uses ``skopeo`` + ``umoci`` which can run without root and without
+    a running Docker daemon.
+
+    Args:
+        image_ref: OCI image reference (e.g. ``"docker://ubuntu:22.04"``).
+        output_dir: Target directory for the extracted rootfs.
+
+    Returns:
+        Path to the rootfs bundle directory.
+
+    Raises:
+        RuntimeError: If skopeo/umoci commands fail.
+        FileNotFoundError: If skopeo or umoci is not installed.
+    """
+    import shutil
+    import tempfile
+
+    for tool in ("skopeo", "umoci"):
+        if shutil.which(tool) is None:
+            raise FileNotFoundError(
+                f"{tool} not found. Install it: apt-get install {tool}"
+            )
+
+    output_dir = Path(output_dir)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        oci_dir = Path(tmp) / "oci_image"
+
+        if not image_ref.startswith("docker://"):
+            image_ref = f"docker://{image_ref}"
+
+        logger.info("Copying %s via skopeo", image_ref)
+        result = subprocess.run(
+            ["skopeo", "copy", image_ref, f"oci:{oci_dir}:latest"],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"skopeo copy failed: {result.stderr.strip()}")
+
+        logger.info("Unpacking OCI image via umoci -> %s", output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        result = subprocess.run(
+            ["umoci", "unpack", "--image", f"{oci_dir}:latest", str(output_dir)],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"umoci unpack failed: {result.stderr.strip()}")
+
+    rootfs_path = output_dir / "rootfs"
+    if rootfs_path.is_dir():
+        logger.info("Rootfs ready: %s", rootfs_path)
+        return rootfs_path
+
+    logger.info("Rootfs ready: %s", output_dir)
+    return output_dir
+
+
+def prepare_btrfs_rootfs_from_docker(
+    image_name: str,
+    subvolume_path: str | Path,
+    pull: bool = True,
+) -> Path:
+    """Export a Docker image into a btrfs subvolume for snapshot-based sandboxes.
+
+    The target path must be on a btrfs-formatted filesystem.
+    """
+    import shutil as _shutil
+
+    if _shutil.which("btrfs") is None:
+        raise FileNotFoundError(
+            "btrfs-progs not found. Install: apt-get install btrfs-progs"
+        )
+
+    subvolume_path = Path(subvolume_path)
+
+    if subvolume_path.exists():
+        check = subprocess.run(
+            ["btrfs", "subvolume", "show", str(subvolume_path)],
+            capture_output=True,
+            text=True,
+        )
+        if check.returncode == 0:
+            logger.info("Deleting existing btrfs subvolume: %s", subvolume_path)
+            subprocess.run(
+                ["btrfs", "subvolume", "delete", str(subvolume_path)],
+                capture_output=True,
+            )
+        else:
+            _shutil.rmtree(subvolume_path)
+
+    subvolume_path.parent.mkdir(parents=True, exist_ok=True)
+    result = subprocess.run(
+        ["btrfs", "subvolume", "create", str(subvolume_path)],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"btrfs subvolume create failed: {result.stderr.strip()}. "
+            f"Ensure {subvolume_path.parent} is on a btrfs filesystem."
+        )
+    logger.info("Created btrfs subvolume: %s", subvolume_path)
+
+    if pull:
+        logger.info("Pulling image: %s", image_name)
+        result = subprocess.run(
+            ["docker", "pull", image_name],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"docker pull failed: {result.stderr.strip()}")
+
+    logger.info("Creating temporary container from %s", image_name)
+    create = subprocess.run(
+        ["docker", "create", image_name],
+        capture_output=True,
+        text=True,
+    )
+    if create.returncode != 0:
+        raise RuntimeError(f"docker create failed: {create.stderr.strip()}")
+    container_id = create.stdout.strip()
+
+    try:
+        logger.info(
+            "Exporting %s -> %s (btrfs subvolume)", image_name, subvolume_path
+        )
+        export_proc = subprocess.Popen(
+            ["docker", "export", container_id],
+            stdout=subprocess.PIPE,
+        )
+        tar_proc = subprocess.Popen(
+            ["tar", "-C", str(subvolume_path), "-xf", "-"],
+            stdin=export_proc.stdout,
+        )
+        if export_proc.stdout is not None:
+            export_proc.stdout.close()
+        tar_proc.communicate()
+
+        if tar_proc.returncode != 0:
+            raise RuntimeError(
+                f"tar extraction failed for {image_name} (exit {tar_proc.returncode})"
+            )
+    finally:
+        subprocess.run(["docker", "rm", "-f", container_id], capture_output=True)
+
+    subprocess.run(["docker", "rmi", "-f", image_name], capture_output=True)
+
+    logger.info("btrfs rootfs ready: %s", subvolume_path)
+    return subvolume_path