levain-runtime 0.1.2__tar.gz → 0.1.10__tar.gz

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/.gitignore

@@ -18,6 +18,10 @@ build/
 !**/platform_agents/build/
 !**/src/build/
 
+# Maven build output
+services/flink/target/
+services/flink/dependency-reduced-pom.xml
+
 # Binaries
 bin/
 
@@ -136,3 +140,6 @@ assets/video/*/renders/
 # www/public/videos/, the composition that made them does not.
 assets/video/updates-*/
 assets/video/*/snapshots/
+
+# Maven build output (Flink job); the image build compiles from source.
+services/flink/target/

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: levain-runtime
-Version: 0.1.2
+Version: 0.1.10
 Summary: Self-hosted runtime for Levain agents: run an agent's nodes on your own machine or container while the platform drives the recipe
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -34,9 +34,12 @@ uv tool install levain-runtime
 ## Set up
 
 ```sh
-# Run nodes directly on this host (one active session at a time):
+# Run nodes directly on this host:
 levain-runtime init --mode direct
 
+# Direct mode serving four sessions at once (Linux with bubblewrap):
+levain-runtime init --mode direct --sessions 4
+
 # Or run each session in a container built from your own image:
 levain-runtime init --mode docker --image ghcr.io/you/agent-runtime:latest
 ```
@@ -76,6 +79,9 @@ and `/sdk`, then runs each node as a short-lived process with the session's
 environment. Everything a node prints is streamed back and lands in the run's
 logs on the platform.
 
-Direct mode allows one active session at a time; `/workspace` points at the
-active session. Docker mode starts one container per session and has no such
-limit.
+Direct mode serves one session at a time by default: `/workspace` points at
+the active session. With `--sessions N` it serves up to N at once, each node
+running under bubblewrap with its session's folders mounted at `/workspace`,
+`/recipes/active` and `/sdk/active`; the real workspace path is in
+`LEVAIN_HOST_WORKSPACE` for tools outside the sandbox such as Docker. Docker
+mode starts one container per session and has no such limit.

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/README.md

@@ -19,9 +19,12 @@ uv tool install levain-runtime
 ## Set up
 
 ```sh
-# Run nodes directly on this host (one active session at a time):
+# Run nodes directly on this host:
 levain-runtime init --mode direct
 
+# Direct mode serving four sessions at once (Linux with bubblewrap):
+levain-runtime init --mode direct --sessions 4
+
 # Or run each session in a container built from your own image:
 levain-runtime init --mode docker --image ghcr.io/you/agent-runtime:latest
 ```
@@ -61,6 +64,9 @@ and `/sdk`, then runs each node as a short-lived process with the session's
 environment. Everything a node prints is streamed back and lands in the run's
 logs on the platform.
 
-Direct mode allows one active session at a time; `/workspace` points at the
-active session. Docker mode starts one container per session and has no such
-limit.
+Direct mode serves one session at a time by default: `/workspace` points at
+the active session. With `--sessions N` it serves up to N at once, each node
+running under bubblewrap with its session's folders mounted at `/workspace`,
+`/recipes/active` and `/sdk/active`; the real workspace path is in
+`LEVAIN_HOST_WORKSPACE` for tools outside the sandbox such as Docker. Docker
+mode starts one container per session and has no such limit.

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/chisel.py

@@ -40,7 +40,7 @@ def _asset_name() -> str:
 
 
 def download(root: Path) -> Path:
-    """Fetch the pinned release into ``<root>/bin`` after verifying its checksum."""
+    """Fetch the pinned release to :func:`binary_path` after verifying its checksum."""
     name = _asset_name()
     base = f"{_RELEASES}/v{CHISEL_VERSION}"
     with httpx.Client(follow_redirects=True, timeout=120) as client:
@@ -58,7 +58,7 @@ def download(root: Path) -> Path:
     actual = hashlib.sha256(blob.content).hexdigest()
     if actual != expected:
         raise RuntimeError(f"chisel checksum mismatch: {actual} != {expected}")
-    target = root / "bin" / "chisel"
+    target = binary_path(root)
     target.parent.mkdir(parents=True, exist_ok=True)
     tmp = target.with_suffix(".tmp")
     tmp.write_bytes(gzip.decompress(blob.content))
@@ -67,11 +67,16 @@ def download(root: Path) -> Path:
     return target
 
 
+def binary_path(root: Path) -> Path:
+    """Where the pinned release lives; named by version so a bump is fetched."""
+    return root / "bin" / f"chisel-{CHISEL_VERSION}"
+
+
 def resolve_binary(root: Path) -> Path:
     override = os.environ.get("CHISEL_BIN")
     if override:
         return Path(override)
-    local = root / "bin" / "chisel"
+    local = binary_path(root)
     if local.is_file():
         return local
     log.info("downloading chisel %s", CHISEL_VERSION)
@@ -90,6 +95,15 @@ def server_url(gateway_url: str) -> str:
     return gateway_url
 
 
+def gateway(creds: Credentials) -> str:
+    """The gateway to dial: the platform's, unless this host overrides it.
+
+    A runner on the platform cluster's own node dials the gateway Service
+    directly instead of looping out through the public edge and back.
+    """
+    return os.environ.get("LEVAIN_RUNTIME_GATEWAY_URL") or creds.gateway_url
+
+
 def client_argv(binary: Path, creds: Credentials) -> list[str]:
     remotes = [f"R:{creds.reverse_port}:127.0.0.1:{RUNNER_HTTP_PORT}"]
     remotes += [f"{f.local_port}:{f.remote}" for f in creds.forwards]
@@ -102,7 +116,7 @@ def client_argv(binary: Path, creds: Credentials) -> list[str]:
         # default lets the reconnect wait grow to five minutes.
         "--max-retry-interval",
         "30s",
-        server_url(creds.gateway_url),
+        server_url(gateway(creds)),
         *remotes,
     ]
 

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/cli.py

@@ -7,11 +7,13 @@ import json
 import logging
 import os
 import shlex
+import shutil
 import subprocess
 import sys
+import time
 from importlib import resources
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Any
 
 import httpx
 import typer
@@ -22,7 +24,7 @@ from .config import Credentials, Mode, RuntimeConfig, runtime_home
 from .heartbeat import package_version, run_heartbeat
 from .protocol import RUNNER_HTTP_PORT
 from .server import create_app
-from .sessions import SessionManager
+from .sessions import BWRAP, SessionManager
 
 app = typer.Typer(
     name="levain-runtime",
@@ -34,6 +36,9 @@ _HOME_DIRS = ("recipes", "sdk", "sandbox", "sessions", "agents", "bin")
 _SYSTEM_LINKS = ("recipes", "sdk", "sandbox", "workspace")
 _SYNTHETIC_CONF = Path("/etc/synthetic.conf")
 _VENDORED = ("node_runner.py", "run_logging.py")
+# Paths a sandboxed direct runtime binds per session, relative to the
+# home; the root-level links point into it.
+_SESSION_VIEWS = ("workspace", "recipes/active", "sdk/active")
 
 
 def _version_callback(value: bool) -> None:
@@ -71,10 +76,29 @@ def write_sandbox_files(root: Path) -> None:
         (root / "sandbox" / name).write_bytes(data)
 
 
-def prepare_home(root: Path) -> None:
+def prepare_home(root: Path, *, sessions: int | None = 1) -> None:
     for name in _HOME_DIRS:
         (root / name).mkdir(parents=True, exist_ok=True)
     write_sandbox_files(root)
+    if sessions != 1:
+        # bubblewrap mounts over real directories only, never a link.
+        for name in _SESSION_VIEWS:
+            path = root / name
+            if path.is_symlink():
+                path.unlink()
+            path.mkdir(parents=True, exist_ok=True)
+        return
+    for name in _SESSION_VIEWS:
+        path = root / name
+        if path.is_dir() and not path.is_symlink():
+            # Left by a sandboxed setup; flip_link() can't replace a
+            # real directory, so clear it back to empty here.
+            try:
+                path.rmdir()
+            except OSError:
+                raise typer.BadParameter(
+                    f"{path} is a directory with content; move it aside first"
+                ) from None
     idle = root / "sessions" / ".idle" / "workspace"
     idle.mkdir(parents=True, exist_ok=True)
     link = root / "workspace"
@@ -82,6 +106,47 @@ def prepare_home(root: Path) -> None:
         link.symlink_to(idle)
 
 
+def check_sandbox(sessions: int | None) -> int | None:
+    """The sessions a direct runtime here serves at once.
+
+    ``None`` asks for no cap, which needs Linux and bubblewrap: a
+    machine without them serves one, and the reason is printed.  An
+    explicit count above one must be servable or the command fails.
+    """
+    if sessions is not None and sessions < 1:
+        raise typer.BadParameter("--sessions must be at least 1")
+    if sessions == 1:
+        return 1
+    problem = _sandbox_problem()
+    if problem is None:
+        return sessions
+    if sessions is None:
+        print(f"serving one session at a time: {problem}")
+        return 1
+    raise typer.BadParameter(problem)
+
+
+def _sandbox_problem() -> str | None:
+    if sys.platform != "linux":
+        return "several sessions at once need Linux; macOS serves one"
+    if shutil.which(BWRAP) is None:
+        return (
+            "several sessions at once need bubblewrap; "
+            "install it (apt install bubblewrap) and retry"
+        )
+    probe = subprocess.run(
+        [BWRAP, "--dev-bind", "/", "/", "--die-with-parent", "--", "true"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if probe.returncode != 0:
+        return "bubblewrap cannot start a sandbox here: " + (
+            probe.stderr.strip() or f"exit {probe.returncode}"
+        )
+    return None
+
+
 def system_link_commands(root: Path) -> list[list[str]]:
     """``ln`` commands still needed for ``/recipes`` & co.; empty when done."""
     cmds: list[list[str]] = []
@@ -171,12 +236,24 @@ def init(
             help="Copy this host's Claude login into docker sessions",
         ),
     ] = True,
+    sessions: Annotated[
+        int | None,
+        typer.Option(
+            help="Cap on sessions served at once in direct mode; by default "
+            "as many as the machine can isolate (above 1 needs Linux and "
+            "bubblewrap)"
+        ),
+    ] = None,
 ) -> None:
     """Prepare the runtime home and, in direct mode, the system links."""
     if mode == "docker" and not image:
         raise typer.BadParameter("--image is required in docker mode")
+    if mode == "direct":
+        sessions = check_sandbox(sessions)
+    else:
+        sessions = 1
     home = runtime_home()
-    prepare_home(home)
+    prepare_home(home, sessions=sessions)
     existing = (
         RuntimeConfig.load(home) if (home / "config.json").is_file() else None
     )
@@ -185,12 +262,23 @@ def init(
         mode=mode,
         image=image,
         share_claude_login=share_claude_login,
+        sessions=sessions,
         credentials=existing.credentials if existing else None,
     )
     config.save()
     if mode == "direct":
         ensure_system_links(home)
-    print(f"runtime ready at {home} ({mode})")
+    served = (
+        "no cap on sessions at once"
+        if sessions is None
+        else f"{sessions} sessions at once"
+        if sessions > 1
+        else ""
+    )
+    print(
+        f"runtime ready at {home} ({mode}"
+        + (f", {served})" if served else ")")
+    )
 
 
 @app.command()
@@ -283,6 +371,82 @@ def register(
     print("next: levain-runtime up")
 
 
+def _detach_and_send(
+    root: Path,
+    creds: Credentials,
+    event_type: str,
+    payload: object,
+    agent: str | None,
+) -> None:
+    """Deliver an event from a detached child; the caller returns now.
+
+    A hook must never wait on the network, so the parent forks and
+    exits while the child drops the inherited stdio (a caller reading
+    our pipes would otherwise block on the child) and posts the event.
+    Failures land in ``emit.log`` under the runtime home.
+    """
+    if os.fork() > 0:
+        return
+    os.setsid()
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in (0, 1, 2):
+        os.dup2(devnull, fd)
+    try:
+        platform_api.post_event(creds, event_type, payload, agent)
+    except Exception as exc:  # nothing above us to handle it
+        with (root / "emit.log").open("a") as fh:
+            fh.write(
+                f"{time.strftime('%Y-%m-%dT%H:%M:%S%z')} {event_type}: {exc}\n"
+            )
+    finally:
+        os._exit(0)
+
+
+@app.command()
+def emit(
+    event_type: Annotated[
+        str,
+        typer.Argument(help="Event name, e.g. runtime.claude_code.stop"),
+    ],
+    agent: Annotated[
+        str | None,
+        typer.Option(help="Only dispatch this agent (slug)"),
+    ] = None,
+    payload: Annotated[
+        str | None,
+        typer.Option(help="JSON payload, or - to read it from stdin"),
+    ] = None,
+    wait: Annotated[
+        bool,
+        typer.Option(
+            "--wait", help="Send in the foreground and print the result"
+        ),
+    ] = False,
+) -> None:
+    """Raise an event for agents subscribed to it on this runtime.
+
+    Reads the payload, then returns immediately while a detached child
+    delivers it — safe to call from hooks. Use --wait to send in the
+    foreground and see the platform's response.
+    """
+    config = _load_config()
+    creds = config.credentials
+    if creds is None:
+        raise typer.BadParameter(
+            "no credentials; run `levain-runtime connect`"
+        )
+    raw = sys.stdin.read() if payload == "-" else payload
+    try:
+        parsed = json.loads(raw) if raw and raw.strip() else None
+    except json.JSONDecodeError as exc:
+        raise typer.BadParameter(f"payload is not valid JSON: {exc}") from None
+    if wait or not hasattr(os, "fork"):
+        result = platform_api.post_event(creds, event_type, parsed, agent)
+        print(json.dumps(result))
+        return
+    _detach_and_send(config.root, creds, event_type, parsed, agent)
+
+
 @app.command()
 def up() -> None:
     """Start the node server, the tunnel and the heartbeat."""
@@ -294,32 +458,57 @@ def up() -> None:
     _serve(config)
 
 
+def _live_health() -> dict[str, Any] | None:
+    try:
+        resp = httpx.get(
+            f"http://127.0.0.1:{RUNNER_HTTP_PORT}/v1/health", timeout=2.0
+        )
+        resp.raise_for_status()
+        body = resp.json()
+    except (httpx.HTTPError, ValueError):
+        return None
+    return body if isinstance(body, dict) else None
+
+
 @app.command()
 def status() -> None:
     """Show the configuration and the sessions on this runtime."""
     config = _load_config()
-    manager = SessionManager(config)
+    # Read-only: the startup reset belongs to the running service.
+    manager = SessionManager(config, restore=False)
+    live = _live_health()
     print(f"home: {config.root}")
     print(
         f"mode: {config.mode}" + (f" ({config.image})" if config.image else "")
     )
+    print("service: " + ("running" if live else "not running"))
     if config.credentials:
         print(f"runtime: {config.credentials.runtime_id}")
-        print(f"gateway: {config.credentials.gateway_url}")
+        if live and live.get("gateway"):
+            print(f"gateway: {live['gateway']}")
+        else:
+            print(
+                f"gateway: {chisel.gateway(config.credentials)}"
+                " (from this shell)"
+            )
     else:
         print("runtime: not connected")
     if config.mode == "direct":
+        print(f"sessions at once: {config.sessions or 'no cap'}")
         pending = system_link_commands(config.root)
         print("system links: " + ("ok" if not pending else "missing"))
     print(
         f"chisel: {os.environ.get('CHISEL_BIN') or config.root / 'bin' / 'chisel'}"
     )
     sessions = manager.store.all()
-    print(
-        f"sessions: {len(sessions)} ({len(manager.active_sessions())} active)"
-    )
+    if live:
+        active = set(live.get("active_sessions") or [])
+    else:
+        active = {m.sid for m in sessions if m.state == "active"}
+    print(f"sessions: {len(sessions)} ({len(active)} active)")
     for meta in sessions:
-        print(f"  {meta.sid} {meta.state} {meta.workspace_dir}")
+        state = "active" if meta.sid in active else "paused"
+        print(f"  {meta.sid} {state} {meta.workspace_dir}")
 
 
 def _serve(config: RuntimeConfig) -> None:

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/config.py

@@ -82,6 +82,10 @@ class RuntimeConfig:
     mode: Mode = "direct"
     image: str | None = None
     share_claude_login: bool = True
+    # Sessions a direct runtime serves at once, ``None`` for no cap.
+    # Unless it is one, each node runs under bubblewrap with its
+    # session's folders at /workspace, /recipes/active and /sdk/active.
+    sessions: int | None = 1
     credentials: Credentials | None = None
 
     @property
@@ -97,6 +101,11 @@ class RuntimeConfig:
             mode=raw.get("mode", "direct"),
             image=raw.get("image"),
             share_claude_login=bool(raw.get("share_claude_login", True)),
+            # Absent (an older runner wrote the file) means one; an
+            # explicit null means no cap.
+            sessions=None
+            if (count := raw.get("sessions", 1)) is None
+            else max(1, int(count or 1)),
             credentials=Credentials.from_dict(creds) if creds else None,
         )
 
@@ -106,6 +115,7 @@ class RuntimeConfig:
             "mode": self.mode,
             "image": self.image,
             "share_claude_login": self.share_claude_login,
+            "sessions": self.sessions,
             "credentials": self.credentials.to_dict()
             if self.credentials
             else None,

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/heartbeat.py

@@ -31,6 +31,7 @@ async def run_heartbeat(
             payload = {
                 "version": package_version(),
                 "mode": manager.mode,
+                "capacity": manager.capacity,
                 "active_sessions": len(manager.active_sessions()),
             }
             try:

levain_runtime-0.1.10/levain_runtime/jobs.py

@@ -0,0 +1,177 @@
+"""Execs that outlive the HTTP stream that started them (#2760).
+
+A job drains ``stream_process`` into a numbered record list, so a
+reader whose tunnel dropped can attach again at the record it last
+saw. The process is killed on cancel, on its own timeout, or once no
+reader has been attached for ``ORPHAN_SECONDS``: a node never outlives
+the platform's interest in it, and a finished job nobody collects is
+forgotten the same way.
+
+A reattach whose last-seen record has already been trimmed from the
+buffer is a gap the server can no longer fill; ``Job.has_gap`` lets
+the caller refuse it rather than silently resuming past missing
+records.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+import uuid
+from collections.abc import AsyncIterator, Callable
+from typing import Any
+
+from .protocol import JOB_ORPHAN_SECONDS
+
+logger = logging.getLogger(__name__)
+
+ORPHAN_SECONDS: float = JOB_ORPHAN_SECONDS
+
+
+class Job:
+    def __init__(
+        self,
+        job_id: str,
+        sid: str,
+        stream: AsyncIterator[dict[str, Any]],
+        *,
+        orphan_seconds: float,
+    ) -> None:
+        self.id = job_id
+        self.sid = sid
+        self.records: list[dict[str, Any]] = []
+        self.done = False
+        self.readers = 0
+        self._trimmed = 0  # highest seq ever dropped from the front
+        self._generation = 0  # bumped on each attach; only the latest trims
+        self._idle_since = time.monotonic()
+        self._orphan = orphan_seconds
+        self._changed = asyncio.Condition()
+        self._drain = asyncio.create_task(self._run(stream))
+
+    def _next_seq(self) -> int:
+        return self._trimmed + len(self.records) + 1
+
+    async def _run(self, stream: AsyncIterator[dict[str, Any]]) -> None:
+        try:
+            async for item in stream:
+                async with self._changed:
+                    self.records.append({"seq": self._next_seq(), **item})
+                    self._changed.notify_all()
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            logger.exception("job %s stream failed", self.id)
+            async with self._changed:
+                self.records.append(
+                    {
+                        "seq": self._next_seq(),
+                        "exit": -1,
+                        "error": str(exc),
+                    }
+                )
+                self._changed.notify_all()
+        finally:
+            async with self._changed:
+                self.done = True
+                self._changed.notify_all()
+
+    def _trim(self, through: int) -> None:
+        """Drop buffered records with ``seq <= through``."""
+        drop = min(max(through, 0) - self._trimmed, len(self.records))
+        if drop > 0:
+            del self.records[:drop]
+            self._trimmed += drop
+
+    def has_gap(self, after: int) -> bool:
+        """True if a resuming reader's ``after`` can't be served.
+
+        Records are trimmed as soon as they're yielded to a reader,
+        not once that reader's tunnel has actually delivered them, so
+        a reattach can ask for a ``after`` that's already gone from
+        the buffer: the records between would be silently skipped.
+        A fresh attach always starts while ``_trimmed`` is still 0,
+        so ``after < self._trimmed`` already excludes it without
+        special-casing ``after == 0``.
+        """
+        return after < self._trimmed
+
+    async def attach(self, after: int) -> AsyncIterator[dict[str, Any]]:
+        """Records with ``seq > after``, live until the exit record.
+
+        Acknowledged records are dropped from the buffer as they're
+        yielded, so a chatty, long-running job doesn't grow memory
+        without bound even over a single unbroken stream; ``seq`` stays
+        monotonic across the trim. Only the most recently attached
+        reader trims: a reattach (the reader's tunnel dropped) bumps
+        the generation, so a prior reader whose disconnect hasn't been
+        noticed yet stops deleting records the new one still needs.
+        """
+        self.readers += 1
+        self._generation += 1
+        generation = self._generation
+        try:
+            async with self._changed:
+                self._trim(after)
+            i = 0
+            while True:
+                while i < len(self.records):
+                    yield self.records[i]
+                    i += 1
+                async with self._changed:
+                    if i > 0 and self._generation == generation:
+                        self._trim(self._trimmed + i)
+                        i = 0
+                    if self.done and i >= len(self.records):
+                        return
+                    await self._changed.wait_for(self._grew_past(i))
+        finally:
+            self.readers -= 1
+            self._idle_since = time.monotonic()
+
+    def _grew_past(self, n: int) -> Callable[[], bool]:
+        return lambda: self.done or len(self.records) > n
+
+    def orphaned(self) -> bool:
+        return (
+            self.readers == 0
+            and time.monotonic() - self._idle_since >= self._orphan
+        )
+
+    async def cancel(self) -> None:
+        # Cancelling the drain closes the generator, whose ``finally``
+        # kills the process group.
+        self._drain.cancel()
+        await asyncio.wait({self._drain})
+
+
+class JobRegistry:
+    def __init__(self) -> None:
+        self._jobs: dict[str, Job] = {}
+        self._reaper: asyncio.Task[None] | None = None
+
+    def start(self, sid: str, stream: AsyncIterator[dict[str, Any]]) -> Job:
+        job = Job(uuid.uuid4().hex, sid, stream, orphan_seconds=ORPHAN_SECONDS)
+        self._jobs[job.id] = job
+        if self._reaper is None or self._reaper.done():
+            self._reaper = asyncio.create_task(self._reap())
+        return job
+
+    def get(self, sid: str, job_id: str) -> Job | None:
+        job = self._jobs.get(job_id)
+        return job if job is not None and job.sid == sid else None
+
+    async def cancel(self, job: Job) -> None:
+        self._jobs.pop(job.id, None)
+        await job.cancel()
+
+    async def cancel_for(self, sid: str) -> None:
+        for job in [j for j in self._jobs.values() if j.sid == sid]:
+            await self.cancel(job)
+
+    async def _reap(self) -> None:
+        while self._jobs:
+            await asyncio.sleep(min(ORPHAN_SECONDS / 4, 5.0))
+            for job in [j for j in self._jobs.values() if j.orphaned()]:
+                await self.cancel(job)

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/platform_api.py

@@ -50,6 +50,35 @@ def refresh_connection(creds: Credentials) -> bool:
     return True
 
 
+def post_event(
+    creds: Credentials,
+    event_type: str,
+    payload: Any,
+    agent: str | None = None,
+) -> dict[str, Any]:
+    """``POST /api/v1/runtimes/{id}/events``; returns the response body.
+
+    Raises ``httpx.HTTPStatusError`` with the API's message on refusal.
+    """
+    body: dict[str, Any] = {"event_type": event_type, "payload": payload}
+    if agent is not None:
+        body["agent"] = agent
+    with _client() as client:
+        resp = client.post(
+            f"{creds.api_url}/api/v1/runtimes/{creds.runtime_id}/events",
+            json=body,
+            headers={"Authorization": f"Bearer {creds.secret}"},
+        )
+    if resp.status_code >= 400:
+        raise httpx.HTTPStatusError(
+            _error_message(resp), request=resp.request, response=resp
+        )
+    data = resp.json()
+    if not isinstance(data, dict):
+        raise ValueError("unexpected response from the platform")
+    return data
+
+
 def register_runtime(
     api_url: str, api_key: str, body: dict[str, Any]
 ) -> dict[str, Any]:

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/protocol.py

@@ -6,4 +6,8 @@ RUNNER_HTTP_PORT = 17777
 FORWARD_SANDBOX_API_PORT = 18010
 FORWARD_PROXY_PORT = 14100
 
-CHISEL_VERSION = "1.11.3"
+CHISEL_VERSION = "1.12.0"
+
+# A job whose reader has been gone this long is killed; the platform
+# gives up reattaching after the same window (``remote.py``).
+JOB_ORPHAN_SECONDS = 300

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/server.py

@@ -13,11 +13,13 @@ from starlette.requests import Request
 from starlette.responses import JSONResponse, Response, StreamingResponse
 from starlette.routing import Mount, Route
 
+from . import chisel
 from .config import RuntimeConfig
 from .sessions import (
     MissingTarget,
     RuntimeBusy,
     SessionManager,
+    UnknownJob,
     UnknownSession,
 )
 
@@ -58,8 +60,19 @@ def _env_of(body: dict[str, Any]) -> dict[str, str]:
 
 async def health(request: Request) -> Response:
     m = _manager(request)
+    creds = m.config.credentials
     return JSONResponse(
-        {"ok": True, "mode": m.mode, "active_sessions": m.active_sessions()}
+        {
+            "ok": True,
+            "mode": m.mode,
+            "capacity": m.capacity,
+            "active_sessions": m.active_sessions(),
+            # Execs can run as jobs the platform reattaches to (#2760).
+            "jobs": True,
+            # Resolved in the service's own environment, which may
+            # override the configured gateway.
+            "gateway": chisel.gateway(creds) if creds else None,
+        }
     )
 
 
@@ -94,9 +107,7 @@ async def resume(request: Request) -> Response:
     return Response(status_code=204)
 
 
-async def exec_(request: Request) -> Response:
-    m = _manager(request)
-    sid = _sid(request)
+async def _exec_body(request: Request) -> dict[str, Any]:
     body = await _json_body(request)
     cmd = body.get("cmd")
     if not isinstance(cmd, str) or not cmd:
@@ -104,9 +115,15 @@ async def exec_(request: Request) -> Response:
     stdin = body.get("stdin")
     if stdin is not None and not isinstance(stdin, str):
         raise ValueError("stdin must be a string")
-    timeout = float(body.get("timeout") or 0)
-    stream = m.exec(sid, cmd, _env_of(body), stdin=stdin, timeout=timeout)
+    return {
+        "cmd": cmd,
+        "env": _env_of(body),
+        "stdin": stdin,
+        "timeout": float(body.get("timeout") or 0),
+    }
+
 
+def _ndjson(stream: AsyncIterator[dict[str, Any]]) -> StreamingResponse:
     async def lines() -> AsyncIterator[bytes]:
         async for item in stream:
             yield (json.dumps(item) + "\n").encode()
@@ -114,6 +131,36 @@ async def exec_(request: Request) -> Response:
     return StreamingResponse(lines(), media_type="application/x-ndjson")
 
 
+async def exec_(request: Request) -> Response:
+    m = _manager(request)
+    sid = _sid(request)
+    args = await _exec_body(request)
+    return _ndjson(m.exec(sid, **args))
+
+
+async def jobs(request: Request) -> Response:
+    m = _manager(request)
+    sid = _sid(request)
+    args = await _exec_body(request)
+    job = m.start_job(sid, **args)
+    return JSONResponse({"job_id": job.id}, status_code=201)
+
+
+async def job(request: Request) -> Response:
+    m = _manager(request)
+    job = m.job(_sid(request), request.path_params["job_id"])
+    if request.method == "DELETE":
+        await m.jobs.cancel(job)
+        return Response(status_code=204)
+    try:
+        after = int(request.query_params.get("after", "0"))
+    except ValueError as exc:
+        raise ValueError("after must be an integer") from exc
+    if job.has_gap(after):
+        return Response(status_code=410)
+    return _ndjson(job.attach(after))
+
+
 async def capture(request: Request) -> Response:
     m = _manager(request)
     sid = _sid(request)
@@ -184,6 +231,13 @@ def _error(status: int) -> Any:
     return handler
 
 
+async def _busy(request: Request, exc: Exception) -> Response:
+    assert isinstance(exc, RuntimeBusy)
+    return JSONResponse(
+        {"detail": str(exc), "session_id": exc.session_id}, status_code=409
+    )
+
+
 def create_app(config: RuntimeConfig) -> Starlette:
     app = Starlette(
         routes=[
@@ -199,6 +253,12 @@ def create_app(config: RuntimeConfig) -> Starlette:
                     Route("/sessions/{sid}/pause", pause, methods=["POST"]),
                     Route("/sessions/{sid}/resume", resume, methods=["POST"]),
                     Route("/sessions/{sid}/exec", exec_, methods=["POST"]),
+                    Route("/sessions/{sid}/jobs", jobs, methods=["POST"]),
+                    Route(
+                        "/sessions/{sid}/jobs/{job_id}",
+                        job,
+                        methods=["GET", "DELETE"],
+                    ),
                     Route(
                         "/sessions/{sid}/capture", capture, methods=["POST"]
                     ),
@@ -218,8 +278,9 @@ def create_app(config: RuntimeConfig) -> Starlette:
         exception_handlers={
             ValueError: _error(400),
             UnknownSession: _error(404),
+            UnknownJob: _error(404),
             MissingTarget: _error(404),
-            RuntimeBusy: _error(409),
+            RuntimeBusy: _busy,
         },
     )
     app.state.manager = SessionManager(config)

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/levain_runtime/sessions.py

@@ -10,11 +10,12 @@ import shutil
 import tarfile
 import uuid
 from collections.abc import AsyncIterator, Awaitable, Callable
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 from typing import IO, Any, Literal
 
 from .config import RuntimeConfig
+from .jobs import Job, JobRegistry
 from .process import host_env, run_process, stream_process
 
 log = logging.getLogger("levain_runtime")
@@ -25,12 +26,20 @@ _SESSION_FILE = "session.json"
 # Trees under these roots are host bind mounts in docker mode, so they
 # are extracted on the host for both backends.
 _HOST_TREE_ROOTS = ("/recipes", "/sdk")
+WORKSPACE_PATH = "/workspace"
+BWRAP = "bwrap"
 _CLAUDE_READY_SENTINEL = "/tmp/.levain-runtime-ready"
 
 Runner = Callable[..., Awaitable[tuple[int, bytes, bytes]]]
 
 
 class RuntimeBusy(Exception):
+    def __init__(self, message: str, *, session_id: str) -> None:
+        super().__init__(message)
+        self.session_id = session_id
+
+
+class UnknownJob(Exception):
     pass
 
 
@@ -52,6 +61,9 @@ class SessionMeta:
     env: dict[str, str]
     workspace_dir: str
     state: State = "paused"
+    # Per-session views (``/recipes/active`` -> ``/recipes/<fp>``) on a
+    # sandboxed direct runtime; the host symlink is flipped elsewhere.
+    trees: dict[str, str] = field(default_factory=dict)
 
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)
@@ -194,11 +206,25 @@ class SessionStore:
 
 
 class DirectBackend:
-    """Nodes run as host processes; one active session owns ``/workspace``."""
+    """Nodes run as host processes.
+
+    With a capacity of one, the active session owns the root-level
+    ``/workspace`` link. Above one, or with no cap (``None``), each
+    node runs under bubblewrap,
+    which binds the session's own folders over ``/workspace``,
+    ``/recipes/active`` and ``/sdk/active`` in a private mount
+    namespace; the rest of the host looks unchanged.
+    """
 
-    def __init__(self, root: Path) -> None:
+    def __init__(self, root: Path, *, capacity: int | None = 1) -> None:
         self.root = root
-        self.active: str | None = None
+        self.capacity = None if capacity is None else max(1, capacity)
+        # Holders, oldest first.
+        self.active: list[str] = []
+
+    @property
+    def sandboxed(self) -> bool:
+        return self.capacity != 1
 
     @property
     def idle_workspace(self) -> Path:
@@ -212,24 +238,81 @@ class DirectBackend:
         os.replace(tmp, link)
 
     async def activate(self, meta: SessionMeta) -> None:
-        if self.active not in (None, meta.sid):
-            raise RuntimeBusy("runtime busy")
-        self._point_workspace(Path(meta.workspace_dir))
-        self.active = meta.sid
+        if meta.sid not in self.active:
+            if self.capacity is not None and len(self.active) >= self.capacity:
+                raise RuntimeBusy(
+                    f"runtime full ({self.capacity} sessions)"
+                    if self.sandboxed
+                    else "runtime busy",
+                    session_id=self.active[0],
+                )
+            self.active.append(meta.sid)
+        if not self.sandboxed:
+            self._point_workspace(Path(meta.workspace_dir))
 
     async def release(self, meta: SessionMeta) -> None:
-        if self.active == meta.sid:
-            self.active = None
+        if meta.sid not in self.active:
+            return
+        self.active.remove(meta.sid)
+        if not self.sandboxed:
             self._point_workspace(self.idle_workspace)
 
     async def destroy(self, meta: SessionMeta) -> None:
         await self.release(meta)
 
     def status(self, meta: SessionMeta) -> State:
-        return "active" if self.active == meta.sid else "paused"
+        return "active" if meta.sid in self.active else "paused"
+
+    def views(self, meta: SessionMeta) -> dict[str, str]:
+        """Node-visible path -> the session's folder behind it."""
+        return {WORKSPACE_PATH: meta.workspace_dir, **meta.trees}
+
+    def session_path(self, meta: SessionMeta, path: str) -> Path:
+        """Where ``path`` lives on the host for this session.
+
+        Only differs from ``path`` in sandboxed mode, where the views
+        exist inside each node's mount namespace.
+        """
+        if self.sandboxed:
+            for view, real in self.views(meta).items():
+                if path == view or path.startswith(view + "/"):
+                    return Path(real + path[len(view) :])
+        return Path(path)
+
+    def _tmp_dir(self, meta: SessionMeta) -> Path:
+        return self.root / "sessions" / meta.sid / "tmp"
 
     def _env(self, meta: SessionMeta, extra: dict[str, str]) -> dict[str, str]:
-        return {**host_env(), **meta.env, **extra}
+        env = {**host_env(), **meta.env, **extra}
+        if self.sandboxed:
+            # Daemons outside the sandbox (docker, a devcontainer CLI)
+            # resolve paths on the host, so recipes hand them this.
+            env["LEVAIN_HOST_WORKSPACE"] = meta.workspace_dir
+            # /tmp stays shared (host tools keep sockets there); the
+            # session's own scratch keeps file names from clashing.
+            tmp = self._tmp_dir(meta)
+            tmp.mkdir(parents=True, exist_ok=True)
+            env["TMPDIR"] = str(tmp)
+        return env
+
+    def argv(self, meta: SessionMeta, cmd: str) -> list[str]:
+        shell = ["/bin/sh", "-c", cmd]
+        if not self.sandboxed:
+            return shell
+        argv = [BWRAP, "--dev-bind", "/", "/"]
+        for view, real in self.views(meta).items():
+            # bubblewrap mounts on the resolved host path, never on a
+            # symlink; ``init`` made these real directories.
+            argv += ["--bind", real, os.path.realpath(view)]
+        argv += ["--chdir", WORKSPACE_PATH, "--die-with-parent", "--", *shell]
+        return argv
+
+    def _cwd(self, meta: SessionMeta) -> str | None:
+        if self.sandboxed:
+            return None
+        return (
+            meta.workspace_dir if Path(meta.workspace_dir).is_dir() else None
+        )
 
     def exec(
         self,
@@ -241,13 +324,11 @@ class DirectBackend:
         timeout: float,
     ) -> AsyncIterator[dict[str, Any]]:
         return stream_process(
-            ["/bin/sh", "-c", cmd],
+            self.argv(meta, cmd),
             env=self._env(meta, env),
             stdin=stdin.encode() if stdin is not None else None,
             timeout=timeout,
-            cwd=meta.workspace_dir
-            if Path(meta.workspace_dir).is_dir()
-            else None,
+            cwd=self._cwd(meta),
         )
 
     async def capture(
@@ -259,33 +340,37 @@ class DirectBackend:
         timeout: float,
     ) -> tuple[str, int]:
         rc, out, _ = await run_process(
-            ["/bin/sh", "-c", cmd],
+            self.argv(meta, cmd),
             env=self._env(meta, env),
             timeout=timeout,
-            cwd=meta.workspace_dir
-            if Path(meta.workspace_dir).is_dir()
-            else None,
+            cwd=self._cwd(meta),
         )
         return out.decode("utf-8", "replace"), rc
 
     async def put_file(
         self, meta: SessionMeta, path: str, data: bytes
     ) -> None:
-        p = Path(path)
+        p = self.session_path(meta, path)
         p.parent.mkdir(parents=True, exist_ok=True)
         p.write_bytes(data)
 
     async def get_file(self, meta: SessionMeta, path: str) -> bytes | None:
-        p = Path(path)
+        p = self.session_path(meta, path)
         return p.read_bytes() if p.is_file() else None
 
     async def put_tree(
         self, meta: SessionMeta, path: str, src: IO[bytes]
     ) -> None:
-        extract_tar(src, Path(path))
+        extract_tar(src, self.session_path(meta, path))
 
     async def link(self, meta: SessionMeta, path: str, target: str) -> None:
-        flip_link(Path(path), target, exists_at=Path(target))
+        if not self.sandboxed:
+            flip_link(Path(path), target, exists_at=Path(target))
+            return
+        # The view is per session: recorded here, bound at each exec.
+        if not Path(target).is_dir():
+            raise MissingTarget(f"{target} is not on this runtime")
+        meta.trees[path] = target
 
 
 class DockerBackend:
@@ -552,9 +637,10 @@ class DockerBackend:
 class SessionManager:
     """Routes session operations to the configured backend."""
 
-    def __init__(self, config: RuntimeConfig) -> None:
+    def __init__(self, config: RuntimeConfig, *, restore: bool = True) -> None:
         self.config = config
         self.store = SessionStore(config.root)
+        self.jobs = JobRegistry()
         self.backend: DirectBackend | DockerBackend
         if config.mode == "docker":
             if not config.image:
@@ -565,14 +651,15 @@ class SessionManager:
                 share_claude_login=config.share_claude_login,
             )
         else:
-            self.backend = DirectBackend(config.root)
-            self._restore_direct_slot()
+            self.backend = DirectBackend(config.root, capacity=config.sessions)
+            if restore:
+                self._restore_direct_slots()
 
-    def _restore_direct_slot(self) -> None:
+    def _restore_direct_slots(self) -> None:
         assert isinstance(self.backend, DirectBackend)
         active = [m for m in self.store.all() if m.state == "active"]
-        # Only one session may own /workspace; after a restart, any
-        # recorded holder is stale until the platform resumes it.
+        # Slots live in memory; after a restart, any recorded holder is
+        # stale until the platform resumes it.
         for meta in active:
             meta.state = "paused"
             self.store.save(meta)
@@ -587,7 +674,16 @@ class SessionManager:
             raise UnknownSession(sid)
         return meta
 
+    @property
+    def capacity(self) -> int | None:
+        """Sessions served at once; ``None`` when nothing bounds it."""
+        if isinstance(self.backend, DirectBackend):
+            return self.backend.capacity
+        return None
+
     def active_sessions(self) -> list[str]:
+        if isinstance(self.backend, DirectBackend):
+            return list(self.backend.active)
         return [m.sid for m in self.store.all() if m.state == "active"]
 
     async def provision(
@@ -648,6 +744,7 @@ class SessionManager:
         meta = self.store.get(sid)
         if meta is None:
             return
+        await self.jobs.cancel_for(sid)
         await self.backend.destroy(meta)
         self.store.delete(sid, keep_workspace=keep_workspace)
 
@@ -663,6 +760,26 @@ class SessionManager:
         meta = self.require(sid)
         return self.backend.exec(meta, cmd, env, stdin=stdin, timeout=timeout)
 
+    def start_job(
+        self,
+        sid: str,
+        cmd: str,
+        env: dict[str, str],
+        *,
+        stdin: str | None,
+        timeout: float,
+    ) -> Job:
+        return self.jobs.start(
+            sid, self.exec(sid, cmd, env, stdin=stdin, timeout=timeout)
+        )
+
+    def job(self, sid: str, job_id: str) -> Job:
+        self.require(sid)
+        job = self.jobs.get(sid, job_id)
+        if job is None:
+            raise UnknownJob(job_id)
+        return job
+
     async def capture(
         self, sid: str, cmd: str, env: dict[str, str], *, timeout: float
     ) -> tuple[str, int]:
@@ -685,11 +802,11 @@ class SessionManager:
         )
 
     async def link(self, sid: str, path: str, target: str) -> None:
+        meta = self.require(sid)
         await self.backend.link(
-            self.require(sid),
-            validate_abs_path(path),
-            validate_abs_path(target),
+            meta, validate_abs_path(path), validate_abs_path(target)
         )
+        self.store.save(meta)
 
     async def reset_agent_workspace(
         self, agent_id: str, *, user: str | None
@@ -702,20 +819,28 @@ class SessionManager:
             scope="user" if user else "agent",
             user=user,
         )
+        live = set(self.active_sessions())
         for meta in self.store.all():
-            if meta.workspace_dir == str(target) and meta.state == "active":
-                raise RuntimeBusy(f"workspace in use by session {meta.sid}")
+            if meta.workspace_dir == str(target) and (
+                meta.state == "active" or meta.sid in live
+            ):
+                raise RuntimeBusy(
+                    f"workspace in use by session {meta.sid}",
+                    session_id=meta.sid,
+                )
         shutil.rmtree(target, ignore_errors=True)
         target.mkdir(parents=True, exist_ok=True)
 
 
 __all__ = [
+    "BWRAP",
     "DirectBackend",
     "DockerBackend",
     "MissingTarget",
     "RuntimeBusy",
     "SessionManager",
     "SessionMeta",
+    "UnknownJob",
     "UnknownSession",
     "validate_abs_path",
 ]

{levain_runtime-0.1.2 → levain_runtime-0.1.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "levain-runtime"
-version = "0.1.2"
+version = "0.1.10"
 description = "Self-hosted runtime for Levain agents: run an agent's nodes on your own machine or container while the platform drives the recipe"
 readme = "README.md"
 license = "Apache-2.0"