sbxloop 0.5.3__tar.gz → 0.5.5__tar.gz

{sbxloop-0.5.3 → sbxloop-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbxloop
-Version: 0.5.3
+Version: 0.5.5
 Summary: Agentic loop orchestration on Docker Sandboxes (sbx) with isolated credential domains
 Project-URL: Homepage, https://github.com/brettbergin/sbxloop
 Project-URL: Repository, https://github.com/brettbergin/sbxloop

{sbxloop-0.5.3 → sbxloop-0.5.5}/src/sbxloop/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.5.3'
-__version_tuple__ = version_tuple = (0, 5, 3)
+__version__ = version = '0.5.5'
+__version_tuple__ = version_tuple = (0, 5, 5)
 
 __commit_id__ = commit_id = None

{sbxloop-0.5.3 → sbxloop-0.5.5}/src/sbxloop/cli/app.py

@@ -25,6 +25,7 @@ from sbxloop.errors import SdxloopError
 from sbxloop.events import Event
 from sbxloop.sbx.cli import SbxCLI
 from sbxloop.sbx.provision import sandbox_name
+from sbxloop.sbx.prune import classify_sandboxes, format_age, remove_sandbox
 
 app = typer.Typer(
     name="sbxloop",
@@ -414,6 +415,84 @@ def sandbox_rm(
             console.print(f"[yellow]skip {target}:[/] {exc}")
 
 
+@sandbox_app.command("prune")
+def sandbox_prune(
+    force: Annotated[
+        bool,
+        typer.Option("--force", "--yes", help="Actually remove (default is a dry run)."),
+    ] = False,
+    min_age: Annotated[
+        float,
+        typer.Option(
+            "--min-age",
+            help="Hours a run must be inactive before its sandboxes count as orphaned.",
+        ),
+    ] = 1.0,
+    include_kept: Annotated[
+        bool,
+        typer.Option("--include-kept", help="Also prune kept-for-debugging sandboxes."),
+    ] = False,
+) -> None:
+    """Garbage-collect orphaned sbxloop sandboxes (crashed hosts, killed runs).
+
+    Cross-references `sbx ls` against this working copy's state DB. Dry-run
+    by default: prints the classification and removes nothing without
+    --force.
+    """
+    config = load_config()
+    store = _store(config)
+    cli = SbxCLI(app_name=config.app_name or None)
+    try:
+        verdicts = classify_sandboxes(
+            cli.ls(), store, min_age_s=min_age * 3600.0, include_kept=include_kept
+        )
+    except SdxloopError as exc:
+        console.print(f"[bold red]{exc}[/]")
+        raise typer.Exit(2) from exc
+    if not verdicts:
+        console.print("no sbxloop sandboxes found")
+        return
+
+    table = Table(title="sbxloop sandbox prune")
+    for column in ("sandbox", "run", "run state", "age", "verdict"):
+        table.add_column(column)
+    for v in verdicts:
+        table.add_row(
+            v.name,
+            v.run_id or "",
+            v.run_state or "[dim]unknown[/]",
+            format_age(v.age_s),
+            ("[red]orphan[/] — " if v.orphan else "[green]keep[/] — ") + v.reason,
+        )
+    console.print(table)
+    console.print(
+        "[dim]note: the state DB is per working copy — 'unknown' sandboxes may "
+        "belong to another checkout's runs on this sbx host[/]"
+    )
+
+    orphans = [v for v in verdicts if v.orphan]
+    if not orphans:
+        console.print("nothing to prune")
+        return
+    if not force:
+        console.print(f"dry run: {len(orphans)} orphan candidate(s); re-run with --force to remove")
+        return
+    failures = 0
+    for v in orphans:
+        try:
+            remove_sandbox(cli, v.name)
+        except SdxloopError as exc:
+            failures += 1
+            console.print(f"[yellow]skip {v.name}:[/] {exc}")
+            continue
+        console.print(f"removed {v.name}")
+        # A pruned kept run is no longer kept; keep the DB marker honest.
+        if v.kept_reason is not None and v.run_id is not None:
+            store.set_run_kept(v.run_id, None)
+    if failures:
+        raise typer.Exit(1)
+
+
 @config_app.command("show")
 def config_show() -> None:
     """Show the resolved configuration and where each value came from."""

{sbxloop-0.5.3 → sbxloop-0.5.5}/src/sbxloop/cli/doctor.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import sqlite3
 from collections.abc import Callable
 from dataclasses import dataclass
 
@@ -9,6 +10,7 @@ from rich.console import Console
 from rich.table import Table
 
 from sbxloop.config import load_config
+from sbxloop.engine.store import StateStore
 from sbxloop.errors import SbxError, SbxNotFoundError
 from sbxloop.sbx.cli import SbxCLI
 from sbxloop.sbx.provision import (
@@ -16,6 +18,7 @@ from sbxloop.sbx.provision import (
     COPILOT_TOKEN_ENV,
     GH_TOKEN_ENVS,
 )
+from sbxloop.sbx.prune import count_orphans
 from sbxloop.worker.wheel import resolve_worker_wheel
 
 TESTED_SBX_SERIES = "0.35"
@@ -67,8 +70,10 @@ def collect_checks(
     if version is not None:
         # login / daemon reachable
         report("checking sbx login")
+        logged_in = False
         try:
             cli.ls()
+            logged_in = True
             checks.append(Check("sbx login", True, "sbx ls succeeded"))
         except SbxError as exc:
             login_cmd = (
@@ -76,6 +81,25 @@ def collect_checks(
             )
             checks.append(Check("sbx login", False, f"sbx ls failed ({exc}); run `{login_cmd}`"))
 
+        # orphaned sandboxes (leaked pairs from crashed/killed runs)
+        if logged_in:
+            report("checking for orphaned sandboxes")
+            try:
+                orphans = count_orphans(cli, StateStore(config.state_dir / "state.db"))
+            except (SbxError, OSError, sqlite3.Error):
+                orphans = None
+            if orphans is not None:
+                checks.append(
+                    Check(
+                        "orphaned sandboxes",
+                        orphans == 0,
+                        "none found"
+                        if orphans == 0
+                        else f"{orphans} orphan candidate(s) — run `sbxloop sandbox prune`",
+                        hard=False,
+                    )
+                )
+
         # network policy reachable for the copilot hosts
         for host in AGENT_TOKEN_HOSTS:
             report(f"checking network policy for {host}")

{sbxloop-0.5.3 → sbxloop-0.5.5}/src/sbxloop/engine/model.py

@@ -142,6 +142,10 @@ class RunRecord(_Model):
     # agent VM (False → artifacts are harvested to runs/<run>/artifacts).
     workspace: Path | None = None
     mounted: bool = False
+    # Why this run's sandboxes were deliberately left alive ("debug",
+    # "manual"); None means normal teardown applied. `sandbox prune`
+    # excludes kept runs unless asked to include them.
+    kept_reason: str | None = None
 
 
 class RunResult(_Model):

{sbxloop-0.5.3 → sbxloop-0.5.5}/src/sbxloop/engine/store.py

@@ -33,7 +33,8 @@ CREATE TABLE IF NOT EXISTS runs (
     created_at REAL NOT NULL,
     updated_at REAL NOT NULL,
     workspace  TEXT,
-    mounted    INTEGER NOT NULL DEFAULT 0
+    mounted    INTEGER NOT NULL DEFAULT 0,
+    kept_reason TEXT
 );
 CREATE TABLE IF NOT EXISTS tasks (
     run_id     TEXT NOT NULL,
@@ -75,6 +76,7 @@ CREATE INDEX IF NOT EXISTS idx_events_run ON events (run_id, seq);
 _RUNS_MIGRATIONS = (
     ("workspace", "ALTER TABLE runs ADD COLUMN workspace TEXT"),
     ("mounted", "ALTER TABLE runs ADD COLUMN mounted INTEGER NOT NULL DEFAULT 0"),
+    ("kept_reason", "ALTER TABLE runs ADD COLUMN kept_reason TEXT"),
 )
 
 
@@ -130,6 +132,18 @@ class StateStore:
             raise StateError(f"unknown run {run_id}")
         self._conn.commit()
 
+    def set_run_kept(self, run_id: str, reason: str | None) -> None:
+        """Mark a run's sandboxes as deliberately kept (``"debug"``,
+        ``"manual"``), or clear the marker with None. ``sandbox prune``
+        excludes kept runs unless explicitly told otherwise."""
+        cursor = self._conn.execute(
+            "UPDATE runs SET kept_reason = ? WHERE run_id = ?",
+            (reason, run_id),
+        )
+        if cursor.rowcount == 0:
+            raise StateError(f"unknown run {run_id}")
+        self._conn.commit()
+
     @staticmethod
     def _run_record(row: sqlite3.Row) -> RunRecord:
         return RunRecord(
@@ -140,6 +154,7 @@ class StateStore:
             updated_at=row["updated_at"],
             workspace=Path(row["workspace"]) if row["workspace"] else None,
             mounted=bool(row["mounted"]),
+            kept_reason=row["kept_reason"],
         )
 
     def get_run(self, run_id: str) -> RunRecord:
@@ -243,6 +258,18 @@ class StateStore:
 
     # -- events ------------------------------------------------------------
 
+    def last_event_ts(self, run_id: str) -> float | None:
+        """Timestamp of the run's most recent persisted event, if any.
+
+        Every bus event (heartbeats included) is persisted, so this is the
+        best liveness signal available for a run whose recorded state says
+        non-terminal but whose process may be long dead.
+        """
+        row = self._conn.execute(
+            "SELECT MAX(ts) AS ts FROM events WHERE run_id = ?", (run_id,)
+        ).fetchone()
+        return row["ts"] if row and row["ts"] is not None else None
+
     def append_event(self, event: Event) -> None:
         self._conn.execute(
             "INSERT INTO events (run_id, ts, type, job_id, data_json) VALUES (?, ?, ?, ?, ?)",

sbxloop-0.5.5/src/sbxloop/sbx/prune.py

@@ -0,0 +1,191 @@
+"""Orphaned-sandbox classification for ``sbxloop sandbox prune`` and doctor.
+
+The in-process cleanup registry only protects against failures inside a live
+sbxloop process; a host crash, OOM-kill, or ``kill -9`` leaves the run's
+sandbox pair running indefinitely. This module cross-references ``sbx ls``
+against the state DB to find such leaks safely.
+
+Honesty caveat: the state DB is per working copy, but sandboxes live on the
+sbx machine, which may serve several working copies. A sandbox "unknown to
+this state DB" may simply belong to another checkout's runs — verdicts say
+so explicitly, and nothing is ever removed without ``--force``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import re
+import time
+
+from pydantic import BaseModel, ConfigDict
+
+from sbxloop.engine.store import StateStore
+from sbxloop.errors import SbxError, StateError
+from sbxloop.ids import is_run_id
+from sbxloop.sbx.cli import SbxCLI
+from sbxloop.sbx.models import SandboxInfo
+
+# A run in one of these states has already had (or never needed) its
+# teardown: any of its sandboxes still present are leaked. "failed" is safe
+# to include even though it is resumable — resume re-provisions a fresh pair.
+TERMINAL_RUN_STATES = frozenset({"completed", "failed", "cancelled"})
+
+# Age a run must be inactive before its sandboxes count as orphaned. Guards
+# against racing a run that another terminal just started or is mid-phase.
+DEFAULT_MIN_AGE_S = 3600.0
+
+_NAME_RE = re.compile(r"^sbxloop-(?P<run>[^-]+)-(?P<role>agent|github)$")
+
+
+class SandboxVerdict(BaseModel):
+    """One sandbox's classification: what we know and whether it is prunable."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    run_id: str | None = None  # parsed from the name; None → unrecognized
+    role: str | None = None
+    run_state: str | None = None  # None → unknown to this state DB
+    kept_reason: str | None = None
+    age_s: float | None = None  # since last DB activity; None → no signal
+    orphan: bool = False
+    reason: str
+
+
+def format_age(age_s: float | None) -> str:
+    if age_s is None:
+        return "?"
+    if age_s < 3600:
+        return f"{age_s / 60:.0f}m"
+    if age_s < 48 * 3600:
+        return f"{age_s / 3600:.1f}h"
+    return f"{age_s / 86400:.1f}d"
+
+
+def classify_sandboxes(
+    infos: list[SandboxInfo],
+    store: StateStore,
+    *,
+    min_age_s: float = DEFAULT_MIN_AGE_S,
+    include_kept: bool = False,
+    now: float | None = None,
+) -> list[SandboxVerdict]:
+    """Classify every ``sbxloop-*`` sandbox against the state DB.
+
+    Non-sbxloop sandboxes are never considered. Names that carry the prefix
+    but do not match the ``sbxloop-<run>-<role>`` scheme (future taxonomies:
+    warm-pool standby, etc.) are reported but never marked orphaned.
+    """
+    now = time.time() if now is None else now
+    verdicts: list[SandboxVerdict] = []
+    for info in infos:
+        if not info.name.startswith("sbxloop-"):
+            continue
+        verdicts.append(
+            _classify_one(info.name, store, min_age_s=min_age_s, include_kept=include_kept, now=now)
+        )
+    return verdicts
+
+
+def _classify_one(
+    name: str,
+    store: StateStore,
+    *,
+    min_age_s: float,
+    include_kept: bool,
+    now: float,
+) -> SandboxVerdict:
+    match = _NAME_RE.match(name)
+    if match is None or not is_run_id(match.group("run")):
+        return SandboxVerdict(
+            name=name,
+            reason="unrecognized sbxloop naming scheme; not touched",
+        )
+    run_id, role = match.group("run"), match.group("role")
+
+    try:
+        run = store.get_run(run_id)
+    except StateError:
+        return SandboxVerdict(
+            name=name,
+            run_id=run_id,
+            role=role,
+            orphan=True,
+            reason="unknown to this state DB (may belong to another working copy)",
+        )
+
+    if run.kept_reason is not None and not include_kept:
+        return SandboxVerdict(
+            name=name,
+            run_id=run_id,
+            role=role,
+            run_state=run.state,
+            kept_reason=run.kept_reason,
+            age_s=now - run.updated_at,
+            reason=f"kept ({run.kept_reason}); use --include-kept to prune",
+        )
+
+    # Liveness = the newest thing the run ever wrote: state transitions bump
+    # updated_at, and every bus event (heartbeats included) is persisted.
+    last_activity = max(run.updated_at, store.last_event_ts(run_id) or 0.0)
+    age_s = now - last_activity
+    kept_note = f", kept ({run.kept_reason})" if run.kept_reason is not None else ""
+
+    if run.state in TERMINAL_RUN_STATES:
+        if age_s >= min_age_s:
+            return SandboxVerdict(
+                name=name,
+                run_id=run_id,
+                role=role,
+                run_state=run.state,
+                kept_reason=run.kept_reason,
+                age_s=age_s,
+                orphan=True,
+                reason=f"run {run.state} {format_age(age_s)} ago{kept_note}",
+            )
+        return SandboxVerdict(
+            name=name,
+            run_id=run_id,
+            role=role,
+            run_state=run.state,
+            kept_reason=run.kept_reason,
+            age_s=age_s,
+            reason=f"run {run.state} only {format_age(age_s)} ago (younger than --min-age)",
+        )
+
+    if age_s >= min_age_s:
+        return SandboxVerdict(
+            name=name,
+            run_id=run_id,
+            role=role,
+            run_state=run.state,
+            kept_reason=run.kept_reason,
+            age_s=age_s,
+            orphan=True,
+            reason=f"run {run.state} but silent for {format_age(age_s)}{kept_note}",
+        )
+    return SandboxVerdict(
+        name=name,
+        run_id=run_id,
+        role=role,
+        run_state=run.state,
+        kept_reason=run.kept_reason,
+        age_s=age_s,
+        reason=f"run {run.state}, active {format_age(age_s)} ago (possibly live)",
+    )
+
+
+def remove_sandbox(cli: SbxCLI, name: str) -> None:
+    """Stop (best-effort) then force-remove one sandbox.
+
+    ``stop`` failing is expected for already-stopped sandboxes; ``rm``
+    failing propagates so callers can report it.
+    """
+    with contextlib.suppress(SbxError):
+        cli.stop(name)
+    cli.rm(name, force=True)
+
+
+def count_orphans(cli: SbxCLI, store: StateStore) -> int:
+    """Orphan-candidate count with default thresholds (doctor's view)."""
+    return sum(1 for v in classify_sandboxes(cli.ls(), store) if v.orphan)