sbxloop-worker 0.5.6__tar.gz → 0.5.8__tar.gz

{sbxloop_worker-0.5.6 → sbxloop_worker-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbxloop-worker
-Version: 0.5.6
+Version: 0.5.8
 Summary: In-sandbox worker runtime for sbxloop (protocol models, agent backends, job runner)
 Project-URL: Homepage, https://github.com/brettbergin/sbxloop
 Project-URL: Repository, https://github.com/brettbergin/sbxloop

{sbxloop_worker-0.5.6 → sbxloop_worker-0.5.8}/src/sbxloop_worker/__main__.py

@@ -66,6 +66,11 @@ def main(argv: list[str] | None = None) -> int:
     run.add_argument("--heartbeat", type=float, default=15.0)
     run.add_argument("--env-file", type=Path, default=DEFAULT_ENV_FILE)
     run.add_argument("--cwd", type=Path, default=None)
+    # Resource guardrail thresholds (percent used; 0 disables). Sampling on
+    # the heartbeat always happens — thresholds only add warn/abort levels.
+    run.add_argument("--disk-warn", type=float, default=0.0)
+    run.add_argument("--disk-abort", type=float, default=0.0)
+    run.add_argument("--mem-warn", type=float, default=0.0)
     args = parser.parse_args(argv)
 
     apply_env_file(args.env_file)
@@ -95,6 +100,9 @@ def main(argv: list[str] | None = None) -> int:
             events_path=args.events,
             result_path=args.result,
             heartbeat_s=args.heartbeat,
+            disk_warn=args.disk_warn,
+            disk_abort=args.disk_abort,
+            mem_warn=args.mem_warn,
         ).run()
     except BaseException as exc:
         print(f"sbxloop_worker: fatal: {exc}", file=sys.stderr)

{sbxloop_worker-0.5.6 → sbxloop_worker-0.5.8}/src/sbxloop_worker/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.5.6'
-__version_tuple__ = version_tuple = (0, 5, 6)
+__version__ = version = '0.5.8'
+__version_tuple__ = version_tuple = (0, 5, 8)
 
 __commit_id__ = commit_id = None

{sbxloop_worker-0.5.6 → sbxloop_worker-0.5.8}/src/sbxloop_worker/protocol.py

@@ -193,3 +193,8 @@ class EventTypes:
 
     GH_OP_START = "gh.op_start"
     GH_OP_END = "gh.op_end"
+
+    # Resource telemetry sampled on the heartbeat cadence. Worker-emitted,
+    # but sandbox-scoped: the host enriches these with the sandbox role.
+    SANDBOX_RESOURCES = "sandbox.resources"
+    SANDBOX_RESOURCES_WARNING = "sandbox.resources_warning"

sbxloop_worker-0.5.8/src/sbxloop_worker/resources.py

@@ -0,0 +1,77 @@
+"""Cheap in-sandbox resource sampling for the heartbeat cadence.
+
+Everything here reads local kernel state (statvfs, /proc, loadavg) — no
+subprocesses — so a sample is unobservable in run wall-clock. Fields that
+cannot be read on the current platform are simply omitted; consumers treat
+every field as optional.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from pathlib import Path
+from typing import Any
+
+MEMINFO_PATH = Path("/proc/meminfo")
+
+# Severity ordering for escalation edges (ok -> warn -> abort).
+LEVEL_SEVERITY = {"ok": 0, "warn": 1, "abort": 2}
+
+
+def sample_resources(path: str = ".") -> dict[str, Any]:
+    """One resource sample: disk usage of ``path``'s filesystem (the worker
+    runs with cwd = the run workspace), memory from /proc/meminfo, and the
+    1-minute load average."""
+    sample: dict[str, Any] = {}
+    try:
+        st = os.statvfs(path)
+        total = st.f_blocks * st.f_frsize
+        free = st.f_bavail * st.f_frsize
+        if total > 0:
+            sample["disk_total_bytes"] = total
+            sample["disk_free_bytes"] = free
+            sample["disk_used_pct"] = round(100.0 * (1.0 - free / total), 1)
+    except OSError:
+        pass
+    try:
+        fields: dict[str, int] = {}
+        for line in MEMINFO_PATH.read_text().splitlines():
+            key, _, rest = line.partition(":")
+            if key in ("MemTotal", "MemAvailable"):
+                fields[key] = int(rest.split()[0])  # kB
+        total_kb = fields.get("MemTotal", 0)
+        available_kb = fields.get("MemAvailable")
+        if total_kb > 0 and available_kb is not None:
+            sample["mem_total_kb"] = total_kb
+            sample["mem_available_kb"] = available_kb
+            sample["mem_used_pct"] = round(100.0 * (1.0 - available_kb / total_kb), 1)
+    except (OSError, ValueError, IndexError):
+        pass
+    with contextlib.suppress(OSError):
+        sample["load1"] = round(os.getloadavg()[0], 2)
+    return sample
+
+
+def classify_level(
+    sample: dict[str, Any],
+    *,
+    disk_warn: float = 0.0,
+    disk_abort: float = 0.0,
+    mem_warn: float = 0.0,
+) -> str:
+    """Guardrail level for a sample: "ok", "warn", or "abort".
+
+    A threshold of 0 (the worker default when the host passes none) disables
+    that guardrail. Only disk has an abort level — memory pressure warns.
+    """
+    disk = sample.get("disk_used_pct")
+    if isinstance(disk, (int, float)):
+        if disk_abort > 0 and disk >= disk_abort:
+            return "abort"
+        if disk_warn > 0 and disk >= disk_warn:
+            return "warn"
+    mem = sample.get("mem_used_pct")
+    if isinstance(mem, (int, float)) and mem_warn > 0 and mem >= mem_warn:
+        return "warn"
+    return "ok"

{sbxloop_worker-0.5.6 → sbxloop_worker-0.5.8}/src/sbxloop_worker/runner.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from sbxloop_worker.backends import get_backend
 from sbxloop_worker.events import EventWriter
 from sbxloop_worker.protocol import ErrorInfo, EventTypes, JobRequest, JobResult
+from sbxloop_worker.resources import LEVEL_SEVERITY, classify_level, sample_resources
 
 OUTPUT_TAIL_CHARS = 20_000
 
@@ -23,12 +24,20 @@ class JobRunner:
         *,
         heartbeat_s: float = 15.0,
         backend_name: str | None = None,
+        disk_warn: float = 0.0,
+        disk_abort: float = 0.0,
+        mem_warn: float = 0.0,
     ) -> None:
         self.job = job
         self.events_path = events_path
         self.result_path = result_path
         self.heartbeat_s = heartbeat_s
         self.backend_name = backend_name
+        self.disk_warn = disk_warn
+        self.disk_abort = disk_abort
+        self.mem_warn = mem_warn
+        self._resource_level = "ok"
+        self._resource_abort: str | None = None
 
     def run(self) -> JobResult:
         """Execute the job and write the authoritative result file.
@@ -39,6 +48,11 @@ class JobRunner:
             heartbeat_stop = self._start_heartbeat(writer)
             try:
                 writer.emit(EventTypes.WORKER_START, kind=self.job.kind)
+                # Baseline resource sample: even a job that finishes inside
+                # one heartbeat gets a datapoint, and a sandbox already past
+                # a threshold is flagged before work starts.
+                if self.heartbeat_s > 0:
+                    self._sample_and_emit(writer)
                 result = self._dispatch(writer)
             except subprocess.TimeoutExpired:
                 result = self._error_result("timeout", "Timeout", "job timed out")
@@ -52,6 +66,22 @@ class JobRunner:
             finally:
                 heartbeat_stop.set()
 
+            if self._resource_abort and result.status != "ok":
+                # The sandbox blew past disk_abort while this job ran: name
+                # the real cause instead of whatever confusing failure the
+                # in-VM tooling produced on a full disk.
+                original = ""
+                if result.error is not None:
+                    original = f"underlying failure: {result.error.type}: {result.error.message}"
+                    if result.error.detail:
+                        original += f"\n{result.error.detail}"
+                result = self._error_result(
+                    "error",
+                    "SandboxResourcesExhausted",
+                    self._resource_abort,
+                    detail=original[-OUTPUT_TAIL_CHARS:] or None,
+                )
+
             self.result_path.parent.mkdir(parents=True, exist_ok=True)
             self.result_path.write_text(result.model_dump_json())
             if result.status == "ok":
@@ -152,9 +182,50 @@ class JobRunner:
             while not stop.wait(self.heartbeat_s):
                 try:
                     writer.emit(EventTypes.WORKER_HEARTBEAT)
+                    self._sample_and_emit(writer)
                 except Exception:  # pragma: no cover - writer closed during shutdown
                     return
 
         thread = threading.Thread(target=beat, name="sbxloop-heartbeat", daemon=True)
         thread.start()
         return stop
+
+    def _sample_and_emit(self, writer: EventWriter) -> None:
+        """Emit one ``sandbox.resources`` sample; escalations additionally
+        emit a prominent warning event (edge-triggered, so a long run at 90%
+        disk produces one warning, not one per beat)."""
+        sample = sample_resources()
+        if not sample:
+            return
+        level = classify_level(
+            sample,
+            disk_warn=self.disk_warn,
+            disk_abort=self.disk_abort,
+            mem_warn=self.mem_warn,
+        )
+        writer.emit(EventTypes.SANDBOX_RESOURCES, level=level, **sample)
+        if LEVEL_SEVERITY[level] > LEVEL_SEVERITY[self._resource_level]:
+            writer.emit(
+                EventTypes.SANDBOX_RESOURCES_WARNING,
+                level=level,
+                message=self._level_message(level, sample),
+                **sample,
+            )
+        if level == "abort" and self._resource_abort is None:
+            self._resource_abort = self._level_message(level, sample)
+        self._resource_level = level
+
+    def _level_message(self, level: str, sample: dict[str, object]) -> str:
+        disk = sample.get("disk_used_pct")
+        mem = sample.get("mem_used_pct")
+        if level == "abort":
+            return (
+                f"sandbox disk exhausted: {disk}% of the workspace filesystem is used "
+                f"(disk_abort threshold: {self.disk_abort}%)"
+            )
+        parts = []
+        if isinstance(disk, (int, float)) and self.disk_warn > 0 and disk >= self.disk_warn:
+            parts.append(f"disk {disk}% used (disk_warn: {self.disk_warn}%)")
+        if isinstance(mem, (int, float)) and self.mem_warn > 0 and mem >= self.mem_warn:
+            parts.append(f"memory {mem}% used (mem_warn: {self.mem_warn}%)")
+        return "sandbox resources under pressure: " + ", ".join(parts)

sbxloop_worker-0.5.8/tests/test_resources.py

@@ -0,0 +1,178 @@
+"""Resource telemetry tests: heartbeat sampling, guardrail levels, and the
+disk-abort result rewrite (issue #54)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, ClassVar
+
+import pytest
+
+import sbxloop_worker.runner as runner_mod
+from sbxloop_worker.protocol import Event, EventTypes, JobRequest
+from sbxloop_worker.resources import classify_level, sample_resources
+from sbxloop_worker.runner import JobRunner
+
+
+def agent_job(**overrides: object) -> JobRequest:
+    base: dict[str, object] = {
+        "job_id": "j1",
+        "run_id": "r1",
+        "kind": "agent.session",
+        "prompt": "hello",
+    }
+    base.update(overrides)
+    return JobRequest.model_validate(base)
+
+
+def failing_job() -> JobRequest:
+    # github.op with no params fails fast with GithubOpError — a stand-in
+    # for "whatever confusing way tooling fails on a full disk".
+    return JobRequest(job_id="j1", run_id="r1", kind="github.op", op="issue.create")
+
+
+def run_with_sample(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    sample: dict[str, Any],
+    *,
+    job: JobRequest | None = None,
+    **runner_kwargs: float,
+) -> tuple[Any, list[Event]]:
+    """Run a job with a deterministic resource sample. heartbeat_s is large
+    so only the synchronous baseline sample fires."""
+    monkeypatch.setattr(runner_mod, "sample_resources", lambda path=".": dict(sample))
+    monkeypatch.setenv("SBXLOOP_WORKER_BACKEND", "echo")
+    events_path = tmp_path / "events.jsonl"
+    result = JobRunner(
+        job if job is not None else agent_job(),
+        events_path=events_path,
+        result_path=tmp_path / "result.json",
+        heartbeat_s=3600.0,
+        backend_name="echo",
+        **runner_kwargs,
+    ).run()
+    events = [Event.from_json_line(line) for line in events_path.read_text().splitlines()]
+    return result, events
+
+
+class TestSampleResources:
+    def test_disk_fields_present(self) -> None:
+        sample = sample_resources(".")
+        assert 0.0 <= sample["disk_used_pct"] <= 100.0
+        assert sample["disk_total_bytes"] > 0
+        assert sample["disk_free_bytes"] >= 0
+
+    def test_unreadable_path_omits_disk(self) -> None:
+        sample = sample_resources("/nonexistent/definitely/nope")
+        assert "disk_used_pct" not in sample
+
+
+class TestClassifyLevel:
+    LIMITS: ClassVar[dict[str, float]] = {"disk_warn": 85.0, "disk_abort": 95.0, "mem_warn": 90.0}
+
+    def test_zero_thresholds_disable(self) -> None:
+        assert classify_level({"disk_used_pct": 99.9, "mem_used_pct": 99.9}) == "ok"
+
+    def test_disk_levels(self) -> None:
+        assert classify_level({"disk_used_pct": 84.9}, **self.LIMITS) == "ok"
+        assert classify_level({"disk_used_pct": 85.0}, **self.LIMITS) == "warn"
+        assert classify_level({"disk_used_pct": 95.0}, **self.LIMITS) == "abort"
+
+    def test_memory_only_warns(self) -> None:
+        assert classify_level({"mem_used_pct": 99.0}, **self.LIMITS) == "warn"
+
+    def test_empty_sample_is_ok(self) -> None:
+        assert classify_level({}, **self.LIMITS) == "ok"
+
+
+class TestRunnerTelemetry:
+    def test_baseline_sample_emitted_after_start(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        result, events = run_with_sample(tmp_path, monkeypatch, {"disk_used_pct": 10.0})
+        assert result.status == "ok"
+        types = [e.type for e in events]
+        assert types[0] == EventTypes.WORKER_START
+        assert EventTypes.SANDBOX_RESOURCES in types
+        sample_event = next(e for e in events if e.type == EventTypes.SANDBOX_RESOURCES)
+        assert sample_event.data["level"] == "ok"
+        assert sample_event.data["disk_used_pct"] == 10.0
+        assert EventTypes.SANDBOX_RESOURCES_WARNING not in types
+
+    def test_no_sampling_when_heartbeat_disabled(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("SBXLOOP_WORKER_BACKEND", "echo")
+        events_path = tmp_path / "events.jsonl"
+        JobRunner(
+            agent_job(),
+            events_path=events_path,
+            result_path=tmp_path / "result.json",
+            heartbeat_s=0.0,
+            backend_name="echo",
+            disk_warn=85.0,
+            disk_abort=95.0,
+        ).run()
+        events = [Event.from_json_line(line) for line in events_path.read_text().splitlines()]
+        assert EventTypes.SANDBOX_RESOURCES not in [e.type for e in events]
+
+    def test_warn_threshold_emits_edge_triggered_warning(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        result, events = run_with_sample(
+            tmp_path,
+            monkeypatch,
+            {"disk_used_pct": 90.0},
+            disk_warn=85.0,
+            disk_abort=95.0,
+        )
+        assert result.status == "ok"  # warn never fails a job
+        warnings = [e for e in events if e.type == EventTypes.SANDBOX_RESOURCES_WARNING]
+        assert len(warnings) == 1
+        assert warnings[0].data["level"] == "warn"
+        assert "disk 90.0% used" in warnings[0].data["message"]
+
+    def test_abort_rewrites_failed_result(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        result, events = run_with_sample(
+            tmp_path,
+            monkeypatch,
+            {"disk_used_pct": 97.0},
+            job=failing_job(),
+            disk_warn=85.0,
+            disk_abort=95.0,
+        )
+        assert result.status == "error"
+        assert result.error.type == "SandboxResourcesExhausted"
+        assert "sandbox disk exhausted" in result.error.message
+        assert "97.0%" in result.error.message
+        # The original failure is preserved for diagnosis, not discarded.
+        assert "GithubOpError" in (result.error.detail or "")
+        warnings = [e for e in events if e.type == EventTypes.SANDBOX_RESOURCES_WARNING]
+        assert warnings and warnings[0].data["level"] == "abort"
+
+    def test_abort_never_rewrites_success(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        result, _ = run_with_sample(
+            tmp_path,
+            monkeypatch,
+            {"disk_used_pct": 97.0},
+            disk_warn=85.0,
+            disk_abort=95.0,
+        )
+        assert result.status == "ok"
+
+    def test_mem_pressure_warns(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+        result, events = run_with_sample(
+            tmp_path,
+            monkeypatch,
+            {"disk_used_pct": 10.0, "mem_used_pct": 95.0},
+            mem_warn=90.0,
+        )
+        assert result.status == "ok"
+        warnings = [e for e in events if e.type == EventTypes.SANDBOX_RESOURCES_WARNING]
+        assert len(warnings) == 1
+        assert "memory 95.0% used" in warnings[0].data["message"]