sbxloop 0.5.4__tar.gz → 0.5.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sbxloop-0.5.4 → sbxloop-0.5.6}/PKG-INFO +1 -1
- sbxloop-0.5.4/src/sbxloop/_vendor/sbxloop_worker-0.5.4-py3-none-any.whl → sbxloop-0.5.6/src/sbxloop/_vendor/sbxloop_worker-0.5.6-py3-none-any.whl +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/_version.py +2 -2
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/cli/app.py +148 -1
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/cli/doctor.py +24 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/config.py +5 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/engine.py +69 -34
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/model.py +7 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/store.py +28 -1
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/events.py +1 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/cli.py +16 -0
- sbxloop-0.5.6/src/sbxloop/sbx/prune.py +191 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/.gitignore +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/README.md +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/hatch_build.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/pyproject.toml +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/cli/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/cli/tui.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/deliver.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/phases.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/prompts/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/prompts/decompose.md +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/prompts/execute.md +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/prompts/plan.md +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/prompts/scrutinize.md +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/engine/prompts/validate.md +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/errors.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/gh/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/gh/ops.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/gh/reporter.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/ids.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/py.typed +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/models.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/pair.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/parse.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/provision.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/sbx/sandbox.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/worker/__init__.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/worker/client.py +0 -0
- {sbxloop-0.5.4 → sbxloop-0.5.6}/src/sbxloop/worker/wheel.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sbxloop
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.6
|
|
4
4
|
Summary: Agentic loop orchestration on Docker Sandboxes (sbx) with isolated credential domains
|
|
5
5
|
Project-URL: Homepage, https://github.com/brettbergin/sbxloop
|
|
6
6
|
Project-URL: Repository, https://github.com/brettbergin/sbxloop
|
|
Binary file
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.5.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 5,
|
|
21
|
+
__version__ = version = '0.5.6'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 5, 6)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -25,6 +25,7 @@ from sbxloop.errors import SdxloopError
|
|
|
25
25
|
from sbxloop.events import Event
|
|
26
26
|
from sbxloop.sbx.cli import SbxCLI
|
|
27
27
|
from sbxloop.sbx.provision import sandbox_name
|
|
28
|
+
from sbxloop.sbx.prune import classify_sandboxes, format_age, remove_sandbox
|
|
28
29
|
|
|
29
30
|
app = typer.Typer(
|
|
30
31
|
name="sbxloop",
|
|
@@ -168,6 +169,10 @@ def _finish(result: RunResult, config: Config) -> None:
|
|
|
168
169
|
for task in result.tasks:
|
|
169
170
|
console.print(f" {task.spec.id}: {task.state} ({task.spec.title})")
|
|
170
171
|
_print_artifacts_summary(result, config)
|
|
172
|
+
if result.kept_sandboxes:
|
|
173
|
+
console.print(f"\n[bold yellow]sandboxes kept:[/] {', '.join(result.kept_sandboxes)}")
|
|
174
|
+
console.print(f" inspect: [cyan]sbxloop shell {result.run_id}[/] (--role github)")
|
|
175
|
+
console.print(f" remove: [cyan]sbxloop sandbox rm --run {result.run_id}[/]")
|
|
171
176
|
raise typer.Exit(0 if result.succeeded else 1)
|
|
172
177
|
|
|
173
178
|
|
|
@@ -192,10 +197,21 @@ def run(
|
|
|
192
197
|
keep_sandboxes: Annotated[
|
|
193
198
|
bool, typer.Option("--keep-sandboxes", help="Do not remove sandboxes at the end.")
|
|
194
199
|
] = False,
|
|
200
|
+
keep_on_failure: Annotated[
|
|
201
|
+
bool | None,
|
|
202
|
+
typer.Option(
|
|
203
|
+
"--keep-on-failure/--no-keep-on-failure",
|
|
204
|
+
help="Keep the sandbox pair alive when the run fails (inspect with `sbxloop shell`).",
|
|
205
|
+
),
|
|
206
|
+
] = None,
|
|
195
207
|
tui: Annotated[bool, typer.Option("--tui/--no-tui", help="Live dashboard.")] = True,
|
|
196
208
|
) -> None:
|
|
197
209
|
"""Run an agentic loop for OUTCOME in a fresh sandbox pair."""
|
|
198
|
-
config = _config_with_overrides(
|
|
210
|
+
config = _config_with_overrides(
|
|
211
|
+
model=model,
|
|
212
|
+
keep_sandboxes=keep_sandboxes or None,
|
|
213
|
+
keep_on_failure=keep_on_failure,
|
|
214
|
+
)
|
|
199
215
|
if report is not None:
|
|
200
216
|
config = config.model_copy(
|
|
201
217
|
update={"github": config.github.model_copy(update={"report": report})}
|
|
@@ -330,6 +346,57 @@ def logs(
|
|
|
330
346
|
time.sleep(0.5)
|
|
331
347
|
|
|
332
348
|
|
|
349
|
+
# Prefer bash when the template has it, fall back to POSIX sh. `sbx exec`
|
|
350
|
+
# has no documented -it flags; terminal attachment is inherited stdio.
|
|
351
|
+
_INTERACTIVE_SHELL = ("sh", "-c", "command -v bash >/dev/null && exec bash -l; exec sh -l")
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@app.command()
|
|
355
|
+
def shell(
|
|
356
|
+
run_id: Annotated[str, typer.Argument(help="Run id.")],
|
|
357
|
+
role: Annotated[
|
|
358
|
+
str, typer.Option("--role", help="Which sandbox of the pair: agent or github.")
|
|
359
|
+
] = "agent",
|
|
360
|
+
command: Annotated[
|
|
361
|
+
str | None,
|
|
362
|
+
typer.Option(
|
|
363
|
+
"--command", "-c", help="Run one shell command instead of an interactive shell."
|
|
364
|
+
),
|
|
365
|
+
] = None,
|
|
366
|
+
) -> None:
|
|
367
|
+
"""Open a shell inside a run's sandbox (kept, in-flight, or leaked).
|
|
368
|
+
|
|
369
|
+
Attaching to an in-flight run is meant as observation: the worker owns
|
|
370
|
+
its env files and workspace, so avoid mutating them mid-phase.
|
|
371
|
+
"""
|
|
372
|
+
if role not in ("agent", "github"):
|
|
373
|
+
console.print(f"[bold red]invalid --role {role!r}:[/] must be agent or github")
|
|
374
|
+
raise typer.Exit(2)
|
|
375
|
+
config = load_config()
|
|
376
|
+
store = _store(config)
|
|
377
|
+
try:
|
|
378
|
+
store.get_run(run_id)
|
|
379
|
+
except SdxloopError as exc:
|
|
380
|
+
console.print(f"[bold red]{exc}[/]")
|
|
381
|
+
raise typer.Exit(2) from exc
|
|
382
|
+
cli = SbxCLI(app_name=config.app_name or None)
|
|
383
|
+
name = sandbox_name(run_id, "agent" if role == "agent" else "github")
|
|
384
|
+
try:
|
|
385
|
+
live = any(info.name == name for info in cli.ls())
|
|
386
|
+
except SdxloopError as exc:
|
|
387
|
+
console.print(f"[bold red]{exc}[/]")
|
|
388
|
+
raise typer.Exit(2) from exc
|
|
389
|
+
if not live:
|
|
390
|
+
console.print(
|
|
391
|
+
f"[bold red]sandbox {name} is not running.[/] Sandboxes are removed at run end "
|
|
392
|
+
"unless kept (keep_on_failure, --keep-sandboxes), and kept ones may have been "
|
|
393
|
+
"pruned since."
|
|
394
|
+
)
|
|
395
|
+
raise typer.Exit(2)
|
|
396
|
+
argv = ("sh", "-lc", command) if command else _INTERACTIVE_SHELL
|
|
397
|
+
raise typer.Exit(cli.exec_interactive(name, argv))
|
|
398
|
+
|
|
399
|
+
|
|
333
400
|
@app.command()
|
|
334
401
|
def artifacts(
|
|
335
402
|
run_id: Annotated[str, typer.Argument(help="Run id.")],
|
|
@@ -414,6 +481,84 @@ def sandbox_rm(
|
|
|
414
481
|
console.print(f"[yellow]skip {target}:[/] {exc}")
|
|
415
482
|
|
|
416
483
|
|
|
484
|
+
@sandbox_app.command("prune")
|
|
485
|
+
def sandbox_prune(
|
|
486
|
+
force: Annotated[
|
|
487
|
+
bool,
|
|
488
|
+
typer.Option("--force", "--yes", help="Actually remove (default is a dry run)."),
|
|
489
|
+
] = False,
|
|
490
|
+
min_age: Annotated[
|
|
491
|
+
float,
|
|
492
|
+
typer.Option(
|
|
493
|
+
"--min-age",
|
|
494
|
+
help="Hours a run must be inactive before its sandboxes count as orphaned.",
|
|
495
|
+
),
|
|
496
|
+
] = 1.0,
|
|
497
|
+
include_kept: Annotated[
|
|
498
|
+
bool,
|
|
499
|
+
typer.Option("--include-kept", help="Also prune kept-for-debugging sandboxes."),
|
|
500
|
+
] = False,
|
|
501
|
+
) -> None:
|
|
502
|
+
"""Garbage-collect orphaned sbxloop sandboxes (crashed hosts, killed runs).
|
|
503
|
+
|
|
504
|
+
Cross-references `sbx ls` against this working copy's state DB. Dry-run
|
|
505
|
+
by default: prints the classification and removes nothing without
|
|
506
|
+
--force.
|
|
507
|
+
"""
|
|
508
|
+
config = load_config()
|
|
509
|
+
store = _store(config)
|
|
510
|
+
cli = SbxCLI(app_name=config.app_name or None)
|
|
511
|
+
try:
|
|
512
|
+
verdicts = classify_sandboxes(
|
|
513
|
+
cli.ls(), store, min_age_s=min_age * 3600.0, include_kept=include_kept
|
|
514
|
+
)
|
|
515
|
+
except SdxloopError as exc:
|
|
516
|
+
console.print(f"[bold red]{exc}[/]")
|
|
517
|
+
raise typer.Exit(2) from exc
|
|
518
|
+
if not verdicts:
|
|
519
|
+
console.print("no sbxloop sandboxes found")
|
|
520
|
+
return
|
|
521
|
+
|
|
522
|
+
table = Table(title="sbxloop sandbox prune")
|
|
523
|
+
for column in ("sandbox", "run", "run state", "age", "verdict"):
|
|
524
|
+
table.add_column(column)
|
|
525
|
+
for v in verdicts:
|
|
526
|
+
table.add_row(
|
|
527
|
+
v.name,
|
|
528
|
+
v.run_id or "",
|
|
529
|
+
v.run_state or "[dim]unknown[/]",
|
|
530
|
+
format_age(v.age_s),
|
|
531
|
+
("[red]orphan[/] — " if v.orphan else "[green]keep[/] — ") + v.reason,
|
|
532
|
+
)
|
|
533
|
+
console.print(table)
|
|
534
|
+
console.print(
|
|
535
|
+
"[dim]note: the state DB is per working copy — 'unknown' sandboxes may "
|
|
536
|
+
"belong to another checkout's runs on this sbx host[/]"
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
orphans = [v for v in verdicts if v.orphan]
|
|
540
|
+
if not orphans:
|
|
541
|
+
console.print("nothing to prune")
|
|
542
|
+
return
|
|
543
|
+
if not force:
|
|
544
|
+
console.print(f"dry run: {len(orphans)} orphan candidate(s); re-run with --force to remove")
|
|
545
|
+
return
|
|
546
|
+
failures = 0
|
|
547
|
+
for v in orphans:
|
|
548
|
+
try:
|
|
549
|
+
remove_sandbox(cli, v.name)
|
|
550
|
+
except SdxloopError as exc:
|
|
551
|
+
failures += 1
|
|
552
|
+
console.print(f"[yellow]skip {v.name}:[/] {exc}")
|
|
553
|
+
continue
|
|
554
|
+
console.print(f"removed {v.name}")
|
|
555
|
+
# A pruned kept run is no longer kept; keep the DB marker honest.
|
|
556
|
+
if v.kept_reason is not None and v.run_id is not None:
|
|
557
|
+
store.set_run_kept(v.run_id, None)
|
|
558
|
+
if failures:
|
|
559
|
+
raise typer.Exit(1)
|
|
560
|
+
|
|
561
|
+
|
|
417
562
|
@config_app.command("show")
|
|
418
563
|
def config_show() -> None:
|
|
419
564
|
"""Show the resolved configuration and where each value came from."""
|
|
@@ -477,6 +622,8 @@ app_name = ""
|
|
|
477
622
|
state_dir = ".sbxloop"
|
|
478
623
|
# Keep sandboxes around after a run (for debugging).
|
|
479
624
|
keep_sandboxes = false
|
|
625
|
+
# Keep the pair alive only when a run fails; inspect with `sbxloop shell <run>`.
|
|
626
|
+
keep_on_failure = false
|
|
480
627
|
# Worker transport: "stream" (default) or "poll".
|
|
481
628
|
worker_transport = "stream"
|
|
482
629
|
# Secret injection: "proxy" (sbx keychain proxy; recommended) or "plain-env".
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import sqlite3
|
|
5
6
|
from collections.abc import Callable
|
|
6
7
|
from dataclasses import dataclass
|
|
7
8
|
|
|
@@ -9,6 +10,7 @@ from rich.console import Console
|
|
|
9
10
|
from rich.table import Table
|
|
10
11
|
|
|
11
12
|
from sbxloop.config import load_config
|
|
13
|
+
from sbxloop.engine.store import StateStore
|
|
12
14
|
from sbxloop.errors import SbxError, SbxNotFoundError
|
|
13
15
|
from sbxloop.sbx.cli import SbxCLI
|
|
14
16
|
from sbxloop.sbx.provision import (
|
|
@@ -16,6 +18,7 @@ from sbxloop.sbx.provision import (
|
|
|
16
18
|
COPILOT_TOKEN_ENV,
|
|
17
19
|
GH_TOKEN_ENVS,
|
|
18
20
|
)
|
|
21
|
+
from sbxloop.sbx.prune import count_orphans
|
|
19
22
|
from sbxloop.worker.wheel import resolve_worker_wheel
|
|
20
23
|
|
|
21
24
|
TESTED_SBX_SERIES = "0.35"
|
|
@@ -67,8 +70,10 @@ def collect_checks(
|
|
|
67
70
|
if version is not None:
|
|
68
71
|
# login / daemon reachable
|
|
69
72
|
report("checking sbx login")
|
|
73
|
+
logged_in = False
|
|
70
74
|
try:
|
|
71
75
|
cli.ls()
|
|
76
|
+
logged_in = True
|
|
72
77
|
checks.append(Check("sbx login", True, "sbx ls succeeded"))
|
|
73
78
|
except SbxError as exc:
|
|
74
79
|
login_cmd = (
|
|
@@ -76,6 +81,25 @@ def collect_checks(
|
|
|
76
81
|
)
|
|
77
82
|
checks.append(Check("sbx login", False, f"sbx ls failed ({exc}); run `{login_cmd}`"))
|
|
78
83
|
|
|
84
|
+
# orphaned sandboxes (leaked pairs from crashed/killed runs)
|
|
85
|
+
if logged_in:
|
|
86
|
+
report("checking for orphaned sandboxes")
|
|
87
|
+
try:
|
|
88
|
+
orphans = count_orphans(cli, StateStore(config.state_dir / "state.db"))
|
|
89
|
+
except (SbxError, OSError, sqlite3.Error):
|
|
90
|
+
orphans = None
|
|
91
|
+
if orphans is not None:
|
|
92
|
+
checks.append(
|
|
93
|
+
Check(
|
|
94
|
+
"orphaned sandboxes",
|
|
95
|
+
orphans == 0,
|
|
96
|
+
"none found"
|
|
97
|
+
if orphans == 0
|
|
98
|
+
else f"{orphans} orphan candidate(s) — run `sbxloop sandbox prune`",
|
|
99
|
+
hard=False,
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
|
|
79
103
|
# network policy reachable for the copilot hosts
|
|
80
104
|
for host in AGENT_TOKEN_HOSTS:
|
|
81
105
|
report(f"checking network policy for {host}")
|
|
@@ -85,6 +85,11 @@ class Config(_ConfigModel):
|
|
|
85
85
|
app_name: str = ""
|
|
86
86
|
state_dir: Path = Path(".sbxloop")
|
|
87
87
|
keep_sandboxes: bool = False
|
|
88
|
+
# Keep the pair alive only when a run fails, so the evidence (worker
|
|
89
|
+
# stderr, install leftovers, workspace state) survives for
|
|
90
|
+
# `sbxloop shell <run>`. Kept runs are marked in the state DB and
|
|
91
|
+
# eventually collectable via `sbxloop sandbox prune --include-kept`.
|
|
92
|
+
keep_on_failure: bool = False
|
|
88
93
|
worker_transport: WorkerTransport = "stream"
|
|
89
94
|
secret_strategy: SecretStrategy = "proxy"
|
|
90
95
|
# Advanced: in-sandbox interpreter for the worker, and whether to run the
|
|
@@ -113,49 +113,60 @@ class LoopEngine:
|
|
|
113
113
|
pair = provisioner.ensure_pair(run_id)
|
|
114
114
|
assert pair.workspace is not None
|
|
115
115
|
self.store.set_run_workspace(run_id, pair.workspace, pair.mounted)
|
|
116
|
+
if pair.keep:
|
|
117
|
+
# keep_sandboxes: mark up front so `sandbox prune` respects it.
|
|
118
|
+
self.store.set_run_kept(run_id, "manual")
|
|
116
119
|
try:
|
|
117
120
|
with pair:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
transport=self.config.worker_transport,
|
|
122
|
-
python=self.worker_python,
|
|
123
|
-
)
|
|
124
|
-
github = (
|
|
125
|
-
WorkerClient(
|
|
126
|
-
pair.github,
|
|
121
|
+
try:
|
|
122
|
+
agent = WorkerClient(
|
|
123
|
+
pair.agent,
|
|
127
124
|
self.bus,
|
|
128
125
|
transport=self.config.worker_transport,
|
|
129
126
|
python=self.worker_python,
|
|
130
127
|
)
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
reporter, detach = self._attach_reporter(github, run_id, outcome)
|
|
141
|
-
try:
|
|
142
|
-
phases = PhaseRunner(
|
|
143
|
-
agent, self.config, run_id, outcome, workdir=pair.agent_workdir
|
|
128
|
+
github = (
|
|
129
|
+
WorkerClient(
|
|
130
|
+
pair.github,
|
|
131
|
+
self.bus,
|
|
132
|
+
transport=self.config.worker_transport,
|
|
133
|
+
python=self.worker_python,
|
|
134
|
+
)
|
|
135
|
+
if pair.github is not None
|
|
136
|
+
else None
|
|
144
137
|
)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
138
|
+
if self.install_workers:
|
|
139
|
+
# ensure_dev_tools: the agent builds projects in this VM
|
|
140
|
+
# (venvs, pip) — the github sandbox only runs API ops.
|
|
141
|
+
agent.install(extras="copilot", ensure_dev_tools=True)
|
|
142
|
+
if github is not None:
|
|
143
|
+
github.install(extras="")
|
|
144
|
+
reporter, detach = self._attach_reporter(github, run_id, outcome)
|
|
145
|
+
try:
|
|
146
|
+
phases = PhaseRunner(
|
|
147
|
+
agent, self.config, run_id, outcome, workdir=pair.agent_workdir
|
|
148
|
+
)
|
|
149
|
+
state = self._run_phases(run_id, phases, deadline, pair)
|
|
150
|
+
# Summary must post while the github sandbox is alive;
|
|
151
|
+
# on an infra exception the run is resumable and the
|
|
152
|
+
# resumed run reopens the same tracking issue.
|
|
153
|
+
if reporter is not None:
|
|
154
|
+
reporter.close_run(run_id, state)
|
|
155
|
+
finally:
|
|
156
|
+
detach()
|
|
157
|
+
# Harvest even when a phase raised: the sandbox is still
|
|
158
|
+
# alive here, and partial artifacts beat none.
|
|
159
|
+
self._harvest(run_id, pair)
|
|
160
|
+
self._report_artifacts(run_id, pair)
|
|
161
|
+
except SdxloopError:
|
|
162
|
+
# Infra failures (install, worker, sbx) are exactly what
|
|
163
|
+
# gets diagnosed in-sandbox; decide keep before pair exit.
|
|
164
|
+
self._keep_on_failure(run_id, pair)
|
|
165
|
+
raise
|
|
157
166
|
if state == "completed":
|
|
158
167
|
self._deliver(run_id, outcome, pair, github)
|
|
168
|
+
else:
|
|
169
|
+
self._keep_on_failure(run_id, pair)
|
|
159
170
|
except SdxloopError:
|
|
160
171
|
# State is already persisted; the exception is the kill signal.
|
|
161
172
|
raise
|
|
@@ -168,6 +179,30 @@ class LoopEngine:
|
|
|
168
179
|
tasks=tasks,
|
|
169
180
|
workspace=pair.workspace,
|
|
170
181
|
mounted=pair.mounted,
|
|
182
|
+
kept_sandboxes=self._pair_names(pair) if pair.keep else [],
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
@staticmethod
|
|
186
|
+
def _pair_names(pair: SandboxPair) -> list[str]:
|
|
187
|
+
return [s.name for s in (pair.agent, pair.github) if s is not None]
|
|
188
|
+
|
|
189
|
+
def _keep_on_failure(self, run_id: str, pair: SandboxPair) -> None:
|
|
190
|
+
"""Flip the pair to kept when configured, so a failed run's evidence
|
|
191
|
+
survives for `sbxloop shell`. Marked in the DB for `sandbox prune`."""
|
|
192
|
+
if not self.config.keep_on_failure or pair.keep:
|
|
193
|
+
return
|
|
194
|
+
pair.keep = True
|
|
195
|
+
self.store.set_run_kept(run_id, "debug")
|
|
196
|
+
names = self._pair_names(pair)
|
|
197
|
+
self.bus.emit(
|
|
198
|
+
HostEventTypes.RUN_KEEP,
|
|
199
|
+
run_id,
|
|
200
|
+
sandboxes=names,
|
|
201
|
+
reason="debug",
|
|
202
|
+
message=(
|
|
203
|
+
f"sandboxes kept for debugging: {', '.join(names)} — "
|
|
204
|
+
f"inspect with `sbxloop shell {run_id}`"
|
|
205
|
+
),
|
|
171
206
|
)
|
|
172
207
|
|
|
173
208
|
def _attach_reporter(
|
|
@@ -142,6 +142,10 @@ class RunRecord(_Model):
|
|
|
142
142
|
# agent VM (False → artifacts are harvested to runs/<run>/artifacts).
|
|
143
143
|
workspace: Path | None = None
|
|
144
144
|
mounted: bool = False
|
|
145
|
+
# Why this run's sandboxes were deliberately left alive ("debug",
|
|
146
|
+
# "manual"); None means normal teardown applied. `sandbox prune`
|
|
147
|
+
# excludes kept runs unless asked to include them.
|
|
148
|
+
kept_reason: str | None = None
|
|
145
149
|
|
|
146
150
|
|
|
147
151
|
class RunResult(_Model):
|
|
@@ -152,6 +156,9 @@ class RunResult(_Model):
|
|
|
152
156
|
tasks: list[TaskRecord] = Field(default_factory=list)
|
|
153
157
|
workspace: Path | None = None
|
|
154
158
|
mounted: bool = False
|
|
159
|
+
# Sandbox names deliberately left alive (keep_sandboxes/keep_on_failure),
|
|
160
|
+
# so callers can point the user at `sbxloop shell`.
|
|
161
|
+
kept_sandboxes: list[str] = Field(default_factory=list)
|
|
155
162
|
|
|
156
163
|
@property
|
|
157
164
|
def succeeded(self) -> bool:
|
|
@@ -33,7 +33,8 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
33
33
|
created_at REAL NOT NULL,
|
|
34
34
|
updated_at REAL NOT NULL,
|
|
35
35
|
workspace TEXT,
|
|
36
|
-
mounted INTEGER NOT NULL DEFAULT 0
|
|
36
|
+
mounted INTEGER NOT NULL DEFAULT 0,
|
|
37
|
+
kept_reason TEXT
|
|
37
38
|
);
|
|
38
39
|
CREATE TABLE IF NOT EXISTS tasks (
|
|
39
40
|
run_id TEXT NOT NULL,
|
|
@@ -75,6 +76,7 @@ CREATE INDEX IF NOT EXISTS idx_events_run ON events (run_id, seq);
|
|
|
75
76
|
_RUNS_MIGRATIONS = (
|
|
76
77
|
("workspace", "ALTER TABLE runs ADD COLUMN workspace TEXT"),
|
|
77
78
|
("mounted", "ALTER TABLE runs ADD COLUMN mounted INTEGER NOT NULL DEFAULT 0"),
|
|
79
|
+
("kept_reason", "ALTER TABLE runs ADD COLUMN kept_reason TEXT"),
|
|
78
80
|
)
|
|
79
81
|
|
|
80
82
|
|
|
@@ -130,6 +132,18 @@ class StateStore:
|
|
|
130
132
|
raise StateError(f"unknown run {run_id}")
|
|
131
133
|
self._conn.commit()
|
|
132
134
|
|
|
135
|
+
def set_run_kept(self, run_id: str, reason: str | None) -> None:
|
|
136
|
+
"""Mark a run's sandboxes as deliberately kept (``"debug"``,
|
|
137
|
+
``"manual"``), or clear the marker with None. ``sandbox prune``
|
|
138
|
+
excludes kept runs unless explicitly told otherwise."""
|
|
139
|
+
cursor = self._conn.execute(
|
|
140
|
+
"UPDATE runs SET kept_reason = ? WHERE run_id = ?",
|
|
141
|
+
(reason, run_id),
|
|
142
|
+
)
|
|
143
|
+
if cursor.rowcount == 0:
|
|
144
|
+
raise StateError(f"unknown run {run_id}")
|
|
145
|
+
self._conn.commit()
|
|
146
|
+
|
|
133
147
|
@staticmethod
|
|
134
148
|
def _run_record(row: sqlite3.Row) -> RunRecord:
|
|
135
149
|
return RunRecord(
|
|
@@ -140,6 +154,7 @@ class StateStore:
|
|
|
140
154
|
updated_at=row["updated_at"],
|
|
141
155
|
workspace=Path(row["workspace"]) if row["workspace"] else None,
|
|
142
156
|
mounted=bool(row["mounted"]),
|
|
157
|
+
kept_reason=row["kept_reason"],
|
|
143
158
|
)
|
|
144
159
|
|
|
145
160
|
def get_run(self, run_id: str) -> RunRecord:
|
|
@@ -243,6 +258,18 @@ class StateStore:
|
|
|
243
258
|
|
|
244
259
|
# -- events ------------------------------------------------------------
|
|
245
260
|
|
|
261
|
+
def last_event_ts(self, run_id: str) -> float | None:
|
|
262
|
+
"""Timestamp of the run's most recent persisted event, if any.
|
|
263
|
+
|
|
264
|
+
Every bus event (heartbeats included) is persisted, so this is the
|
|
265
|
+
best liveness signal available for a run whose recorded state says
|
|
266
|
+
non-terminal but whose process may be long dead.
|
|
267
|
+
"""
|
|
268
|
+
row = self._conn.execute(
|
|
269
|
+
"SELECT MAX(ts) AS ts FROM events WHERE run_id = ?", (run_id,)
|
|
270
|
+
).fetchone()
|
|
271
|
+
return row["ts"] if row and row["ts"] is not None else None
|
|
272
|
+
|
|
246
273
|
def append_event(self, event: Event) -> None:
|
|
247
274
|
self._conn.execute(
|
|
248
275
|
"INSERT INTO events (run_id, ts, type, job_id, data_json) VALUES (?, ?, ?, ?, ?)",
|
|
@@ -155,6 +155,22 @@ class SbxCLI:
|
|
|
155
155
|
raise self._error_for(result)
|
|
156
156
|
return result
|
|
157
157
|
|
|
158
|
+
def exec_interactive(self, name: str, cmd: Sequence[str]) -> int:
|
|
159
|
+
"""Run a command in a sandbox with the caller's terminal attached
|
|
160
|
+
(stdin/stdout/stderr inherited); returns the command's exit code.
|
|
161
|
+
|
|
162
|
+
`sbx exec` documents no -it-style flags; terminal attachment is
|
|
163
|
+
simply inheriting the caller's stdio.
|
|
164
|
+
"""
|
|
165
|
+
argv = self.argv("exec", name, *cmd)
|
|
166
|
+
try:
|
|
167
|
+
return subprocess.run(argv, check=False).returncode
|
|
168
|
+
except FileNotFoundError as exc:
|
|
169
|
+
raise SbxNotFoundError(
|
|
170
|
+
f"sbx binary {self.binary!r} not found on PATH",
|
|
171
|
+
argv=argv,
|
|
172
|
+
) from exc
|
|
173
|
+
|
|
158
174
|
def cp(self, src: str, dst: str, *, timeout: float | None = None) -> None:
|
|
159
175
|
self.run("cp", src, dst, timeout=timeout)
|
|
160
176
|
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Orphaned-sandbox classification for ``sbxloop sandbox prune`` and doctor.
|
|
2
|
+
|
|
3
|
+
The in-process cleanup registry only protects against failures inside a live
|
|
4
|
+
sbxloop process; a host crash, OOM-kill, or ``kill -9`` leaves the run's
|
|
5
|
+
sandbox pair running indefinitely. This module cross-references ``sbx ls``
|
|
6
|
+
against the state DB to find such leaks safely.
|
|
7
|
+
|
|
8
|
+
Honesty caveat: the state DB is per working copy, but sandboxes live on the
|
|
9
|
+
sbx machine, which may serve several working copies. A sandbox "unknown to
|
|
10
|
+
this state DB" may simply belong to another checkout's runs — verdicts say
|
|
11
|
+
so explicitly, and nothing is ever removed without ``--force``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import contextlib
|
|
17
|
+
import re
|
|
18
|
+
import time
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict
|
|
21
|
+
|
|
22
|
+
from sbxloop.engine.store import StateStore
|
|
23
|
+
from sbxloop.errors import SbxError, StateError
|
|
24
|
+
from sbxloop.ids import is_run_id
|
|
25
|
+
from sbxloop.sbx.cli import SbxCLI
|
|
26
|
+
from sbxloop.sbx.models import SandboxInfo
|
|
27
|
+
|
|
28
|
+
# A run in one of these states has already had (or never needed) its
|
|
29
|
+
# teardown: any of its sandboxes still present are leaked. "failed" is safe
|
|
30
|
+
# to include even though it is resumable — resume re-provisions a fresh pair.
|
|
31
|
+
TERMINAL_RUN_STATES = frozenset({"completed", "failed", "cancelled"})
|
|
32
|
+
|
|
33
|
+
# Age a run must be inactive before its sandboxes count as orphaned. Guards
|
|
34
|
+
# against racing a run that another terminal just started or is mid-phase.
|
|
35
|
+
DEFAULT_MIN_AGE_S = 3600.0
|
|
36
|
+
|
|
37
|
+
_NAME_RE = re.compile(r"^sbxloop-(?P<run>[^-]+)-(?P<role>agent|github)$")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class SandboxVerdict(BaseModel):
|
|
41
|
+
"""One sandbox's classification: what we know and whether it is prunable."""
|
|
42
|
+
|
|
43
|
+
model_config = ConfigDict(extra="forbid")
|
|
44
|
+
|
|
45
|
+
name: str
|
|
46
|
+
run_id: str | None = None # parsed from the name; None → unrecognized
|
|
47
|
+
role: str | None = None
|
|
48
|
+
run_state: str | None = None # None → unknown to this state DB
|
|
49
|
+
kept_reason: str | None = None
|
|
50
|
+
age_s: float | None = None # since last DB activity; None → no signal
|
|
51
|
+
orphan: bool = False
|
|
52
|
+
reason: str
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def format_age(age_s: float | None) -> str:
|
|
56
|
+
if age_s is None:
|
|
57
|
+
return "?"
|
|
58
|
+
if age_s < 3600:
|
|
59
|
+
return f"{age_s / 60:.0f}m"
|
|
60
|
+
if age_s < 48 * 3600:
|
|
61
|
+
return f"{age_s / 3600:.1f}h"
|
|
62
|
+
return f"{age_s / 86400:.1f}d"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def classify_sandboxes(
|
|
66
|
+
infos: list[SandboxInfo],
|
|
67
|
+
store: StateStore,
|
|
68
|
+
*,
|
|
69
|
+
min_age_s: float = DEFAULT_MIN_AGE_S,
|
|
70
|
+
include_kept: bool = False,
|
|
71
|
+
now: float | None = None,
|
|
72
|
+
) -> list[SandboxVerdict]:
|
|
73
|
+
"""Classify every ``sbxloop-*`` sandbox against the state DB.
|
|
74
|
+
|
|
75
|
+
Non-sbxloop sandboxes are never considered. Names that carry the prefix
|
|
76
|
+
but do not match the ``sbxloop-<run>-<role>`` scheme (future taxonomies:
|
|
77
|
+
warm-pool standby, etc.) are reported but never marked orphaned.
|
|
78
|
+
"""
|
|
79
|
+
now = time.time() if now is None else now
|
|
80
|
+
verdicts: list[SandboxVerdict] = []
|
|
81
|
+
for info in infos:
|
|
82
|
+
if not info.name.startswith("sbxloop-"):
|
|
83
|
+
continue
|
|
84
|
+
verdicts.append(
|
|
85
|
+
_classify_one(info.name, store, min_age_s=min_age_s, include_kept=include_kept, now=now)
|
|
86
|
+
)
|
|
87
|
+
return verdicts
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _classify_one(
|
|
91
|
+
name: str,
|
|
92
|
+
store: StateStore,
|
|
93
|
+
*,
|
|
94
|
+
min_age_s: float,
|
|
95
|
+
include_kept: bool,
|
|
96
|
+
now: float,
|
|
97
|
+
) -> SandboxVerdict:
|
|
98
|
+
match = _NAME_RE.match(name)
|
|
99
|
+
if match is None or not is_run_id(match.group("run")):
|
|
100
|
+
return SandboxVerdict(
|
|
101
|
+
name=name,
|
|
102
|
+
reason="unrecognized sbxloop naming scheme; not touched",
|
|
103
|
+
)
|
|
104
|
+
run_id, role = match.group("run"), match.group("role")
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
run = store.get_run(run_id)
|
|
108
|
+
except StateError:
|
|
109
|
+
return SandboxVerdict(
|
|
110
|
+
name=name,
|
|
111
|
+
run_id=run_id,
|
|
112
|
+
role=role,
|
|
113
|
+
orphan=True,
|
|
114
|
+
reason="unknown to this state DB (may belong to another working copy)",
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
if run.kept_reason is not None and not include_kept:
|
|
118
|
+
return SandboxVerdict(
|
|
119
|
+
name=name,
|
|
120
|
+
run_id=run_id,
|
|
121
|
+
role=role,
|
|
122
|
+
run_state=run.state,
|
|
123
|
+
kept_reason=run.kept_reason,
|
|
124
|
+
age_s=now - run.updated_at,
|
|
125
|
+
reason=f"kept ({run.kept_reason}); use --include-kept to prune",
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
# Liveness = the newest thing the run ever wrote: state transitions bump
|
|
129
|
+
# updated_at, and every bus event (heartbeats included) is persisted.
|
|
130
|
+
last_activity = max(run.updated_at, store.last_event_ts(run_id) or 0.0)
|
|
131
|
+
age_s = now - last_activity
|
|
132
|
+
kept_note = f", kept ({run.kept_reason})" if run.kept_reason is not None else ""
|
|
133
|
+
|
|
134
|
+
if run.state in TERMINAL_RUN_STATES:
|
|
135
|
+
if age_s >= min_age_s:
|
|
136
|
+
return SandboxVerdict(
|
|
137
|
+
name=name,
|
|
138
|
+
run_id=run_id,
|
|
139
|
+
role=role,
|
|
140
|
+
run_state=run.state,
|
|
141
|
+
kept_reason=run.kept_reason,
|
|
142
|
+
age_s=age_s,
|
|
143
|
+
orphan=True,
|
|
144
|
+
reason=f"run {run.state} {format_age(age_s)} ago{kept_note}",
|
|
145
|
+
)
|
|
146
|
+
return SandboxVerdict(
|
|
147
|
+
name=name,
|
|
148
|
+
run_id=run_id,
|
|
149
|
+
role=role,
|
|
150
|
+
run_state=run.state,
|
|
151
|
+
kept_reason=run.kept_reason,
|
|
152
|
+
age_s=age_s,
|
|
153
|
+
reason=f"run {run.state} only {format_age(age_s)} ago (younger than --min-age)",
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
if age_s >= min_age_s:
|
|
157
|
+
return SandboxVerdict(
|
|
158
|
+
name=name,
|
|
159
|
+
run_id=run_id,
|
|
160
|
+
role=role,
|
|
161
|
+
run_state=run.state,
|
|
162
|
+
kept_reason=run.kept_reason,
|
|
163
|
+
age_s=age_s,
|
|
164
|
+
orphan=True,
|
|
165
|
+
reason=f"run {run.state} but silent for {format_age(age_s)}{kept_note}",
|
|
166
|
+
)
|
|
167
|
+
return SandboxVerdict(
|
|
168
|
+
name=name,
|
|
169
|
+
run_id=run_id,
|
|
170
|
+
role=role,
|
|
171
|
+
run_state=run.state,
|
|
172
|
+
kept_reason=run.kept_reason,
|
|
173
|
+
age_s=age_s,
|
|
174
|
+
reason=f"run {run.state}, active {format_age(age_s)} ago (possibly live)",
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def remove_sandbox(cli: SbxCLI, name: str) -> None:
|
|
179
|
+
"""Stop (best-effort) then force-remove one sandbox.
|
|
180
|
+
|
|
181
|
+
``stop`` failing is expected for already-stopped sandboxes; ``rm``
|
|
182
|
+
failing propagates so callers can report it.
|
|
183
|
+
"""
|
|
184
|
+
with contextlib.suppress(SbxError):
|
|
185
|
+
cli.stop(name)
|
|
186
|
+
cli.rm(name, force=True)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def count_orphans(cli: SbxCLI, store: StateStore) -> int:
|
|
190
|
+
"""Orphan-candidate count with default thresholds (doctor's view)."""
|
|
191
|
+
return sum(1 for v in classify_sandboxes(cli.ls(), store) if v.orphan)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|