devzero-sandbox 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devzero_sandbox/__init__.py +41 -0
- devzero_sandbox/_bgscript.py +104 -0
- devzero_sandbox/_proto/__init__.py +1 -0
- devzero_sandbox/_proto/sandbox/__init__.py +1 -0
- devzero_sandbox/_proto/sandbox/v1/__init__.py +1 -0
- devzero_sandbox/_proto/sandbox/v1/error_pb2.py +40 -0
- devzero_sandbox/_proto/sandbox/v1/error_pb2.pyi +23 -0
- devzero_sandbox/_proto/sandbox/v1/service_connect.py +381 -0
- devzero_sandbox/_proto/sandbox/v1/service_pb2.py +227 -0
- devzero_sandbox/_proto/sandbox/v1/service_pb2.pyi +898 -0
- devzero_sandbox/auth.py +363 -0
- devzero_sandbox/commands.py +818 -0
- devzero_sandbox/connect.py +381 -0
- devzero_sandbox/errors.py +453 -0
- devzero_sandbox/files.py +217 -0
- devzero_sandbox/incluster.py +42 -0
- devzero_sandbox/mcp/__init__.py +55 -0
- devzero_sandbox/mcp/__main__.py +7 -0
- devzero_sandbox/mcp/config.py +132 -0
- devzero_sandbox/mcp/server.py +259 -0
- devzero_sandbox/mcp/tools.py +1007 -0
- devzero_sandbox/py.typed +0 -0
- devzero_sandbox/redaction.py +78 -0
- devzero_sandbox/sandbox.py +634 -0
- devzero_sandbox/terminal.py +428 -0
- devzero_sandbox/transport.py +546 -0
- devzero_sandbox-0.1.0.dist-info/METADATA +113 -0
- devzero_sandbox-0.1.0.dist-info/RECORD +31 -0
- devzero_sandbox-0.1.0.dist-info/WHEEL +4 -0
- devzero_sandbox-0.1.0.dist-info/entry_points.txt +2 -0
- devzero_sandbox-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,818 @@
|
|
|
1
|
+
"""Commands in a sandbox: ``sandbox.commands`` (ADR-007-W13).
|
|
2
|
+
|
|
3
|
+
Three ways to run a command, all through the gateway's exec bridge (ADR-007
|
|
4
|
+
§URL scheme, §SSE exec protocol):
|
|
5
|
+
|
|
6
|
+
- **Unary** (the default): ``POST /v1/exec``. One request, one answer with the
|
|
7
|
+
exit code and the whole stdout and stderr (each at most 4 MiB; more is
|
|
8
|
+
``413 payload_too_large``, never truncated). Used while ``timeout_s`` is at
|
|
9
|
+
most 60 s and nothing asks for streaming.
|
|
10
|
+
- **Streaming**: ``POST /v1/exec/stream`` (SSE: ``init``, ``stdout``,
|
|
11
|
+
``stderr``, ``exit``, ``error``, ``drain``). Used when ``stream=True``, when
|
|
12
|
+
``timeout_s`` is above 60 s or unbounded, or when output callbacks are given.
|
|
13
|
+
``sandboxd`` kills the process when its stream closes, so a stream that ends
|
|
14
|
+
before ``exit`` (a gateway rollout past its 30 min drain window, a dropped
|
|
15
|
+
connection) raises :class:`~devzero_sandbox.errors.StreamInterrupted` with
|
|
16
|
+
the pid; the command is gone and is not re-run.
|
|
17
|
+
- **Background**: ``run(cmd, background=True)`` returns a :class:`Process`.
|
|
18
|
+
The command runs outside the stream's process tree with its output in
|
|
19
|
+
``/workspace/.dz/<pid>.log`` (see :mod:`devzero_sandbox._bgscript`), so it
|
|
20
|
+
survives the SDK exiting; ``attach(pid)`` finds it again from any process
|
|
21
|
+
and replays the log tail. A holder stream reports the exit while the SDK
|
|
22
|
+
lives and is re-opened after a drain or a dropped connection.
|
|
23
|
+
|
|
24
|
+
Retries: before a command starts, the gateway's ``503``/``429`` answers that
|
|
25
|
+
mean it was never dialled are retried for 10 s (the transport's window).
|
|
26
|
+
Nothing is retried once a command may have run, so it never runs twice: a
|
|
27
|
+
unary exec the gateway's drain window cut short (``503 exec_interrupted``)
|
|
28
|
+
raises :class:`~devzero_sandbox.errors.StreamInterrupted` like a stream would.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import base64
|
|
34
|
+
import json
|
|
35
|
+
import math
|
|
36
|
+
import socket
|
|
37
|
+
import threading
|
|
38
|
+
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
39
|
+
from contextlib import AbstractContextManager
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from typing import Any, Literal, overload
|
|
42
|
+
|
|
43
|
+
import httpx
|
|
44
|
+
from httpx_sse import EventSource, SSEError
|
|
45
|
+
|
|
46
|
+
from devzero_sandbox import errors
|
|
47
|
+
from devzero_sandbox._bgscript import SCRIPT as _BG_SCRIPT
|
|
48
|
+
from devzero_sandbox.redaction import get_logger
|
|
49
|
+
from devzero_sandbox.transport import GatewayTransport
|
|
50
|
+
|
|
51
|
+
log = get_logger(__name__)
|
|
52
|
+
|
|
53
|
+
#: The sandbox root ``sandboxd`` serves files from (its ``--root-dir``).
|
|
54
|
+
WORKSPACE = "/workspace"
|
|
55
|
+
#: The largest ``timeout_s`` that still uses unary exec.
|
|
56
|
+
UNARY_MAX_TIMEOUT_S = 60.0
|
|
57
|
+
#: How often a background holder prints a line, well inside the gateway's
|
|
58
|
+
#: 30 min SSE idle limit (which keepalive comments do not reset).
|
|
59
|
+
HEARTBEAT_S = 60
|
|
60
|
+
#: The gateway's limit on one stdin write; ``run(stdin=...)`` sends larger input in pieces.
|
|
61
|
+
STDIN_MAX_BYTES = 1 << 20
|
|
62
|
+
#: How much of a background process's log ``attach`` replays by default.
|
|
63
|
+
DEFAULT_TAIL_BYTES = 64 * 1024
|
|
64
|
+
#: How long :meth:`Process.detach` waits for the holder thread to close its stream.
|
|
65
|
+
DETACH_JOIN_S = 5.0
|
|
66
|
+
|
|
67
|
+
# Answers that mean the command was never started, so sending it again cannot
|
|
68
|
+
# run it twice: the routing and admission refusals before the gateway dials.
|
|
69
|
+
# 503 sandbox_unreachable is left out: Execute can fail with it mid-run, and
|
|
70
|
+
# so is 503 exec_interrupted (the drain window ended while the command ran).
|
|
71
|
+
EXEC_RETRY_CODES = frozenset(
|
|
72
|
+
{
|
|
73
|
+
"sandbox_not_ready",
|
|
74
|
+
"sandbox_not_found",
|
|
75
|
+
"index_stale",
|
|
76
|
+
"ambiguous_backend",
|
|
77
|
+
"gateway_at_capacity",
|
|
78
|
+
"auth_backend_unavailable",
|
|
79
|
+
"draining",
|
|
80
|
+
"too_many_streams",
|
|
81
|
+
"principal_budget_exceeded",
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# The unary exec's answer when the gateway's drain window ends while the
|
|
86
|
+
# command runs (ADR-007 §Error-code table): it ran, and was killed.
|
|
87
|
+
EXEC_INTERRUPTED = "exec_interrupted"
|
|
88
|
+
|
|
89
|
+
# An SSE stream sees a keepalive comment every 15 s; four missed is a dead link.
|
|
90
|
+
_STREAM_TIMEOUT = httpx.Timeout(30.0, read=60.0)
|
|
91
|
+
_BG_PREFIX = "dz-bg:"
|
|
92
|
+
|
|
93
|
+
OutputCallback = Callable[[bytes], object]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class CommandResult:
|
|
98
|
+
"""A finished command: exit code and output (signal N is ``128 + N``)."""
|
|
99
|
+
|
|
100
|
+
exit_code: int
|
|
101
|
+
stdout: bytes = b""
|
|
102
|
+
stderr: bytes = b""
|
|
103
|
+
pid: int | None = None
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def ok(self) -> bool:
|
|
107
|
+
return self.exit_code == 0
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def stdout_text(self) -> str:
|
|
111
|
+
return self.stdout.decode("utf-8", "replace")
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def stderr_text(self) -> str:
|
|
115
|
+
return self.stderr.decode("utf-8", "replace")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class BackgroundLaunchFailed(errors.DevZeroSandboxError):
|
|
119
|
+
"""The background launcher could not start the command (``code`` says why)."""
|
|
120
|
+
|
|
121
|
+
def __init__(self, code: str, message: str) -> None:
|
|
122
|
+
self.code = code
|
|
123
|
+
super().__init__(f"background launch failed ({code}): {message}")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _argv(cmd: str | Sequence[str]) -> list[str]:
|
|
127
|
+
if isinstance(cmd, str):
|
|
128
|
+
return ["/bin/sh", "-c", cmd]
|
|
129
|
+
argv = [str(a) for a in cmd]
|
|
130
|
+
if not argv or not argv[0]:
|
|
131
|
+
raise ValueError("cmd must not be empty")
|
|
132
|
+
return argv
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _seconds(timeout_s: float | None) -> int:
|
|
136
|
+
"""``timeout_s`` for the gateway: whole seconds, 0 for no bound."""
|
|
137
|
+
return 0 if not timeout_s else max(1, math.ceil(timeout_s))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@dataclass
|
|
141
|
+
class _Event:
|
|
142
|
+
name: str
|
|
143
|
+
data: dict[str, Any] = field(default_factory=dict)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _events(resp: httpx.Response) -> Iterator[_Event]:
|
|
147
|
+
"""The stream's events; keepalive comments are skipped by the parser."""
|
|
148
|
+
for sse in EventSource(resp).iter_sse():
|
|
149
|
+
try:
|
|
150
|
+
data = json.loads(sse.data) if sse.data else {}
|
|
151
|
+
except ValueError:
|
|
152
|
+
data = {}
|
|
153
|
+
yield _Event(sse.event, data if isinstance(data, dict) else {})
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _b64(data: dict[str, Any]) -> bytes:
|
|
157
|
+
return base64.b64decode(str(data.get("b64", "")))
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class Commands:
|
|
161
|
+
"""Run, attach to and kill commands in one sandbox; see the module docstring."""
|
|
162
|
+
|
|
163
|
+
def __init__(
|
|
164
|
+
self,
|
|
165
|
+
transport: GatewayTransport,
|
|
166
|
+
*,
|
|
167
|
+
workspace: str = WORKSPACE,
|
|
168
|
+
heartbeat_s: int = HEARTBEAT_S,
|
|
169
|
+
sleep: Callable[[float], None] | None = None,
|
|
170
|
+
) -> None:
|
|
171
|
+
self._t = transport
|
|
172
|
+
self.workspace = workspace.rstrip("/") or "/"
|
|
173
|
+
self.heartbeat_s = heartbeat_s
|
|
174
|
+
# None: a holder's backoff waits on its stop flag, so detach() cuts it short.
|
|
175
|
+
self._sleep = sleep
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def background_dir(self) -> str:
|
|
179
|
+
"""Where background processes keep ``<pid>.log``, ``.pid`` and ``.exit``."""
|
|
180
|
+
return f"{self.workspace}/.dz"
|
|
181
|
+
|
|
182
|
+
# --- run -----------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
@overload
|
|
185
|
+
def run(
|
|
186
|
+
self,
|
|
187
|
+
cmd: str | Sequence[str],
|
|
188
|
+
*,
|
|
189
|
+
background: Literal[True],
|
|
190
|
+
timeout_s: float | None = ...,
|
|
191
|
+
stream: bool = ...,
|
|
192
|
+
cwd: str | None = ...,
|
|
193
|
+
env: Mapping[str, str] | None = ...,
|
|
194
|
+
stdin: bytes | None = ...,
|
|
195
|
+
on_stdout: OutputCallback | None = ...,
|
|
196
|
+
on_stderr: OutputCallback | None = ...,
|
|
197
|
+
) -> Process: ...
|
|
198
|
+
|
|
199
|
+
@overload
|
|
200
|
+
def run(
|
|
201
|
+
self,
|
|
202
|
+
cmd: str | Sequence[str],
|
|
203
|
+
*,
|
|
204
|
+
background: Literal[False] = ...,
|
|
205
|
+
timeout_s: float | None = ...,
|
|
206
|
+
stream: bool = ...,
|
|
207
|
+
cwd: str | None = ...,
|
|
208
|
+
env: Mapping[str, str] | None = ...,
|
|
209
|
+
stdin: bytes | None = ...,
|
|
210
|
+
on_stdout: OutputCallback | None = ...,
|
|
211
|
+
on_stderr: OutputCallback | None = ...,
|
|
212
|
+
) -> CommandResult: ...
|
|
213
|
+
|
|
214
|
+
def run(
|
|
215
|
+
self,
|
|
216
|
+
cmd: str | Sequence[str],
|
|
217
|
+
*,
|
|
218
|
+
background: bool = False,
|
|
219
|
+
timeout_s: float | None = 60,
|
|
220
|
+
stream: bool = False,
|
|
221
|
+
cwd: str | None = None,
|
|
222
|
+
env: Mapping[str, str] | None = None,
|
|
223
|
+
stdin: bytes | None = None,
|
|
224
|
+
on_stdout: OutputCallback | None = None,
|
|
225
|
+
on_stderr: OutputCallback | None = None,
|
|
226
|
+
) -> CommandResult | Process:
|
|
227
|
+
"""Run ``cmd`` (a shell string, or an argv list) and return its result.
|
|
228
|
+
|
|
229
|
+
``timeout_s`` bounds the command (``None`` or ``0``: no bound); the
|
|
230
|
+
gateway kills it when the time is up (``UpstreamTimeout``). Unary
|
|
231
|
+
exec is used unless ``stream``, ``timeout_s`` above 60 s or unbounded,
|
|
232
|
+
``stdin`` or an output callback asks for the stream. A non-zero exit
|
|
233
|
+
is a result, not an exception.
|
|
234
|
+
|
|
235
|
+
``background=True`` starts the command detached and returns a
|
|
236
|
+
:class:`Process` at once; ``timeout_s`` does not apply to it.
|
|
237
|
+
"""
|
|
238
|
+
argv = _argv(cmd)
|
|
239
|
+
if background:
|
|
240
|
+
return self._launch(argv, cwd=cwd, env=env)
|
|
241
|
+
if (
|
|
242
|
+
stream
|
|
243
|
+
or not timeout_s
|
|
244
|
+
or timeout_s > UNARY_MAX_TIMEOUT_S
|
|
245
|
+
or stdin is not None
|
|
246
|
+
or on_stdout is not None
|
|
247
|
+
or on_stderr is not None
|
|
248
|
+
):
|
|
249
|
+
return self._stream(
|
|
250
|
+
argv,
|
|
251
|
+
timeout_s=timeout_s,
|
|
252
|
+
cwd=cwd,
|
|
253
|
+
env=env,
|
|
254
|
+
stdin=stdin,
|
|
255
|
+
on_stdout=on_stdout,
|
|
256
|
+
on_stderr=on_stderr,
|
|
257
|
+
)
|
|
258
|
+
return self._unary(argv, timeout_s=timeout_s, cwd=cwd, env=env)
|
|
259
|
+
|
|
260
|
+
def _body(
|
|
261
|
+
self,
|
|
262
|
+
argv: list[str],
|
|
263
|
+
timeout_s: float | None,
|
|
264
|
+
cwd: str | None,
|
|
265
|
+
env: Mapping[str, str] | None,
|
|
266
|
+
) -> dict[str, Any]:
|
|
267
|
+
body: dict[str, Any] = {"cmd": argv, "timeout_s": _seconds(timeout_s)}
|
|
268
|
+
if cwd is not None:
|
|
269
|
+
body["cwd"] = cwd
|
|
270
|
+
if env:
|
|
271
|
+
body["env"] = dict(env)
|
|
272
|
+
return body
|
|
273
|
+
|
|
274
|
+
def _unary(
|
|
275
|
+
self,
|
|
276
|
+
argv: list[str],
|
|
277
|
+
*,
|
|
278
|
+
timeout_s: float,
|
|
279
|
+
cwd: str | None,
|
|
280
|
+
env: Mapping[str, str] | None,
|
|
281
|
+
) -> CommandResult:
|
|
282
|
+
secs = _seconds(timeout_s)
|
|
283
|
+
try:
|
|
284
|
+
resp = self._t.request(
|
|
285
|
+
"POST",
|
|
286
|
+
"/v1/exec",
|
|
287
|
+
scope="exec",
|
|
288
|
+
json=self._body(argv, timeout_s, cwd, env),
|
|
289
|
+
headers={"X-Timeout": str(secs)},
|
|
290
|
+
# The gateway answers 504 at the deadline; leave room for that answer.
|
|
291
|
+
timeout=httpx.Timeout(30.0, read=secs + 30.0),
|
|
292
|
+
retry_codes=EXEC_RETRY_CODES,
|
|
293
|
+
)
|
|
294
|
+
except errors.GatewayError as e:
|
|
295
|
+
if e.code == EXEC_INTERRUPTED:
|
|
296
|
+
# The command ran and the gateway's drain killed it (unary
|
|
297
|
+
# exec reports no pid); never re-run it.
|
|
298
|
+
raise errors.StreamInterrupted(
|
|
299
|
+
pid=None, code="drain", message=e.message
|
|
300
|
+
) from None
|
|
301
|
+
raise
|
|
302
|
+
out = resp.json()
|
|
303
|
+
return CommandResult(
|
|
304
|
+
exit_code=int(out["exit_code"]),
|
|
305
|
+
stdout=base64.b64decode(out.get("stdout_b64") or ""),
|
|
306
|
+
stderr=base64.b64decode(out.get("stderr_b64") or ""),
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
def _open(
|
|
310
|
+
self,
|
|
311
|
+
argv: list[str],
|
|
312
|
+
timeout_s: float | None,
|
|
313
|
+
cwd: str | None,
|
|
314
|
+
env: Mapping[str, str] | None,
|
|
315
|
+
) -> AbstractContextManager[httpx.Response]:
|
|
316
|
+
return self._t.stream(
|
|
317
|
+
"POST",
|
|
318
|
+
"/v1/exec/stream",
|
|
319
|
+
scope="exec",
|
|
320
|
+
json=self._body(argv, timeout_s, cwd, env),
|
|
321
|
+
headers={"Accept": "text/event-stream"},
|
|
322
|
+
timeout=_STREAM_TIMEOUT,
|
|
323
|
+
retry_codes=EXEC_RETRY_CODES,
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
def _stream(
|
|
327
|
+
self,
|
|
328
|
+
argv: list[str],
|
|
329
|
+
*,
|
|
330
|
+
timeout_s: float | None,
|
|
331
|
+
cwd: str | None,
|
|
332
|
+
env: Mapping[str, str] | None,
|
|
333
|
+
stdin: bytes | None,
|
|
334
|
+
on_stdout: OutputCallback | None,
|
|
335
|
+
on_stderr: OutputCallback | None,
|
|
336
|
+
) -> CommandResult:
|
|
337
|
+
pid: int | None = None
|
|
338
|
+
drained = False
|
|
339
|
+
out: list[bytes] = []
|
|
340
|
+
err: list[bytes] = []
|
|
341
|
+
try:
|
|
342
|
+
with self._open(argv, timeout_s, cwd, env) as resp:
|
|
343
|
+
for ev in _events(resp):
|
|
344
|
+
if ev.name == "init":
|
|
345
|
+
pid = int(ev.data["pid"])
|
|
346
|
+
if stdin is not None:
|
|
347
|
+
self._write_all_stdin(pid, stdin)
|
|
348
|
+
elif ev.name in ("stdout", "stderr"):
|
|
349
|
+
chunk = _b64(ev.data)
|
|
350
|
+
(out if ev.name == "stdout" else err).append(chunk)
|
|
351
|
+
cb = on_stdout if ev.name == "stdout" else on_stderr
|
|
352
|
+
if cb is not None:
|
|
353
|
+
cb(chunk)
|
|
354
|
+
elif ev.name == "exit":
|
|
355
|
+
return CommandResult(
|
|
356
|
+
int(ev.data.get("code", -1)),
|
|
357
|
+
b"".join(out),
|
|
358
|
+
b"".join(err),
|
|
359
|
+
pid,
|
|
360
|
+
)
|
|
361
|
+
elif ev.name == "drain":
|
|
362
|
+
# A gateway rollout: the stream keeps running for the
|
|
363
|
+
# drain window (30 min), then the gateway closes it.
|
|
364
|
+
drained = True
|
|
365
|
+
log.debug("exec stream for pid %s: gateway draining", pid)
|
|
366
|
+
elif ev.name == "error":
|
|
367
|
+
raise _stream_error(pid, ev.data, drained)
|
|
368
|
+
except (httpx.TransportError, SSEError) as e:
|
|
369
|
+
raise errors.StreamInterrupted(
|
|
370
|
+
pid=pid,
|
|
371
|
+
code="drain" if drained else "disconnected",
|
|
372
|
+
message=f"the exec stream broke ({type(e).__name__}); the process was killed",
|
|
373
|
+
) from None
|
|
374
|
+
raise errors.StreamInterrupted(
|
|
375
|
+
pid=pid,
|
|
376
|
+
code="drain" if drained else "disconnected",
|
|
377
|
+
message="the exec stream ended before the process exited; the process was killed",
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
# --- control calls -------------------------------------------------------------
|
|
381
|
+
|
|
382
|
+
def _write_all_stdin(self, pid: int, data: bytes) -> None:
|
|
383
|
+
# In order, each piece within the gateway's per-call limit; eof closes
|
|
384
|
+
# stdin with the last piece (an empty input is one empty eof write).
|
|
385
|
+
last = max(len(data) - 1, 0) // STDIN_MAX_BYTES * STDIN_MAX_BYTES
|
|
386
|
+
for i in range(0, last + 1, STDIN_MAX_BYTES):
|
|
387
|
+
self.write_stdin(pid, data[i : i + STDIN_MAX_BYTES], eof=i == last)
|
|
388
|
+
|
|
389
|
+
def write_stdin(self, pid: int, data: bytes, *, eof: bool = False) -> None:
|
|
390
|
+
"""Write to a streamed process's stdin (at most 1 MiB per call); ``eof`` closes it."""
|
|
391
|
+
self._t.request(
|
|
392
|
+
"POST",
|
|
393
|
+
f"/v1/exec/{int(pid)}/stdin",
|
|
394
|
+
scope="exec",
|
|
395
|
+
content=data,
|
|
396
|
+
params={"eof": "true"} if eof else None,
|
|
397
|
+
headers={"Content-Type": "application/octet-stream"},
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
def signal(self, pid: int, signal: Literal["INT", "TERM", "KILL"]) -> None:
|
|
401
|
+
"""Send ``INT``, ``TERM`` or ``KILL`` to a streamed process's group."""
|
|
402
|
+
self._t.request(
|
|
403
|
+
"POST", f"/v1/exec/{int(pid)}/signal", scope="exec", json={"signal": signal}
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
def kill(self, pid: int) -> None:
|
|
407
|
+
"""Kill a process: a background one (its whole tree), or a streamed one.
|
|
408
|
+
|
|
409
|
+
Raises :class:`~devzero_sandbox.errors.ProcessNotFound` when neither
|
|
410
|
+
exists (or the process has already exited).
|
|
411
|
+
"""
|
|
412
|
+
res = self._unary(self._bg_argv("kill", pid), timeout_s=30, cwd=None, env=None)
|
|
413
|
+
if res.exit_code == 0:
|
|
414
|
+
return
|
|
415
|
+
if res.exit_code not in (3, 4):
|
|
416
|
+
raise BackgroundLaunchFailed(
|
|
417
|
+
"kill_failed", res.stderr_text or res.stdout_text
|
|
418
|
+
)
|
|
419
|
+
# Not a running background process: the pid of a stream (DELETE = SIGKILL).
|
|
420
|
+
self._t.request("DELETE", f"/v1/exec/{int(pid)}", scope="exec")
|
|
421
|
+
|
|
422
|
+
# --- background ----------------------------------------------------------------
|
|
423
|
+
|
|
424
|
+
def _bg_argv(
|
|
425
|
+
self, mode: str, pid: int | None = None, argv: Sequence[str] = ()
|
|
426
|
+
) -> list[str]:
|
|
427
|
+
out = [
|
|
428
|
+
"/bin/sh",
|
|
429
|
+
"-c",
|
|
430
|
+
_BG_SCRIPT,
|
|
431
|
+
"dz-bg",
|
|
432
|
+
mode,
|
|
433
|
+
self.background_dir,
|
|
434
|
+
str(self.heartbeat_s),
|
|
435
|
+
]
|
|
436
|
+
if pid is not None:
|
|
437
|
+
out.append(str(int(pid)))
|
|
438
|
+
return out + list(argv)
|
|
439
|
+
|
|
440
|
+
def _launch(
|
|
441
|
+
self, argv: list[str], *, cwd: str | None, env: Mapping[str, str] | None
|
|
442
|
+
) -> Process:
|
|
443
|
+
cm = self._open(self._bg_argv("launch", argv=argv), None, cwd, env)
|
|
444
|
+
resp = cm.__enter__()
|
|
445
|
+
try:
|
|
446
|
+
events = _events(resp)
|
|
447
|
+
pid: int | None = None
|
|
448
|
+
lines = _Lines()
|
|
449
|
+
for ev in events:
|
|
450
|
+
if ev.name == "init":
|
|
451
|
+
pid = int(ev.data["pid"])
|
|
452
|
+
# The launcher names its files after the pid sandboxd gave it.
|
|
453
|
+
self.write_stdin(pid, f"{pid}\n".encode(), eof=True)
|
|
454
|
+
elif ev.name == "stdout":
|
|
455
|
+
for line in lines.feed(_b64(ev.data)):
|
|
456
|
+
if line == _BG_PREFIX + "started" and pid is not None:
|
|
457
|
+
proc = Process(self, pid)
|
|
458
|
+
proc._hold(cm, resp, events)
|
|
459
|
+
return proc
|
|
460
|
+
_raise_bg_error(line)
|
|
461
|
+
elif ev.name == "exit":
|
|
462
|
+
raise BackgroundLaunchFailed(
|
|
463
|
+
"exited", f"launcher exited {ev.data.get('code')}"
|
|
464
|
+
)
|
|
465
|
+
elif ev.name == "error":
|
|
466
|
+
raise _stream_error(pid, ev.data, False)
|
|
467
|
+
raise errors.StreamInterrupted(
|
|
468
|
+
pid=pid,
|
|
469
|
+
code="disconnected",
|
|
470
|
+
message="the launch stream ended before the process started",
|
|
471
|
+
)
|
|
472
|
+
except BaseException:
|
|
473
|
+
cm.__exit__(None, None, None)
|
|
474
|
+
raise
|
|
475
|
+
|
|
476
|
+
def attach(
|
|
477
|
+
self, pid: int, *, tail_bytes: int = DEFAULT_TAIL_BYTES, hold: bool = True
|
|
478
|
+
) -> Process:
|
|
479
|
+
"""Find a background process again (from any SDK process) and replay its log tail.
|
|
480
|
+
|
|
481
|
+
The returned :class:`Process` has the last ``tail_bytes`` of the log in
|
|
482
|
+
``tail`` and, while the process runs and ``hold`` is set, a holder
|
|
483
|
+
stream that reports its exit.
|
|
484
|
+
"""
|
|
485
|
+
from devzero_sandbox.files import Files
|
|
486
|
+
|
|
487
|
+
files = Files(self._t, workspace=self.workspace)
|
|
488
|
+
rel = files.relative(f"{self.background_dir}/{int(pid)}")
|
|
489
|
+
try:
|
|
490
|
+
files.read(rel + ".pid")
|
|
491
|
+
except errors.SandboxFileNotFound:
|
|
492
|
+
raise errors.ProcessNotFound(
|
|
493
|
+
status=404,
|
|
494
|
+
code="process_not_found",
|
|
495
|
+
message=f"no background process {pid} in {self.background_dir}",
|
|
496
|
+
pid=int(pid),
|
|
497
|
+
) from None
|
|
498
|
+
proc = Process(self, int(pid))
|
|
499
|
+
proc.tail = proc.logs(tail_bytes)
|
|
500
|
+
code = proc._read_exit()
|
|
501
|
+
if code is not None:
|
|
502
|
+
proc._finish(code, None)
|
|
503
|
+
elif hold:
|
|
504
|
+
proc._reopen()
|
|
505
|
+
return proc
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
class _Lines:
|
|
509
|
+
"""Splits a byte stream into ``dz-bg:`` protocol lines (heartbeats are empty lines)."""
|
|
510
|
+
|
|
511
|
+
def __init__(self) -> None:
|
|
512
|
+
self._buf = b""
|
|
513
|
+
|
|
514
|
+
def feed(self, data: bytes) -> list[str]:
|
|
515
|
+
self._buf += data
|
|
516
|
+
*done, self._buf = self._buf.split(b"\n")
|
|
517
|
+
return [
|
|
518
|
+
d.decode("utf-8", "replace")
|
|
519
|
+
for d in done
|
|
520
|
+
if d.startswith(_BG_PREFIX.encode())
|
|
521
|
+
]
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _raise_bg_error(line: str) -> None:
|
|
525
|
+
if line.startswith(_BG_PREFIX + "error:"):
|
|
526
|
+
code, _, msg = line[len(_BG_PREFIX + "error:") :].partition(":")
|
|
527
|
+
if code == "not_found":
|
|
528
|
+
raise errors.ProcessNotFound(
|
|
529
|
+
status=404, code="process_not_found", message=msg
|
|
530
|
+
)
|
|
531
|
+
raise BackgroundLaunchFailed(code, msg)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _stream_error(
|
|
535
|
+
pid: int | None, data: dict[str, Any], drained: bool
|
|
536
|
+
) -> errors.DevZeroSandboxError:
|
|
537
|
+
code = str(data.get("code") or "")
|
|
538
|
+
message = str(data.get("message") or "")
|
|
539
|
+
if code == "upstream_timeout":
|
|
540
|
+
return errors.UpstreamTimeout(status=504, code=code, message=message, pid=pid)
|
|
541
|
+
return errors.StreamInterrupted(
|
|
542
|
+
pid=pid, code=code or ("drain" if drained else "error"), message=message
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
class Process:
|
|
547
|
+
"""A background process (``run(..., background=True)`` or ``attach(pid)``).
|
|
548
|
+
|
|
549
|
+
The process runs on its own in the sandbox: :meth:`detach`, the SDK exiting
|
|
550
|
+
or the holder stream breaking never stops it; :meth:`kill` does. ``tail``
|
|
551
|
+
holds the log tail ``attach`` replayed.
|
|
552
|
+
"""
|
|
553
|
+
|
|
554
|
+
def __init__(self, commands: Commands, pid: int) -> None:
|
|
555
|
+
self._c = commands
|
|
556
|
+
self.pid = pid
|
|
557
|
+
self.tail = b""
|
|
558
|
+
self._done = threading.Event()
|
|
559
|
+
self._closing = threading.Event()
|
|
560
|
+
self._lock = threading.Lock()
|
|
561
|
+
# The holder's stream. Only the holder thread reads or closes it while
|
|
562
|
+
# that thread runs; other threads only shut its socket down (_wake_holder).
|
|
563
|
+
self._cm: AbstractContextManager[httpx.Response] | None = None
|
|
564
|
+
self._resp: httpx.Response | None = None
|
|
565
|
+
self._thread: threading.Thread | None = None
|
|
566
|
+
self._exit_code: int | None = None
|
|
567
|
+
self._error: BaseException | None = None
|
|
568
|
+
|
|
569
|
+
@property
|
|
570
|
+
def log_path(self) -> str:
|
|
571
|
+
"""The log in the sandbox (stdout and stderr, interleaved)."""
|
|
572
|
+
return f"{self._c.background_dir}/{self.pid}.log"
|
|
573
|
+
|
|
574
|
+
@property
|
|
575
|
+
def exit_code(self) -> int | None:
|
|
576
|
+
return self._exit_code
|
|
577
|
+
|
|
578
|
+
def is_running(self) -> bool:
|
|
579
|
+
if self._done.is_set():
|
|
580
|
+
return False
|
|
581
|
+
code = self._read_exit()
|
|
582
|
+
if code is not None:
|
|
583
|
+
self._finish(code, None)
|
|
584
|
+
return False
|
|
585
|
+
return True
|
|
586
|
+
|
|
587
|
+
def logs(self, tail_bytes: int | None = None) -> bytes:
|
|
588
|
+
"""The process's log so far, or its last ``tail_bytes``."""
|
|
589
|
+
from devzero_sandbox.files import Files
|
|
590
|
+
|
|
591
|
+
files = Files(self._c._t, workspace=self._c.workspace)
|
|
592
|
+
return files.read(files.relative(self.log_path), tail_bytes=tail_bytes)
|
|
593
|
+
|
|
594
|
+
def wait(self, timeout_s: float | None = None) -> int:
|
|
595
|
+
"""Block until the process exits and return its exit code (137 when killed)."""
|
|
596
|
+
if not self._done.is_set() and self._thread is None:
|
|
597
|
+
self._reopen()
|
|
598
|
+
if not self._done.wait(timeout_s):
|
|
599
|
+
raise TimeoutError(
|
|
600
|
+
f"background process {self.pid} still running after {timeout_s}s"
|
|
601
|
+
)
|
|
602
|
+
if self._error is not None:
|
|
603
|
+
raise self._error
|
|
604
|
+
assert self._exit_code is not None
|
|
605
|
+
return self._exit_code
|
|
606
|
+
|
|
607
|
+
def kill(self) -> None:
|
|
608
|
+
"""SIGKILL the process and everything it started.
|
|
609
|
+
|
|
610
|
+
The holder (if any) then reports exit code 137 to :meth:`wait`. Killing
|
|
611
|
+
a process that has already exited is a no-op.
|
|
612
|
+
"""
|
|
613
|
+
if self._done.is_set() and self._error is None:
|
|
614
|
+
return
|
|
615
|
+
try:
|
|
616
|
+
self._c.kill(self.pid)
|
|
617
|
+
except errors.ProcessNotFound:
|
|
618
|
+
if self._read_exit() is None:
|
|
619
|
+
raise
|
|
620
|
+
|
|
621
|
+
def detach(self) -> None:
|
|
622
|
+
"""Stop holding the process (close the holder stream); it keeps running.
|
|
623
|
+
|
|
624
|
+
Returns once the holder thread has closed its stream (at most
|
|
625
|
+
``DETACH_JOIN_S``). The stream is closed by the thread that reads it:
|
|
626
|
+
this shuts its socket down to wake that thread, never closes it under
|
|
627
|
+
it. Calling it again is a no-op.
|
|
628
|
+
"""
|
|
629
|
+
self._closing.set()
|
|
630
|
+
thread = self._thread
|
|
631
|
+
if (
|
|
632
|
+
thread is None
|
|
633
|
+
or thread is threading.current_thread()
|
|
634
|
+
or not thread.is_alive()
|
|
635
|
+
):
|
|
636
|
+
self._close_cm() # no holder is reading it
|
|
637
|
+
return
|
|
638
|
+
self._wake_holder()
|
|
639
|
+
thread.join(DETACH_JOIN_S)
|
|
640
|
+
if thread.is_alive():
|
|
641
|
+
# Still opening a stream: it sees _closing and closes it itself.
|
|
642
|
+
log.debug("holder of background pid %d still stopping", self.pid)
|
|
643
|
+
|
|
644
|
+
close = detach
|
|
645
|
+
|
|
646
|
+
def __repr__(self) -> str:
|
|
647
|
+
state = "running" if not self._done.is_set() else f"exited {self._exit_code}"
|
|
648
|
+
return f"Process(pid={self.pid}, {state})"
|
|
649
|
+
|
|
650
|
+
# --- the holder ----------------------------------------------------------------
|
|
651
|
+
|
|
652
|
+
def _read_exit(self) -> int | None:
|
|
653
|
+
from devzero_sandbox.files import Files
|
|
654
|
+
|
|
655
|
+
files = Files(self._c._t, workspace=self._c.workspace)
|
|
656
|
+
try:
|
|
657
|
+
raw = files.read(
|
|
658
|
+
files.relative(f"{self._c.background_dir}/{self.pid}.exit")
|
|
659
|
+
)
|
|
660
|
+
except errors.SandboxFileNotFound:
|
|
661
|
+
return None
|
|
662
|
+
try:
|
|
663
|
+
return int(raw.strip())
|
|
664
|
+
except ValueError:
|
|
665
|
+
return None
|
|
666
|
+
|
|
667
|
+
def _finish(self, code: int | None, error: BaseException | None) -> None:
|
|
668
|
+
self._exit_code, self._error = code, error
|
|
669
|
+
self._done.set()
|
|
670
|
+
|
|
671
|
+
def _hold(
|
|
672
|
+
self,
|
|
673
|
+
cm: AbstractContextManager[httpx.Response],
|
|
674
|
+
resp: httpx.Response,
|
|
675
|
+
events: Iterator[_Event],
|
|
676
|
+
) -> None:
|
|
677
|
+
with self._lock:
|
|
678
|
+
self._cm, self._resp = cm, resp
|
|
679
|
+
self._thread = threading.Thread(
|
|
680
|
+
target=self._run_holder,
|
|
681
|
+
args=(events,),
|
|
682
|
+
name=f"dz-bg-holder-{self.pid}",
|
|
683
|
+
daemon=True,
|
|
684
|
+
)
|
|
685
|
+
self._thread.start()
|
|
686
|
+
|
|
687
|
+
def _reopen(self) -> None:
|
|
688
|
+
"""Open a holder (watch mode) for a process launched elsewhere."""
|
|
689
|
+
self._thread = threading.Thread(
|
|
690
|
+
target=self._run_holder,
|
|
691
|
+
args=(None,),
|
|
692
|
+
name=f"dz-bg-holder-{self.pid}",
|
|
693
|
+
daemon=True,
|
|
694
|
+
)
|
|
695
|
+
self._thread.start()
|
|
696
|
+
|
|
697
|
+
def _run_holder(self, events: Iterator[_Event] | None) -> None:
|
|
698
|
+
failures = 0
|
|
699
|
+
while not self._closing.is_set():
|
|
700
|
+
if events is None:
|
|
701
|
+
try:
|
|
702
|
+
cm = self._c._open(
|
|
703
|
+
self._c._bg_argv("watch", self.pid), None, None, None
|
|
704
|
+
)
|
|
705
|
+
resp = cm.__enter__()
|
|
706
|
+
except errors.GatewayError as e:
|
|
707
|
+
if not e.retryable or failures >= 5:
|
|
708
|
+
self._finish(None, e)
|
|
709
|
+
return
|
|
710
|
+
failures += 1
|
|
711
|
+
self._pause(min(0.5 * 2**failures, 8.0))
|
|
712
|
+
continue
|
|
713
|
+
except errors.DevZeroSandboxError as e:
|
|
714
|
+
self._finish(None, e)
|
|
715
|
+
return
|
|
716
|
+
with self._lock:
|
|
717
|
+
if self._closing.is_set():
|
|
718
|
+
cm.__exit__(None, None, None)
|
|
719
|
+
return
|
|
720
|
+
self._cm, self._resp = cm, resp
|
|
721
|
+
events = _events(resp)
|
|
722
|
+
try:
|
|
723
|
+
code = self._consume(events)
|
|
724
|
+
except errors.ProcessNotFound as e:
|
|
725
|
+
self._finish(None, e)
|
|
726
|
+
return
|
|
727
|
+
except (
|
|
728
|
+
httpx.TransportError,
|
|
729
|
+
SSEError,
|
|
730
|
+
errors.StreamInterrupted,
|
|
731
|
+
errors.UpstreamTimeout,
|
|
732
|
+
):
|
|
733
|
+
code = None
|
|
734
|
+
except BaseException as e: # noqa: BLE001 -- a holder never dies silently: wait() raises it
|
|
735
|
+
if self._closing.is_set():
|
|
736
|
+
return
|
|
737
|
+
self._finish(None, e)
|
|
738
|
+
return
|
|
739
|
+
finally:
|
|
740
|
+
self._close_cm()
|
|
741
|
+
events = None
|
|
742
|
+
if code is not None:
|
|
743
|
+
self._finish(code, None)
|
|
744
|
+
return
|
|
745
|
+
if self._closing.is_set():
|
|
746
|
+
return
|
|
747
|
+
# Drain, the 24 h stream lifetime or a dropped link: the process is
|
|
748
|
+
# unaffected; hold it again from wherever the gateway routes us.
|
|
749
|
+
log.debug("holder for background pid %d ended; re-opening", self.pid)
|
|
750
|
+
failures += 1
|
|
751
|
+
self._pause(min(0.5 * 2 ** min(failures, 4), 5.0))
|
|
752
|
+
if failures > 20:
|
|
753
|
+
self._finish(
|
|
754
|
+
None,
|
|
755
|
+
errors.StreamInterrupted(
|
|
756
|
+
pid=self.pid,
|
|
757
|
+
code="disconnected",
|
|
758
|
+
message="could not hold the background process",
|
|
759
|
+
),
|
|
760
|
+
)
|
|
761
|
+
return
|
|
762
|
+
|
|
763
|
+
def _consume(self, events: Iterator[_Event]) -> int | None:
|
|
764
|
+
lines = _Lines()
|
|
765
|
+
for ev in events:
|
|
766
|
+
if ev.name == "stdout":
|
|
767
|
+
for line in lines.feed(_b64(ev.data)):
|
|
768
|
+
if line.startswith(_BG_PREFIX + "exit:"):
|
|
769
|
+
return int(line[len(_BG_PREFIX + "exit:") :])
|
|
770
|
+
_raise_bg_error(line)
|
|
771
|
+
elif ev.name == "error":
|
|
772
|
+
raise _stream_error(self.pid, ev.data, False)
|
|
773
|
+
return None
|
|
774
|
+
|
|
775
|
+
def _pause(self, seconds: float) -> None:
|
|
776
|
+
if self._c._sleep is not None:
|
|
777
|
+
self._c._sleep(seconds)
|
|
778
|
+
else:
|
|
779
|
+
self._closing.wait(seconds)
|
|
780
|
+
|
|
781
|
+
def _wake_holder(self) -> None:
|
|
782
|
+
"""Wake a holder blocked reading its stream, from another thread.
|
|
783
|
+
|
|
784
|
+
Only the kernel socket is shut down: the reader's next read returns end
|
|
785
|
+
of stream and the holder closes the stream itself. Closing the stream
|
|
786
|
+
here instead would close the TLS object and the file descriptor under
|
|
787
|
+
the reader (a read on a closed or reused fd, or an error from inside
|
|
788
|
+
OpenSSL). ``socket.socket.shutdown``, not ``SSLSocket.shutdown``, which
|
|
789
|
+
also drops the SSL object the reader is using.
|
|
790
|
+
"""
|
|
791
|
+
with self._lock:
|
|
792
|
+
sock = _socket_of(self._resp)
|
|
793
|
+
if sock is None:
|
|
794
|
+
return
|
|
795
|
+
try:
|
|
796
|
+
socket.socket.shutdown(sock, socket.SHUT_RDWR)
|
|
797
|
+
except OSError:
|
|
798
|
+
pass # already closed or never connected: nothing to wake
|
|
799
|
+
|
|
800
|
+
def _close_cm(self) -> None:
|
|
801
|
+
"""Close the holder stream: from the holder thread, or when none runs."""
|
|
802
|
+
with self._lock:
|
|
803
|
+
cm, self._cm, self._resp = self._cm, None, None
|
|
804
|
+
if cm is not None:
|
|
805
|
+
try:
|
|
806
|
+
cm.__exit__(None, None, None)
|
|
807
|
+
except Exception as e: # noqa: BLE001 -- a stream we woke may end uncleanly
|
|
808
|
+
log.debug("closing the holder stream of pid %d: %r", self.pid, e)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _socket_of(resp: httpx.Response | None) -> socket.socket | None:
|
|
812
|
+
"""The connection's socket under a streamed response (httpcore's sync backend)."""
|
|
813
|
+
if resp is None:
|
|
814
|
+
return None
|
|
815
|
+
stream = resp.extensions.get("network_stream")
|
|
816
|
+
get = getattr(stream, "get_extra_info", None)
|
|
817
|
+
sock = get("socket") if get is not None else None
|
|
818
|
+
return sock if isinstance(sock, socket.socket) else None
|