devzero-sandbox 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,818 @@
1
+ """Commands in a sandbox: ``sandbox.commands`` (ADR-007-W13).
2
+
3
+ Three ways to run a command, all through the gateway's exec bridge (ADR-007
4
+ §URL scheme, §SSE exec protocol):
5
+
6
+ - **Unary** (the default): ``POST /v1/exec``. One request, one answer with the
7
+ exit code and the whole stdout and stderr (each at most 4 MiB; more is
8
+ ``413 payload_too_large``, never truncated). Used while ``timeout_s`` is at
9
+ most 60 s and nothing asks for streaming.
10
+ - **Streaming**: ``POST /v1/exec/stream`` (SSE: ``init``, ``stdout``,
11
+ ``stderr``, ``exit``, ``error``, ``drain``). Used when ``stream=True``, when
12
+ ``timeout_s`` is above 60 s or unbounded, or when output callbacks are given.
13
+ ``sandboxd`` kills the process when its stream closes, so a stream that ends
14
+ before ``exit`` (a gateway rollout past its 30 min drain window, a dropped
15
+ connection) raises :class:`~devzero_sandbox.errors.StreamInterrupted` with
16
+ the pid; the command is gone and is not re-run.
17
+ - **Background**: ``run(cmd, background=True)`` returns a :class:`Process`.
18
+ The command runs outside the stream's process tree with its output in
19
+ ``/workspace/.dz/<pid>.log`` (see :mod:`devzero_sandbox._bgscript`), so it
20
+ survives the SDK exiting; ``attach(pid)`` finds it again from any process
21
+ and replays the log tail. A holder stream reports the exit while the SDK
22
+ lives and is re-opened after a drain or a dropped connection.
23
+
24
+ Retries: before a command starts, the gateway's ``503``/``429`` answers that
25
+ mean it was never dialled are retried for 10 s (the transport's window).
26
+ Nothing is retried once a command may have run, so it never runs twice: a
27
+ unary exec the gateway's drain window cut short (``503 exec_interrupted``)
28
+ raises :class:`~devzero_sandbox.errors.StreamInterrupted` like a stream would.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import base64
34
+ import json
35
+ import math
36
+ import socket
37
+ import threading
38
+ from collections.abc import Callable, Iterator, Mapping, Sequence
39
+ from contextlib import AbstractContextManager
40
+ from dataclasses import dataclass, field
41
+ from typing import Any, Literal, overload
42
+
43
+ import httpx
44
+ from httpx_sse import EventSource, SSEError
45
+
46
+ from devzero_sandbox import errors
47
+ from devzero_sandbox._bgscript import SCRIPT as _BG_SCRIPT
48
+ from devzero_sandbox.redaction import get_logger
49
+ from devzero_sandbox.transport import GatewayTransport
50
+
51
+ log = get_logger(__name__)
52
+
53
+ #: The sandbox root ``sandboxd`` serves files from (its ``--root-dir``).
54
+ WORKSPACE = "/workspace"
55
+ #: The largest ``timeout_s`` that still uses unary exec.
56
+ UNARY_MAX_TIMEOUT_S = 60.0
57
+ #: How often a background holder prints a line, well inside the gateway's
58
+ #: 30 min SSE idle limit (which keepalive comments do not reset).
59
+ HEARTBEAT_S = 60
60
+ #: The gateway's limit on one stdin write; ``run(stdin=...)`` sends larger input in pieces.
61
+ STDIN_MAX_BYTES = 1 << 20
62
+ #: How much of a background process's log ``attach`` replays by default.
63
+ DEFAULT_TAIL_BYTES = 64 * 1024
64
+ #: How long :meth:`Process.detach` waits for the holder thread to close its stream.
65
+ DETACH_JOIN_S = 5.0
66
+
67
+ # Answers that mean the command was never started, so sending it again cannot
68
+ # run it twice: the routing and admission refusals before the gateway dials.
69
+ # 503 sandbox_unreachable is left out: Execute can fail with it mid-run, and
70
+ # so is 503 exec_interrupted (the drain window ended while the command ran).
71
+ EXEC_RETRY_CODES = frozenset(
72
+ {
73
+ "sandbox_not_ready",
74
+ "sandbox_not_found",
75
+ "index_stale",
76
+ "ambiguous_backend",
77
+ "gateway_at_capacity",
78
+ "auth_backend_unavailable",
79
+ "draining",
80
+ "too_many_streams",
81
+ "principal_budget_exceeded",
82
+ }
83
+ )
84
+
85
+ # The unary exec's answer when the gateway's drain window ends while the
86
+ # command runs (ADR-007 §Error-code table): it ran, and was killed.
87
+ EXEC_INTERRUPTED = "exec_interrupted"
88
+
89
+ # An SSE stream sees a keepalive comment every 15 s; four missed is a dead link.
90
+ _STREAM_TIMEOUT = httpx.Timeout(30.0, read=60.0)
91
+ _BG_PREFIX = "dz-bg:"
92
+
93
+ OutputCallback = Callable[[bytes], object]
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class CommandResult:
98
+ """A finished command: exit code and output (signal N is ``128 + N``)."""
99
+
100
+ exit_code: int
101
+ stdout: bytes = b""
102
+ stderr: bytes = b""
103
+ pid: int | None = None
104
+
105
+ @property
106
+ def ok(self) -> bool:
107
+ return self.exit_code == 0
108
+
109
+ @property
110
+ def stdout_text(self) -> str:
111
+ return self.stdout.decode("utf-8", "replace")
112
+
113
+ @property
114
+ def stderr_text(self) -> str:
115
+ return self.stderr.decode("utf-8", "replace")
116
+
117
+
118
+ class BackgroundLaunchFailed(errors.DevZeroSandboxError):
119
+ """The background launcher could not start the command (``code`` says why)."""
120
+
121
+ def __init__(self, code: str, message: str) -> None:
122
+ self.code = code
123
+ super().__init__(f"background launch failed ({code}): {message}")
124
+
125
+
126
+ def _argv(cmd: str | Sequence[str]) -> list[str]:
127
+ if isinstance(cmd, str):
128
+ return ["/bin/sh", "-c", cmd]
129
+ argv = [str(a) for a in cmd]
130
+ if not argv or not argv[0]:
131
+ raise ValueError("cmd must not be empty")
132
+ return argv
133
+
134
+
135
+ def _seconds(timeout_s: float | None) -> int:
136
+ """``timeout_s`` for the gateway: whole seconds, 0 for no bound."""
137
+ return 0 if not timeout_s else max(1, math.ceil(timeout_s))
138
+
139
+
140
+ @dataclass
141
+ class _Event:
142
+ name: str
143
+ data: dict[str, Any] = field(default_factory=dict)
144
+
145
+
146
+ def _events(resp: httpx.Response) -> Iterator[_Event]:
147
+ """The stream's events; keepalive comments are skipped by the parser."""
148
+ for sse in EventSource(resp).iter_sse():
149
+ try:
150
+ data = json.loads(sse.data) if sse.data else {}
151
+ except ValueError:
152
+ data = {}
153
+ yield _Event(sse.event, data if isinstance(data, dict) else {})
154
+
155
+
156
+ def _b64(data: dict[str, Any]) -> bytes:
157
+ return base64.b64decode(str(data.get("b64", "")))
158
+
159
+
160
+ class Commands:
161
+ """Run, attach to and kill commands in one sandbox; see the module docstring."""
162
+
163
+ def __init__(
164
+ self,
165
+ transport: GatewayTransport,
166
+ *,
167
+ workspace: str = WORKSPACE,
168
+ heartbeat_s: int = HEARTBEAT_S,
169
+ sleep: Callable[[float], None] | None = None,
170
+ ) -> None:
171
+ self._t = transport
172
+ self.workspace = workspace.rstrip("/") or "/"
173
+ self.heartbeat_s = heartbeat_s
174
+ # None: a holder's backoff waits on its stop flag, so detach() cuts it short.
175
+ self._sleep = sleep
176
+
177
+ @property
178
+ def background_dir(self) -> str:
179
+ """Where background processes keep ``<pid>.log``, ``.pid`` and ``.exit``."""
180
+ return f"{self.workspace}/.dz"
181
+
182
+ # --- run -----------------------------------------------------------------------
183
+
184
+ @overload
185
+ def run(
186
+ self,
187
+ cmd: str | Sequence[str],
188
+ *,
189
+ background: Literal[True],
190
+ timeout_s: float | None = ...,
191
+ stream: bool = ...,
192
+ cwd: str | None = ...,
193
+ env: Mapping[str, str] | None = ...,
194
+ stdin: bytes | None = ...,
195
+ on_stdout: OutputCallback | None = ...,
196
+ on_stderr: OutputCallback | None = ...,
197
+ ) -> Process: ...
198
+
199
+ @overload
200
+ def run(
201
+ self,
202
+ cmd: str | Sequence[str],
203
+ *,
204
+ background: Literal[False] = ...,
205
+ timeout_s: float | None = ...,
206
+ stream: bool = ...,
207
+ cwd: str | None = ...,
208
+ env: Mapping[str, str] | None = ...,
209
+ stdin: bytes | None = ...,
210
+ on_stdout: OutputCallback | None = ...,
211
+ on_stderr: OutputCallback | None = ...,
212
+ ) -> CommandResult: ...
213
+
214
+ def run(
215
+ self,
216
+ cmd: str | Sequence[str],
217
+ *,
218
+ background: bool = False,
219
+ timeout_s: float | None = 60,
220
+ stream: bool = False,
221
+ cwd: str | None = None,
222
+ env: Mapping[str, str] | None = None,
223
+ stdin: bytes | None = None,
224
+ on_stdout: OutputCallback | None = None,
225
+ on_stderr: OutputCallback | None = None,
226
+ ) -> CommandResult | Process:
227
+ """Run ``cmd`` (a shell string, or an argv list) and return its result.
228
+
229
+ ``timeout_s`` bounds the command (``None`` or ``0``: no bound); the
230
+ gateway kills it when the time is up (``UpstreamTimeout``). Unary
231
+ exec is used unless ``stream``, ``timeout_s`` above 60 s or unbounded,
232
+ ``stdin`` or an output callback asks for the stream. A non-zero exit
233
+ is a result, not an exception.
234
+
235
+ ``background=True`` starts the command detached and returns a
236
+ :class:`Process` at once; ``timeout_s`` does not apply to it.
237
+ """
238
+ argv = _argv(cmd)
239
+ if background:
240
+ return self._launch(argv, cwd=cwd, env=env)
241
+ if (
242
+ stream
243
+ or not timeout_s
244
+ or timeout_s > UNARY_MAX_TIMEOUT_S
245
+ or stdin is not None
246
+ or on_stdout is not None
247
+ or on_stderr is not None
248
+ ):
249
+ return self._stream(
250
+ argv,
251
+ timeout_s=timeout_s,
252
+ cwd=cwd,
253
+ env=env,
254
+ stdin=stdin,
255
+ on_stdout=on_stdout,
256
+ on_stderr=on_stderr,
257
+ )
258
+ return self._unary(argv, timeout_s=timeout_s, cwd=cwd, env=env)
259
+
260
+ def _body(
261
+ self,
262
+ argv: list[str],
263
+ timeout_s: float | None,
264
+ cwd: str | None,
265
+ env: Mapping[str, str] | None,
266
+ ) -> dict[str, Any]:
267
+ body: dict[str, Any] = {"cmd": argv, "timeout_s": _seconds(timeout_s)}
268
+ if cwd is not None:
269
+ body["cwd"] = cwd
270
+ if env:
271
+ body["env"] = dict(env)
272
+ return body
273
+
274
+ def _unary(
275
+ self,
276
+ argv: list[str],
277
+ *,
278
+ timeout_s: float,
279
+ cwd: str | None,
280
+ env: Mapping[str, str] | None,
281
+ ) -> CommandResult:
282
+ secs = _seconds(timeout_s)
283
+ try:
284
+ resp = self._t.request(
285
+ "POST",
286
+ "/v1/exec",
287
+ scope="exec",
288
+ json=self._body(argv, timeout_s, cwd, env),
289
+ headers={"X-Timeout": str(secs)},
290
+ # The gateway answers 504 at the deadline; leave room for that answer.
291
+ timeout=httpx.Timeout(30.0, read=secs + 30.0),
292
+ retry_codes=EXEC_RETRY_CODES,
293
+ )
294
+ except errors.GatewayError as e:
295
+ if e.code == EXEC_INTERRUPTED:
296
+ # The command ran and the gateway's drain killed it (unary
297
+ # exec reports no pid); never re-run it.
298
+ raise errors.StreamInterrupted(
299
+ pid=None, code="drain", message=e.message
300
+ ) from None
301
+ raise
302
+ out = resp.json()
303
+ return CommandResult(
304
+ exit_code=int(out["exit_code"]),
305
+ stdout=base64.b64decode(out.get("stdout_b64") or ""),
306
+ stderr=base64.b64decode(out.get("stderr_b64") or ""),
307
+ )
308
+
309
+ def _open(
310
+ self,
311
+ argv: list[str],
312
+ timeout_s: float | None,
313
+ cwd: str | None,
314
+ env: Mapping[str, str] | None,
315
+ ) -> AbstractContextManager[httpx.Response]:
316
+ return self._t.stream(
317
+ "POST",
318
+ "/v1/exec/stream",
319
+ scope="exec",
320
+ json=self._body(argv, timeout_s, cwd, env),
321
+ headers={"Accept": "text/event-stream"},
322
+ timeout=_STREAM_TIMEOUT,
323
+ retry_codes=EXEC_RETRY_CODES,
324
+ )
325
+
326
+ def _stream(
327
+ self,
328
+ argv: list[str],
329
+ *,
330
+ timeout_s: float | None,
331
+ cwd: str | None,
332
+ env: Mapping[str, str] | None,
333
+ stdin: bytes | None,
334
+ on_stdout: OutputCallback | None,
335
+ on_stderr: OutputCallback | None,
336
+ ) -> CommandResult:
337
+ pid: int | None = None
338
+ drained = False
339
+ out: list[bytes] = []
340
+ err: list[bytes] = []
341
+ try:
342
+ with self._open(argv, timeout_s, cwd, env) as resp:
343
+ for ev in _events(resp):
344
+ if ev.name == "init":
345
+ pid = int(ev.data["pid"])
346
+ if stdin is not None:
347
+ self._write_all_stdin(pid, stdin)
348
+ elif ev.name in ("stdout", "stderr"):
349
+ chunk = _b64(ev.data)
350
+ (out if ev.name == "stdout" else err).append(chunk)
351
+ cb = on_stdout if ev.name == "stdout" else on_stderr
352
+ if cb is not None:
353
+ cb(chunk)
354
+ elif ev.name == "exit":
355
+ return CommandResult(
356
+ int(ev.data.get("code", -1)),
357
+ b"".join(out),
358
+ b"".join(err),
359
+ pid,
360
+ )
361
+ elif ev.name == "drain":
362
+ # A gateway rollout: the stream keeps running for the
363
+ # drain window (30 min), then the gateway closes it.
364
+ drained = True
365
+ log.debug("exec stream for pid %s: gateway draining", pid)
366
+ elif ev.name == "error":
367
+ raise _stream_error(pid, ev.data, drained)
368
+ except (httpx.TransportError, SSEError) as e:
369
+ raise errors.StreamInterrupted(
370
+ pid=pid,
371
+ code="drain" if drained else "disconnected",
372
+ message=f"the exec stream broke ({type(e).__name__}); the process was killed",
373
+ ) from None
374
+ raise errors.StreamInterrupted(
375
+ pid=pid,
376
+ code="drain" if drained else "disconnected",
377
+ message="the exec stream ended before the process exited; the process was killed",
378
+ )
379
+
380
+ # --- control calls -------------------------------------------------------------
381
+
382
+ def _write_all_stdin(self, pid: int, data: bytes) -> None:
383
+ # In order, each piece within the gateway's per-call limit; eof closes
384
+ # stdin with the last piece (an empty input is one empty eof write).
385
+ last = max(len(data) - 1, 0) // STDIN_MAX_BYTES * STDIN_MAX_BYTES
386
+ for i in range(0, last + 1, STDIN_MAX_BYTES):
387
+ self.write_stdin(pid, data[i : i + STDIN_MAX_BYTES], eof=i == last)
388
+
389
+ def write_stdin(self, pid: int, data: bytes, *, eof: bool = False) -> None:
390
+ """Write to a streamed process's stdin (at most 1 MiB per call); ``eof`` closes it."""
391
+ self._t.request(
392
+ "POST",
393
+ f"/v1/exec/{int(pid)}/stdin",
394
+ scope="exec",
395
+ content=data,
396
+ params={"eof": "true"} if eof else None,
397
+ headers={"Content-Type": "application/octet-stream"},
398
+ )
399
+
400
+ def signal(self, pid: int, signal: Literal["INT", "TERM", "KILL"]) -> None:
401
+ """Send ``INT``, ``TERM`` or ``KILL`` to a streamed process's group."""
402
+ self._t.request(
403
+ "POST", f"/v1/exec/{int(pid)}/signal", scope="exec", json={"signal": signal}
404
+ )
405
+
406
+ def kill(self, pid: int) -> None:
407
+ """Kill a process: a background one (its whole tree), or a streamed one.
408
+
409
+ Raises :class:`~devzero_sandbox.errors.ProcessNotFound` when neither
410
+ exists (or the process has already exited).
411
+ """
412
+ res = self._unary(self._bg_argv("kill", pid), timeout_s=30, cwd=None, env=None)
413
+ if res.exit_code == 0:
414
+ return
415
+ if res.exit_code not in (3, 4):
416
+ raise BackgroundLaunchFailed(
417
+ "kill_failed", res.stderr_text or res.stdout_text
418
+ )
419
+ # Not a running background process: the pid of a stream (DELETE = SIGKILL).
420
+ self._t.request("DELETE", f"/v1/exec/{int(pid)}", scope="exec")
421
+
422
+ # --- background ----------------------------------------------------------------
423
+
424
+ def _bg_argv(
425
+ self, mode: str, pid: int | None = None, argv: Sequence[str] = ()
426
+ ) -> list[str]:
427
+ out = [
428
+ "/bin/sh",
429
+ "-c",
430
+ _BG_SCRIPT,
431
+ "dz-bg",
432
+ mode,
433
+ self.background_dir,
434
+ str(self.heartbeat_s),
435
+ ]
436
+ if pid is not None:
437
+ out.append(str(int(pid)))
438
+ return out + list(argv)
439
+
440
+ def _launch(
441
+ self, argv: list[str], *, cwd: str | None, env: Mapping[str, str] | None
442
+ ) -> Process:
443
+ cm = self._open(self._bg_argv("launch", argv=argv), None, cwd, env)
444
+ resp = cm.__enter__()
445
+ try:
446
+ events = _events(resp)
447
+ pid: int | None = None
448
+ lines = _Lines()
449
+ for ev in events:
450
+ if ev.name == "init":
451
+ pid = int(ev.data["pid"])
452
+ # The launcher names its files after the pid sandboxd gave it.
453
+ self.write_stdin(pid, f"{pid}\n".encode(), eof=True)
454
+ elif ev.name == "stdout":
455
+ for line in lines.feed(_b64(ev.data)):
456
+ if line == _BG_PREFIX + "started" and pid is not None:
457
+ proc = Process(self, pid)
458
+ proc._hold(cm, resp, events)
459
+ return proc
460
+ _raise_bg_error(line)
461
+ elif ev.name == "exit":
462
+ raise BackgroundLaunchFailed(
463
+ "exited", f"launcher exited {ev.data.get('code')}"
464
+ )
465
+ elif ev.name == "error":
466
+ raise _stream_error(pid, ev.data, False)
467
+ raise errors.StreamInterrupted(
468
+ pid=pid,
469
+ code="disconnected",
470
+ message="the launch stream ended before the process started",
471
+ )
472
+ except BaseException:
473
+ cm.__exit__(None, None, None)
474
+ raise
475
+
476
+ def attach(
477
+ self, pid: int, *, tail_bytes: int = DEFAULT_TAIL_BYTES, hold: bool = True
478
+ ) -> Process:
479
+ """Find a background process again (from any SDK process) and replay its log tail.
480
+
481
+ The returned :class:`Process` has the last ``tail_bytes`` of the log in
482
+ ``tail`` and, while the process runs and ``hold`` is set, a holder
483
+ stream that reports its exit.
484
+ """
485
+ from devzero_sandbox.files import Files
486
+
487
+ files = Files(self._t, workspace=self.workspace)
488
+ rel = files.relative(f"{self.background_dir}/{int(pid)}")
489
+ try:
490
+ files.read(rel + ".pid")
491
+ except errors.SandboxFileNotFound:
492
+ raise errors.ProcessNotFound(
493
+ status=404,
494
+ code="process_not_found",
495
+ message=f"no background process {pid} in {self.background_dir}",
496
+ pid=int(pid),
497
+ ) from None
498
+ proc = Process(self, int(pid))
499
+ proc.tail = proc.logs(tail_bytes)
500
+ code = proc._read_exit()
501
+ if code is not None:
502
+ proc._finish(code, None)
503
+ elif hold:
504
+ proc._reopen()
505
+ return proc
506
+
507
+
508
+ class _Lines:
509
+ """Splits a byte stream into ``dz-bg:`` protocol lines (heartbeats are empty lines)."""
510
+
511
+ def __init__(self) -> None:
512
+ self._buf = b""
513
+
514
+ def feed(self, data: bytes) -> list[str]:
515
+ self._buf += data
516
+ *done, self._buf = self._buf.split(b"\n")
517
+ return [
518
+ d.decode("utf-8", "replace")
519
+ for d in done
520
+ if d.startswith(_BG_PREFIX.encode())
521
+ ]
522
+
523
+
524
+ def _raise_bg_error(line: str) -> None:
525
+ if line.startswith(_BG_PREFIX + "error:"):
526
+ code, _, msg = line[len(_BG_PREFIX + "error:") :].partition(":")
527
+ if code == "not_found":
528
+ raise errors.ProcessNotFound(
529
+ status=404, code="process_not_found", message=msg
530
+ )
531
+ raise BackgroundLaunchFailed(code, msg)
532
+
533
+
534
+ def _stream_error(
535
+ pid: int | None, data: dict[str, Any], drained: bool
536
+ ) -> errors.DevZeroSandboxError:
537
+ code = str(data.get("code") or "")
538
+ message = str(data.get("message") or "")
539
+ if code == "upstream_timeout":
540
+ return errors.UpstreamTimeout(status=504, code=code, message=message, pid=pid)
541
+ return errors.StreamInterrupted(
542
+ pid=pid, code=code or ("drain" if drained else "error"), message=message
543
+ )
544
+
545
+
546
+ class Process:
547
+ """A background process (``run(..., background=True)`` or ``attach(pid)``).
548
+
549
+ The process runs on its own in the sandbox: :meth:`detach`, the SDK exiting
550
+ or the holder stream breaking never stops it; :meth:`kill` does. ``tail``
551
+ holds the log tail ``attach`` replayed.
552
+ """
553
+
554
+ def __init__(self, commands: Commands, pid: int) -> None:
555
+ self._c = commands
556
+ self.pid = pid
557
+ self.tail = b""
558
+ self._done = threading.Event()
559
+ self._closing = threading.Event()
560
+ self._lock = threading.Lock()
561
+ # The holder's stream. Only the holder thread reads or closes it while
562
+ # that thread runs; other threads only shut its socket down (_wake_holder).
563
+ self._cm: AbstractContextManager[httpx.Response] | None = None
564
+ self._resp: httpx.Response | None = None
565
+ self._thread: threading.Thread | None = None
566
+ self._exit_code: int | None = None
567
+ self._error: BaseException | None = None
568
+
569
+ @property
570
+ def log_path(self) -> str:
571
+ """The log in the sandbox (stdout and stderr, interleaved)."""
572
+ return f"{self._c.background_dir}/{self.pid}.log"
573
+
574
+ @property
575
+ def exit_code(self) -> int | None:
576
+ return self._exit_code
577
+
578
+ def is_running(self) -> bool:
579
+ if self._done.is_set():
580
+ return False
581
+ code = self._read_exit()
582
+ if code is not None:
583
+ self._finish(code, None)
584
+ return False
585
+ return True
586
+
587
+ def logs(self, tail_bytes: int | None = None) -> bytes:
588
+ """The process's log so far, or its last ``tail_bytes``."""
589
+ from devzero_sandbox.files import Files
590
+
591
+ files = Files(self._c._t, workspace=self._c.workspace)
592
+ return files.read(files.relative(self.log_path), tail_bytes=tail_bytes)
593
+
594
+ def wait(self, timeout_s: float | None = None) -> int:
595
+ """Block until the process exits and return its exit code (137 when killed)."""
596
+ if not self._done.is_set() and self._thread is None:
597
+ self._reopen()
598
+ if not self._done.wait(timeout_s):
599
+ raise TimeoutError(
600
+ f"background process {self.pid} still running after {timeout_s}s"
601
+ )
602
+ if self._error is not None:
603
+ raise self._error
604
+ assert self._exit_code is not None
605
+ return self._exit_code
606
+
607
+ def kill(self) -> None:
608
+ """SIGKILL the process and everything it started.
609
+
610
+ The holder (if any) then reports exit code 137 to :meth:`wait`. Killing
611
+ a process that has already exited is a no-op.
612
+ """
613
+ if self._done.is_set() and self._error is None:
614
+ return
615
+ try:
616
+ self._c.kill(self.pid)
617
+ except errors.ProcessNotFound:
618
+ if self._read_exit() is None:
619
+ raise
620
+
621
+ def detach(self) -> None:
622
+ """Stop holding the process (close the holder stream); it keeps running.
623
+
624
+ Returns once the holder thread has closed its stream (at most
625
+ ``DETACH_JOIN_S``). The stream is closed by the thread that reads it:
626
+ this shuts its socket down to wake that thread, never closes it under
627
+ it. Calling it again is a no-op.
628
+ """
629
+ self._closing.set()
630
+ thread = self._thread
631
+ if (
632
+ thread is None
633
+ or thread is threading.current_thread()
634
+ or not thread.is_alive()
635
+ ):
636
+ self._close_cm() # no holder is reading it
637
+ return
638
+ self._wake_holder()
639
+ thread.join(DETACH_JOIN_S)
640
+ if thread.is_alive():
641
+ # Still opening a stream: it sees _closing and closes it itself.
642
+ log.debug("holder of background pid %d still stopping", self.pid)
643
+
644
+ close = detach
645
+
646
+ def __repr__(self) -> str:
647
+ state = "running" if not self._done.is_set() else f"exited {self._exit_code}"
648
+ return f"Process(pid={self.pid}, {state})"
649
+
650
+ # --- the holder ----------------------------------------------------------------
651
+
652
+ def _read_exit(self) -> int | None:
653
+ from devzero_sandbox.files import Files
654
+
655
+ files = Files(self._c._t, workspace=self._c.workspace)
656
+ try:
657
+ raw = files.read(
658
+ files.relative(f"{self._c.background_dir}/{self.pid}.exit")
659
+ )
660
+ except errors.SandboxFileNotFound:
661
+ return None
662
+ try:
663
+ return int(raw.strip())
664
+ except ValueError:
665
+ return None
666
+
667
+ def _finish(self, code: int | None, error: BaseException | None) -> None:
668
+ self._exit_code, self._error = code, error
669
+ self._done.set()
670
+
671
+ def _hold(
672
+ self,
673
+ cm: AbstractContextManager[httpx.Response],
674
+ resp: httpx.Response,
675
+ events: Iterator[_Event],
676
+ ) -> None:
677
+ with self._lock:
678
+ self._cm, self._resp = cm, resp
679
+ self._thread = threading.Thread(
680
+ target=self._run_holder,
681
+ args=(events,),
682
+ name=f"dz-bg-holder-{self.pid}",
683
+ daemon=True,
684
+ )
685
+ self._thread.start()
686
+
687
+ def _reopen(self) -> None:
688
+ """Open a holder (watch mode) for a process launched elsewhere."""
689
+ self._thread = threading.Thread(
690
+ target=self._run_holder,
691
+ args=(None,),
692
+ name=f"dz-bg-holder-{self.pid}",
693
+ daemon=True,
694
+ )
695
+ self._thread.start()
696
+
697
+ def _run_holder(self, events: Iterator[_Event] | None) -> None:
698
+ failures = 0
699
+ while not self._closing.is_set():
700
+ if events is None:
701
+ try:
702
+ cm = self._c._open(
703
+ self._c._bg_argv("watch", self.pid), None, None, None
704
+ )
705
+ resp = cm.__enter__()
706
+ except errors.GatewayError as e:
707
+ if not e.retryable or failures >= 5:
708
+ self._finish(None, e)
709
+ return
710
+ failures += 1
711
+ self._pause(min(0.5 * 2**failures, 8.0))
712
+ continue
713
+ except errors.DevZeroSandboxError as e:
714
+ self._finish(None, e)
715
+ return
716
+ with self._lock:
717
+ if self._closing.is_set():
718
+ cm.__exit__(None, None, None)
719
+ return
720
+ self._cm, self._resp = cm, resp
721
+ events = _events(resp)
722
+ try:
723
+ code = self._consume(events)
724
+ except errors.ProcessNotFound as e:
725
+ self._finish(None, e)
726
+ return
727
+ except (
728
+ httpx.TransportError,
729
+ SSEError,
730
+ errors.StreamInterrupted,
731
+ errors.UpstreamTimeout,
732
+ ):
733
+ code = None
734
+ except BaseException as e: # noqa: BLE001 -- a holder never dies silently: wait() raises it
735
+ if self._closing.is_set():
736
+ return
737
+ self._finish(None, e)
738
+ return
739
+ finally:
740
+ self._close_cm()
741
+ events = None
742
+ if code is not None:
743
+ self._finish(code, None)
744
+ return
745
+ if self._closing.is_set():
746
+ return
747
+ # Drain, the 24 h stream lifetime or a dropped link: the process is
748
+ # unaffected; hold it again from wherever the gateway routes us.
749
+ log.debug("holder for background pid %d ended; re-opening", self.pid)
750
+ failures += 1
751
+ self._pause(min(0.5 * 2 ** min(failures, 4), 5.0))
752
+ if failures > 20:
753
+ self._finish(
754
+ None,
755
+ errors.StreamInterrupted(
756
+ pid=self.pid,
757
+ code="disconnected",
758
+ message="could not hold the background process",
759
+ ),
760
+ )
761
+ return
762
+
763
+ def _consume(self, events: Iterator[_Event]) -> int | None:
764
+ lines = _Lines()
765
+ for ev in events:
766
+ if ev.name == "stdout":
767
+ for line in lines.feed(_b64(ev.data)):
768
+ if line.startswith(_BG_PREFIX + "exit:"):
769
+ return int(line[len(_BG_PREFIX + "exit:") :])
770
+ _raise_bg_error(line)
771
+ elif ev.name == "error":
772
+ raise _stream_error(self.pid, ev.data, False)
773
+ return None
774
+
775
+ def _pause(self, seconds: float) -> None:
776
+ if self._c._sleep is not None:
777
+ self._c._sleep(seconds)
778
+ else:
779
+ self._closing.wait(seconds)
780
+
781
+ def _wake_holder(self) -> None:
782
+ """Wake a holder blocked reading its stream, from another thread.
783
+
784
+ Only the kernel socket is shut down: the reader's next read returns end
785
+ of stream and the holder closes the stream itself. Closing the stream
786
+ here instead would close the TLS object and the file descriptor under
787
+ the reader (a read on a closed or reused fd, or an error from inside
788
+ OpenSSL). ``socket.socket.shutdown``, not ``SSLSocket.shutdown``, which
789
+ also drops the SSL object the reader is using.
790
+ """
791
+ with self._lock:
792
+ sock = _socket_of(self._resp)
793
+ if sock is None:
794
+ return
795
+ try:
796
+ socket.socket.shutdown(sock, socket.SHUT_RDWR)
797
+ except OSError:
798
+ pass # already closed or never connected: nothing to wake
799
+
800
+ def _close_cm(self) -> None:
801
+ """Close the holder stream: from the holder thread, or when none runs."""
802
+ with self._lock:
803
+ cm, self._cm, self._resp = self._cm, None, None
804
+ if cm is not None:
805
+ try:
806
+ cm.__exit__(None, None, None)
807
+ except Exception as e: # noqa: BLE001 -- a stream we woke may end uncleanly
808
+ log.debug("closing the holder stream of pid %d: %r", self.pid, e)
809
+
810
+
811
+ def _socket_of(resp: httpx.Response | None) -> socket.socket | None:
812
+ """The connection's socket under a streamed response (httpcore's sync backend)."""
813
+ if resp is None:
814
+ return None
815
+ stream = resp.extensions.get("network_stream")
816
+ get = getattr(stream, "get_extra_info", None)
817
+ sock = get("socket") if get is not None else None
818
+ return sock if isinstance(sock, socket.socket) else None