deft-subspace-kernel 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deft_subspace/kernel/__init__.py +93 -0
- deft_subspace/kernel/abort.py +173 -0
- deft_subspace/kernel/channel.py +244 -0
- deft_subspace/kernel/clock.py +39 -0
- deft_subspace/kernel/errors.py +114 -0
- deft_subspace/kernel/framing.py +171 -0
- deft_subspace/kernel/process.py +614 -0
- deft_subspace/kernel/py.typed +0 -0
- deft_subspace/kernel/rpc.py +815 -0
- deft_subspace/kernel/testing.py +97 -0
- deft_subspace/kernel/trace.py +143 -0
- deft_subspace_kernel-0.1.0a1.dist-info/METADATA +41 -0
- deft_subspace_kernel-0.1.0a1.dist-info/RECORD +15 -0
- deft_subspace_kernel-0.1.0a1.dist-info/WHEEL +4 -0
- deft_subspace_kernel-0.1.0a1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,614 @@
|
|
|
1
|
+
"""Run another program as a supervised child and talk to it over stdio.
|
|
2
|
+
|
|
3
|
+
- On POSIX the child gets its own process group (a new session), so stopping
|
|
4
|
+
it also stops whatever it started (`npx foo` runs `foo` as a grandchild).
|
|
5
|
+
On Windows, which has no process groups to signal, stop() ends the child and
|
|
6
|
+
every process it started with `taskkill /T` (run directly, no shell): without
|
|
7
|
+
/F first, then with /F after the grace period; if taskkill cannot run, the
|
|
8
|
+
child alone is killed. This matters for `.cmd` launches, where the program is
|
|
9
|
+
a child of cmd.exe and would otherwise keep stdout open.
|
|
10
|
+
- stop() closes stdin, sends SIGTERM, then SIGKILL after a grace period.
|
|
11
|
+
- The last stderr output is kept for error messages.
|
|
12
|
+
- `ended` settles only after the process has exited *and* its stdout has
|
|
13
|
+
closed, so a final reply written just before exit is not lost: it is still
|
|
14
|
+
there to read.
|
|
15
|
+
- `exited`, `ended()` and `stop()` can be cancelled (e.g. by `asyncio.wait_for`)
|
|
16
|
+
without affecting the process or later calls.
|
|
17
|
+
- A missing program becomes a SpawnError with a hint.
|
|
18
|
+
- On Windows, a command found through PATH and PATHEXT as a `.cmd` or `.bat`
|
|
19
|
+
file (like `npx.cmd`) is run through cmd.exe with its arguments escaped as
|
|
20
|
+
Rust's std escapes them ("BatBadBut", CVE-2024-24576): quoted, quotes doubled,
|
|
21
|
+
`%` defused, so that neither cmd.exe nor a shim that hands `%*` to cmd.exe
|
|
22
|
+
again can be made to run anything else. An argument containing CR, LF or NUL
|
|
23
|
+
cannot be passed safely and is refused with a SpawnError (code `EINVAL`).
|
|
24
|
+
|
|
25
|
+
Process groups outlive this process unless it stops them, so the live ones are
|
|
26
|
+
killed when it exits: at normal interpreter exit (atexit, which also covers
|
|
27
|
+
Ctrl-C under `asyncio.run`), and, on POSIX, on SIGTERM and SIGHUP. Those
|
|
28
|
+
signal handlers are installed only while supervised groups are alive, only
|
|
29
|
+
from the main thread, and only for signals still at their default action: an
|
|
30
|
+
app's own handler (`signal.signal` or `loop.add_signal_handler`) is left alone,
|
|
31
|
+
and then stopping the children is up to the app. After killing the groups the
|
|
32
|
+
handler restores the default action and re-sends the signal, so the process
|
|
33
|
+
still dies of it.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import asyncio
|
|
39
|
+
import atexit
|
|
40
|
+
import codecs
|
|
41
|
+
import ntpath
|
|
42
|
+
import os
|
|
43
|
+
import re
|
|
44
|
+
import signal as signals
|
|
45
|
+
import subprocess
|
|
46
|
+
import sys
|
|
47
|
+
import threading
|
|
48
|
+
import time
|
|
49
|
+
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
50
|
+
from dataclasses import dataclass
|
|
51
|
+
from types import FrameType
|
|
52
|
+
|
|
53
|
+
from .channel import ChannelClosed, EndOfChannel
|
|
54
|
+
from .errors import SubspaceError
|
|
55
|
+
|
|
56
|
+
_WINDOWS = sys.platform == "win32"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class ExitInfo:
|
|
61
|
+
#: The exit code, or None when a signal ended the process.
|
|
62
|
+
code: int | None
|
|
63
|
+
#: The signal's name (`SIGTERM`), or None.
|
|
64
|
+
signal: str | None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class SpawnError(SubspaceError):
|
|
68
|
+
def __init__(self, command: str, code: str | None, hint: str) -> None:
|
|
69
|
+
super().__init__(f"Could not start {command}{f' ({code})' if code else ''}. {hint}")
|
|
70
|
+
self.command = command
|
|
71
|
+
self.code = code
|
|
72
|
+
self.hint = hint
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# -- Killing live groups when this process ends --------------------------------------------
|
|
76
|
+
|
|
77
|
+
#: How to kill each process group that may still be alive.
|
|
78
|
+
_live: set[Callable[[], None]] = set()
|
|
79
|
+
_atexit_installed = False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _kill_live() -> None:
|
|
83
|
+
for kill in list(_live):
|
|
84
|
+
kill()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _track(kill: Callable[[], None]) -> None:
|
|
88
|
+
global _atexit_installed
|
|
89
|
+
_live.add(kill)
|
|
90
|
+
if not _atexit_installed:
|
|
91
|
+
_atexit_installed = True
|
|
92
|
+
atexit.register(_kill_live)
|
|
93
|
+
_install_signal_hooks()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _untrack(kill: Callable[[], None]) -> None:
|
|
97
|
+
_live.discard(kill)
|
|
98
|
+
if not _live:
|
|
99
|
+
_remove_signal_hooks()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
#: Signals that end a process by default and so skip atexit.
|
|
103
|
+
_FATAL_SIGNALS = ("SIGTERM", "SIGHUP")
|
|
104
|
+
_hooked: set[int] = set()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _on_fatal_signal(signum: int, _frame: FrameType | None) -> None:
|
|
108
|
+
_kill_live()
|
|
109
|
+
signals.signal(signum, signals.SIG_DFL)
|
|
110
|
+
_hooked.discard(signum)
|
|
111
|
+
os.kill(os.getpid(), signum)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _can_set_signal_handlers() -> bool:
|
|
115
|
+
return not _WINDOWS and threading.current_thread() is threading.main_thread()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _install_signal_hooks() -> None:
|
|
119
|
+
"""POSIX: on a fatal signal nobody else handles, kill the live groups and die of it anyway."""
|
|
120
|
+
if not _can_set_signal_handlers():
|
|
121
|
+
return
|
|
122
|
+
for name in _FATAL_SIGNALS:
|
|
123
|
+
sig = getattr(signals, name, None)
|
|
124
|
+
if sig is None or sig in _hooked:
|
|
125
|
+
continue
|
|
126
|
+
try:
|
|
127
|
+
if signals.getsignal(sig) is not signals.SIG_DFL:
|
|
128
|
+
continue # the app (or Python) handles this one
|
|
129
|
+
signals.signal(sig, _on_fatal_signal)
|
|
130
|
+
except (OSError, ValueError):
|
|
131
|
+
continue
|
|
132
|
+
_hooked.add(sig)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _remove_signal_hooks() -> None:
|
|
136
|
+
if not _can_set_signal_handlers():
|
|
137
|
+
return # left in place; with nothing live they only restore the default and re-send
|
|
138
|
+
for sig in list(_hooked):
|
|
139
|
+
try:
|
|
140
|
+
if signals.getsignal(sig) is _on_fatal_signal:
|
|
141
|
+
signals.signal(sig, signals.SIG_DFL)
|
|
142
|
+
except (OSError, ValueError):
|
|
143
|
+
pass
|
|
144
|
+
_hooked.discard(sig)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# -- Windows command resolution ------------------------------------------------------------
|
|
148
|
+
|
|
149
|
+
_DEFAULT_PATHEXT = ".COM;.EXE;.BAT;.CMD"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _env_get(env: Mapping[str, str], name: str) -> str | None:
|
|
153
|
+
"""Windows environment names are case-insensitive (`Path`, `PATH`)."""
|
|
154
|
+
for key, value in env.items():
|
|
155
|
+
if key.upper() == name:
|
|
156
|
+
return value
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _windows_find(command: str, env: Mapping[str, str], is_file: Callable[[str], bool]) -> str | None:
|
|
161
|
+
"""Where Windows would find `command`, trying each PATHEXT extension, or None.
|
|
162
|
+
|
|
163
|
+
A bare name is looked up in PATH; a name with a directory is used as is. A name that
|
|
164
|
+
already ends in a PATHEXT extension, or in one of the default ones (`tool.cmd` with a
|
|
165
|
+
PATHEXT that leaves out .CMD), is looked for as it is.
|
|
166
|
+
"""
|
|
167
|
+
exts = [e for e in (_env_get(env, "PATHEXT") or _DEFAULT_PATHEXT).split(";") if e]
|
|
168
|
+
_, ext = ntpath.splitext(command)
|
|
169
|
+
known = ext.upper() in (e.upper() for e in [*exts, *_DEFAULT_PATHEXT.split(";")])
|
|
170
|
+
names = [command] if ext and known else [command + e for e in exts]
|
|
171
|
+
has_dir = any(c in command for c in "\\/:")
|
|
172
|
+
dirs = [""] if has_dir else [d.strip('"') for d in (_env_get(env, "PATH") or "").split(";") if d]
|
|
173
|
+
for directory in dirs:
|
|
174
|
+
for name in names:
|
|
175
|
+
path = ntpath.join(directory, name) if directory else name
|
|
176
|
+
if is_file(path):
|
|
177
|
+
return path
|
|
178
|
+
return None
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
#: What an argument to a batch file cannot carry: cmd.exe ends the command at CR or LF, and NUL ends the string.
|
|
182
|
+
_BAT_UNSAFE = re.compile(r"[\r\n\0]")
|
|
183
|
+
#: What may stay unquoted, as in Rust's std; any other ASCII character, or a control character, is quoted.
|
|
184
|
+
_BAT_PLAIN = re.compile(r"[A-Za-z0-9#$*+\-./:?@\\_\u00a0-\U0010ffff]*")
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _bat_argument(command: str, arg: str) -> str:
|
|
188
|
+
if _BAT_UNSAFE.search(arg):
|
|
189
|
+
raise SpawnError(command, "EINVAL", "An argument to a .cmd or .bat file cannot contain a line break or NUL.")
|
|
190
|
+
quote = arg == "" or arg.endswith("\\") or _BAT_PLAIN.fullmatch(arg) is None
|
|
191
|
+
out = ['"'] if quote else []
|
|
192
|
+
backslashes = 0
|
|
193
|
+
for c in arg:
|
|
194
|
+
if c == "\\":
|
|
195
|
+
backslashes += 1
|
|
196
|
+
else:
|
|
197
|
+
# A quote is doubled, so cmd.exe never sees the quoting end inside the argument; the
|
|
198
|
+
# program reads `""` within quotes as one `"`. Backslashes before it are doubled too.
|
|
199
|
+
if c == '"':
|
|
200
|
+
out.append("\\" * backslashes + '"')
|
|
201
|
+
# `%` becomes `%%cd:~,%`: cmd.exe expands `%cd:~,%` to nothing instead of a %VAR%.
|
|
202
|
+
elif c == "%":
|
|
203
|
+
out.append("%%cd:~,")
|
|
204
|
+
backslashes = 0
|
|
205
|
+
out.append(c)
|
|
206
|
+
if quote:
|
|
207
|
+
out.append("\\" * backslashes + '"')
|
|
208
|
+
return "".join(out)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _cmd_line(script: str, args: Sequence[str], command: str | None = None) -> str:
|
|
212
|
+
"""The cmd.exe command line that runs the `.cmd`/`.bat` file `script` with `args`.
|
|
213
|
+
|
|
214
|
+
Escaped as Rust's std escapes them since CVE-2024-24576 ("BatBadBut"): an argument with
|
|
215
|
+
anything but plain characters is quoted, its quotes doubled and each `%` defused, so
|
|
216
|
+
cmd.exe finds nothing to interpret in it however often it reads it. That covers shims
|
|
217
|
+
that hand `%*` to cmd.exe again (npx.cmd, npm's node_modules\\.bin shims), wherever they
|
|
218
|
+
are. An argument that cannot be passed safely (CR, LF or NUL) is a SpawnError.
|
|
219
|
+
"""
|
|
220
|
+
command = script if command is None else command
|
|
221
|
+
file = ntpath.normpath(script)
|
|
222
|
+
if '"' in file or file.endswith("\\"):
|
|
223
|
+
raise SpawnError(command, "EINVAL", f"{file!r} is not a valid file name.")
|
|
224
|
+
return " ".join([f'"{file}"', *(_bat_argument(command, a) for a in args)])
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _windows_launch(command: str, args: Sequence[str], env: Mapping[str, str], is_file: Callable[[str], bool]) -> str | None:
|
|
228
|
+
"""Windows can only start a `.cmd`/`.bat` file through cmd.exe: the command line to run, or None
|
|
229
|
+
when `command` is not one (and runs as given). Raises SpawnError for an argument that cannot be
|
|
230
|
+
passed safely."""
|
|
231
|
+
found = _windows_find(command, env, is_file)
|
|
232
|
+
if found is None or ntpath.splitext(found)[1].lower() not in (".cmd", ".bat"):
|
|
233
|
+
return None
|
|
234
|
+
return _cmd_line(found, args, command)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _cmd_exe(line: str, env: Mapping[str, str]) -> tuple[str, str]:
|
|
238
|
+
"""cmd.exe, and the whole command line to start it with, verbatim, to run `line`.
|
|
239
|
+
|
|
240
|
+
No AutoRun commands (/d; they could write to stdout), command extensions on (the `%`
|
|
241
|
+
escape needs them), no delayed `!` expansion, and only the outer quotes stripped (/s),
|
|
242
|
+
as the TypeScript port runs it. (`subprocess_shell` would run `COMSPEC /c "line"`.)
|
|
243
|
+
"""
|
|
244
|
+
system_root = _env_get(env, "SYSTEMROOT") or "C:\\Windows"
|
|
245
|
+
comspec = _env_get(env, "COMSPEC") or ntpath.join(system_root, "System32", "cmd.exe")
|
|
246
|
+
return comspec, f'"{comspec}" /d /e:ON /v:OFF /s /c "{line}"'
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _taskkill_command(pid: int, force: bool, env: Mapping[str, str]) -> list[str]:
|
|
250
|
+
"""The taskkill.exe command that ends process `pid` and every process it started (/T):
|
|
251
|
+
asking them to close, or forcing it (/F). Run directly, not through a shell."""
|
|
252
|
+
system_root = _env_get(env, "SYSTEMROOT") or "C:\\Windows"
|
|
253
|
+
return [ntpath.join(system_root, "System32", "taskkill.exe"), "/pid", str(pid), "/T", *(["/F"] if force else [])]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _not_found_hint(command: str, cwd: object, windows: bool) -> str:
|
|
257
|
+
where = f" (cwd: {cwd})" if cwd else ""
|
|
258
|
+
shims = " On Windows it is looked up with PATHEXT, so .cmd shims such as npx.cmd are found too." if windows else ""
|
|
259
|
+
return f'Is "{command}" installed and on PATH?{where}{shims}'
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
# -- The supervised process ----------------------------------------------------------------
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _exit_info(returncode: int) -> ExitInfo:
|
|
266
|
+
if returncode < 0 and not _WINDOWS:
|
|
267
|
+
try:
|
|
268
|
+
return ExitInfo(None, signals.Signals(-returncode).name)
|
|
269
|
+
except ValueError:
|
|
270
|
+
return ExitInfo(None, f"SIG{-returncode}")
|
|
271
|
+
return ExitInfo(returncode, None)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class _Protocol(asyncio.subprocess.SubprocessStreamProtocol):
|
|
275
|
+
"""The usual stream protocol, plus a future for "stdout has closed", whoever reads it."""
|
|
276
|
+
|
|
277
|
+
def __init__(self, limit: int, loop: asyncio.AbstractEventLoop) -> None:
|
|
278
|
+
super().__init__(limit=limit, loop=loop)
|
|
279
|
+
self.stdout_closed: asyncio.Future[None] = loop.create_future()
|
|
280
|
+
|
|
281
|
+
def _stdout_gone(self) -> None:
|
|
282
|
+
if not self.stdout_closed.done():
|
|
283
|
+
self.stdout_closed.set_result(None)
|
|
284
|
+
|
|
285
|
+
def pipe_connection_lost(self, fd: int, exc: Exception | None) -> None:
|
|
286
|
+
super().pipe_connection_lost(fd, exc)
|
|
287
|
+
if fd == 1:
|
|
288
|
+
self._stdout_gone()
|
|
289
|
+
|
|
290
|
+
def connection_lost(self, exc: Exception | None) -> None:
|
|
291
|
+
super().connection_lost(exc)
|
|
292
|
+
self._stdout_gone()
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class _ProcessChannel:
|
|
296
|
+
"""A child's stdin and stdout as a byte channel."""
|
|
297
|
+
|
|
298
|
+
def __init__(self, process: SupervisedProcess) -> None:
|
|
299
|
+
self._process = process
|
|
300
|
+
|
|
301
|
+
async def receive(self) -> bytes:
|
|
302
|
+
stdout = self._process._proc.stdout
|
|
303
|
+
assert stdout is not None
|
|
304
|
+
data = await stdout.read(65536)
|
|
305
|
+
if not data:
|
|
306
|
+
raise EndOfChannel()
|
|
307
|
+
return data
|
|
308
|
+
|
|
309
|
+
async def send(self, item: bytes) -> None:
|
|
310
|
+
stdin = self._process._proc.stdin
|
|
311
|
+
if stdin is None or stdin.is_closing():
|
|
312
|
+
raise ChannelClosed("stdin closed")
|
|
313
|
+
try:
|
|
314
|
+
stdin.write(item)
|
|
315
|
+
await stdin.drain()
|
|
316
|
+
except (ConnectionError, RuntimeError) as error:
|
|
317
|
+
raise ChannelClosed(str(error)) from error
|
|
318
|
+
|
|
319
|
+
async def close(self) -> None:
|
|
320
|
+
self._process._close_stdin()
|
|
321
|
+
|
|
322
|
+
async def abort(self) -> None:
|
|
323
|
+
"""Close stdin at once, dropping what is not yet written (a child that stopped reading):
|
|
324
|
+
a write stuck waiting for the pipe to drain ends instead of waiting forever."""
|
|
325
|
+
stdin = self._process._proc.stdin
|
|
326
|
+
if stdin is None:
|
|
327
|
+
return
|
|
328
|
+
try:
|
|
329
|
+
stdin.transport.abort()
|
|
330
|
+
except (OSError, RuntimeError, AttributeError):
|
|
331
|
+
self._process._close_stdin()
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
#: How often to check whether an exited leader's group is gone: from this, doubling, up to the max.
|
|
335
|
+
_POLL_FIRST = 0.05
|
|
336
|
+
_POLL_MAX = 1.0
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
class SupervisedProcess:
|
|
340
|
+
"""A running child. One supervisor task owns its lifecycle:
|
|
341
|
+
|
|
342
|
+
running -> exited (the process has exited) -> ended (and stdout has closed),
|
|
343
|
+
with "stopping" (stop() was called) on top. The supervisor also drops the
|
|
344
|
+
group from the exit-time kill list once its last member is gone, so a
|
|
345
|
+
reused group id is never killed. Callers only ever get shielded views of
|
|
346
|
+
the private futures and tasks, so cancelling a wait affects nothing else.
|
|
347
|
+
"""
|
|
348
|
+
|
|
349
|
+
def __init__(
|
|
350
|
+
self,
|
|
351
|
+
command: str,
|
|
352
|
+
proc: asyncio.subprocess.Process,
|
|
353
|
+
stdout_closed: Awaitable[None],
|
|
354
|
+
*,
|
|
355
|
+
stderr_tail_bytes: int,
|
|
356
|
+
kill_grace_ms: float,
|
|
357
|
+
on_stderr: Callable[[str], None] | None,
|
|
358
|
+
) -> None:
|
|
359
|
+
self.command = command
|
|
360
|
+
self._proc = proc
|
|
361
|
+
self.pid: int | None = proc.pid
|
|
362
|
+
self._tail_bytes = stderr_tail_bytes
|
|
363
|
+
self._grace_ms = kill_grace_ms
|
|
364
|
+
self._on_stderr = on_stderr
|
|
365
|
+
self._stderr = b""
|
|
366
|
+
loop = asyncio.get_running_loop()
|
|
367
|
+
self._exited: asyncio.Future[ExitInfo] = loop.create_future()
|
|
368
|
+
self._ended: asyncio.Future[ExitInfo] = loop.create_future()
|
|
369
|
+
self._stopping: asyncio.Task[ExitInfo] | None = None
|
|
370
|
+
#: stdin and stdout as a byte channel.
|
|
371
|
+
self.channel = _ProcessChannel(self)
|
|
372
|
+
self._kill_now = self._kill_group
|
|
373
|
+
_track(self._kill_now)
|
|
374
|
+
self._supervisor = loop.create_task(self._supervise(stdout_closed), name=f"{command}:supervise")
|
|
375
|
+
self._stderr_task = loop.create_task(self._read_stderr(), name=f"{command}:stderr") if proc.stderr else None
|
|
376
|
+
|
|
377
|
+
async def _supervise(self, stdout_closed: Awaitable[None]) -> None:
|
|
378
|
+
info = _exit_info(await self._proc.wait())
|
|
379
|
+
self._exited.set_result(info)
|
|
380
|
+
stdout = asyncio.ensure_future(stdout_closed)
|
|
381
|
+
delay = _POLL_FIRST
|
|
382
|
+
tracked = True
|
|
383
|
+
while True:
|
|
384
|
+
if stdout.done() and not self._ended.done():
|
|
385
|
+
self._ended.set_result(info)
|
|
386
|
+
if tracked and not self._group_alive():
|
|
387
|
+
_untrack(self._kill_now)
|
|
388
|
+
tracked = False
|
|
389
|
+
if self._ended.done() and not tracked:
|
|
390
|
+
return
|
|
391
|
+
if stdout.done():
|
|
392
|
+
await asyncio.sleep(delay)
|
|
393
|
+
else:
|
|
394
|
+
await asyncio.wait([stdout], timeout=delay if tracked else None)
|
|
395
|
+
delay = min(delay * 2, _POLL_MAX)
|
|
396
|
+
|
|
397
|
+
async def _read_stderr(self) -> None:
|
|
398
|
+
stderr = self._proc.stderr
|
|
399
|
+
assert stderr is not None
|
|
400
|
+
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
|
|
401
|
+
while True:
|
|
402
|
+
data = await stderr.read(65536)
|
|
403
|
+
if not data:
|
|
404
|
+
break
|
|
405
|
+
self._stderr = (self._stderr + data)[-self._tail_bytes :] if self._tail_bytes > 0 else b""
|
|
406
|
+
if self._on_stderr is not None:
|
|
407
|
+
try:
|
|
408
|
+
self._on_stderr(decoder.decode(data))
|
|
409
|
+
except Exception: # noqa: BLE001
|
|
410
|
+
pass
|
|
411
|
+
|
|
412
|
+
def _close_stdin(self) -> None:
|
|
413
|
+
stdin = self._proc.stdin
|
|
414
|
+
if stdin is not None and not stdin.is_closing():
|
|
415
|
+
try:
|
|
416
|
+
stdin.close()
|
|
417
|
+
except (OSError, RuntimeError):
|
|
418
|
+
pass
|
|
419
|
+
|
|
420
|
+
@property
|
|
421
|
+
def exited(self) -> asyncio.Future[ExitInfo]:
|
|
422
|
+
"""Settles when the process exits. Await it; cancelling the wait does not affect the process."""
|
|
423
|
+
return asyncio.shield(self._exited)
|
|
424
|
+
|
|
425
|
+
async def ended(self) -> ExitInfo:
|
|
426
|
+
"""Waits until the process has exited and stdout has closed."""
|
|
427
|
+
return await asyncio.shield(self._ended)
|
|
428
|
+
|
|
429
|
+
@property
|
|
430
|
+
def returncode(self) -> int | None:
|
|
431
|
+
return self._proc.returncode
|
|
432
|
+
|
|
433
|
+
def stderr_tail(self) -> str:
|
|
434
|
+
"""The last stderr output (at most `stderr_tail_bytes` bytes), trimmed."""
|
|
435
|
+
tail = self._stderr
|
|
436
|
+
if len(tail) >= self._tail_bytes:
|
|
437
|
+
tail = tail.lstrip(bytes(range(0x80, 0xC0))) # the tail of a character cut in half
|
|
438
|
+
return tail.decode("utf-8", errors="replace").strip()
|
|
439
|
+
|
|
440
|
+
def _signal_group(self, sig: int) -> bool:
|
|
441
|
+
try:
|
|
442
|
+
if sys.platform != "win32" and self.pid is not None:
|
|
443
|
+
os.killpg(self.pid, sig)
|
|
444
|
+
elif sig == signals.SIGTERM:
|
|
445
|
+
self._proc.terminate()
|
|
446
|
+
else:
|
|
447
|
+
self._proc.kill()
|
|
448
|
+
return True
|
|
449
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
450
|
+
return False # already gone
|
|
451
|
+
|
|
452
|
+
def _kill_group(self) -> None:
|
|
453
|
+
if self._tree_kill:
|
|
454
|
+
self._kill_tree_now()
|
|
455
|
+
else:
|
|
456
|
+
self._force_kill()
|
|
457
|
+
|
|
458
|
+
def _force_kill(self) -> None:
|
|
459
|
+
self._signal_group(getattr(signals, "SIGKILL", signals.SIGTERM))
|
|
460
|
+
|
|
461
|
+
# Windows has no process groups to signal, and killing cmd.exe would leave the program a
|
|
462
|
+
# .cmd file started (and its hold on stdout) running: the tree is ended with taskkill.
|
|
463
|
+
|
|
464
|
+
@property
|
|
465
|
+
def _tree_kill(self) -> bool:
|
|
466
|
+
return _WINDOWS and self.pid is not None
|
|
467
|
+
|
|
468
|
+
def _kill_tree_now(self) -> None:
|
|
469
|
+
"""taskkill /T /F, waiting for it (at exit); the child alone if taskkill cannot run or fails."""
|
|
470
|
+
assert self.pid is not None
|
|
471
|
+
try:
|
|
472
|
+
done = subprocess.run(
|
|
473
|
+
_taskkill_command(self.pid, True, os.environ),
|
|
474
|
+
stdin=subprocess.DEVNULL,
|
|
475
|
+
stdout=subprocess.DEVNULL,
|
|
476
|
+
stderr=subprocess.DEVNULL,
|
|
477
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
|
478
|
+
timeout=5,
|
|
479
|
+
check=False,
|
|
480
|
+
)
|
|
481
|
+
if done.returncode == 0:
|
|
482
|
+
return
|
|
483
|
+
except (OSError, subprocess.SubprocessError):
|
|
484
|
+
pass
|
|
485
|
+
self._force_kill()
|
|
486
|
+
|
|
487
|
+
async def _kill_tree(self, force: bool) -> None:
|
|
488
|
+
"""taskkill /T, with /F if `force`; when that cannot run or fails, a forced kill ends the child alone."""
|
|
489
|
+
assert self.pid is not None
|
|
490
|
+
try:
|
|
491
|
+
killer = await asyncio.create_subprocess_exec(
|
|
492
|
+
*_taskkill_command(self.pid, force, os.environ),
|
|
493
|
+
stdin=subprocess.DEVNULL,
|
|
494
|
+
stdout=subprocess.DEVNULL,
|
|
495
|
+
stderr=subprocess.DEVNULL,
|
|
496
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
|
497
|
+
)
|
|
498
|
+
if await killer.wait() == 0:
|
|
499
|
+
return
|
|
500
|
+
except OSError:
|
|
501
|
+
pass
|
|
502
|
+
if force:
|
|
503
|
+
self._force_kill()
|
|
504
|
+
|
|
505
|
+
def _group_alive(self) -> bool:
|
|
506
|
+
if sys.platform == "win32" or self.pid is None:
|
|
507
|
+
return self._proc.returncode is None
|
|
508
|
+
try:
|
|
509
|
+
os.killpg(self.pid, 0)
|
|
510
|
+
return True
|
|
511
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
512
|
+
return False
|
|
513
|
+
|
|
514
|
+
def stop(self, grace_ms: float | None = None) -> asyncio.Future[ExitInfo]:
|
|
515
|
+
"""Stop the process and its process group: end stdin, SIGTERM, then SIGKILL after `grace_ms`.
|
|
516
|
+
|
|
517
|
+
Idempotent: every call waits for the same stop. Cancelling the wait does not cancel the stop.
|
|
518
|
+
"""
|
|
519
|
+
if self._stopping is None:
|
|
520
|
+
self._stopping = asyncio.get_running_loop().create_task(self._stop(grace_ms), name=f"{self.command}:stop")
|
|
521
|
+
return asyncio.shield(self._stopping)
|
|
522
|
+
|
|
523
|
+
async def _stop(self, grace_ms: float | None) -> ExitInfo:
|
|
524
|
+
self._close_stdin()
|
|
525
|
+
if self._group_alive():
|
|
526
|
+
if self._tree_kill:
|
|
527
|
+
await self._kill_tree(False)
|
|
528
|
+
else:
|
|
529
|
+
self._signal_group(signals.SIGTERM)
|
|
530
|
+
grace = (self._grace_ms if grace_ms is None else grace_ms) / 1000
|
|
531
|
+
deadline = time.monotonic() + grace
|
|
532
|
+
while self._group_alive() and time.monotonic() < deadline:
|
|
533
|
+
await asyncio.sleep(0.025)
|
|
534
|
+
if self._group_alive():
|
|
535
|
+
if self._tree_kill:
|
|
536
|
+
await self._kill_tree(True)
|
|
537
|
+
else:
|
|
538
|
+
self._force_kill()
|
|
539
|
+
_untrack(self._kill_now)
|
|
540
|
+
return await asyncio.shield(self._exited)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
async def _start(
|
|
544
|
+
command: str, args: Sequence[str], limit: int, env: Mapping[str, str], **kwds: object
|
|
545
|
+
) -> tuple[asyncio.subprocess.Process, asyncio.Future[None]]:
|
|
546
|
+
"""asyncio.create_subprocess_exec, keeping the protocol to learn when stdout closes."""
|
|
547
|
+
loop = asyncio.get_running_loop()
|
|
548
|
+
protocol: _Protocol | None = None
|
|
549
|
+
|
|
550
|
+
def factory() -> _Protocol:
|
|
551
|
+
nonlocal protocol
|
|
552
|
+
protocol = _Protocol(limit, loop)
|
|
553
|
+
return protocol
|
|
554
|
+
|
|
555
|
+
cwd = kwds.get("cwd")
|
|
556
|
+
|
|
557
|
+
def in_cwd(path: str) -> bool:
|
|
558
|
+
# The child resolves a relative command against its own folder, so look for it there.
|
|
559
|
+
return os.path.isfile(ntpath.join(os.fspath(cwd), path) if cwd and not ntpath.isabs(path) else path) # type: ignore[arg-type]
|
|
560
|
+
|
|
561
|
+
line = _windows_launch(command, args, env, in_cwd) if _WINDOWS else None
|
|
562
|
+
if line is not None:
|
|
563
|
+
# A str for args is passed to CreateProcess as it is, which neither subprocess_exec (it
|
|
564
|
+
# quotes each argument) nor subprocess_shell (no /d) allows: make the transport as
|
|
565
|
+
# subprocess_exec does, from what it would have built.
|
|
566
|
+
executable, command_line = _cmd_exe(line, env)
|
|
567
|
+
stdio = [kwds.pop(name) for name in ("stdin", "stdout", "stderr")]
|
|
568
|
+
transport = await loop._make_subprocess_transport( # type: ignore[attr-defined]
|
|
569
|
+
factory(), command_line, False, *stdio, 0, executable=executable, env=env, **kwds
|
|
570
|
+
)
|
|
571
|
+
else:
|
|
572
|
+
transport, _ = await loop.subprocess_exec(factory, command, *args, env=env, **kwds) # type: ignore[arg-type]
|
|
573
|
+
assert protocol is not None
|
|
574
|
+
return asyncio.subprocess.Process(transport, protocol, loop), protocol.stdout_closed
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
async def spawn_process(
|
|
578
|
+
command: str,
|
|
579
|
+
*,
|
|
580
|
+
args: Sequence[str] = (),
|
|
581
|
+
cwd: str | os.PathLike[str] | None = None,
|
|
582
|
+
env: Mapping[str, str] | None = None,
|
|
583
|
+
stderr_tail_bytes: int = 16 * 1024,
|
|
584
|
+
kill_grace_ms: float = 2000,
|
|
585
|
+
on_stderr: Callable[[str], None] | None = None,
|
|
586
|
+
) -> SupervisedProcess:
|
|
587
|
+
"""Start `command` with `args`. `env` is added to (not replacing) this process's environment."""
|
|
588
|
+
full_env = {**os.environ, **(env or {})}
|
|
589
|
+
extra: dict[str, object] = {}
|
|
590
|
+
if _WINDOWS:
|
|
591
|
+
extra["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined]
|
|
592
|
+
else:
|
|
593
|
+
extra["start_new_session"] = True
|
|
594
|
+
try:
|
|
595
|
+
proc, stdout_closed = await _start(
|
|
596
|
+
command,
|
|
597
|
+
args,
|
|
598
|
+
2**20,
|
|
599
|
+
full_env,
|
|
600
|
+
cwd=cwd,
|
|
601
|
+
stdin=asyncio.subprocess.PIPE,
|
|
602
|
+
stdout=asyncio.subprocess.PIPE,
|
|
603
|
+
stderr=asyncio.subprocess.PIPE,
|
|
604
|
+
**extra,
|
|
605
|
+
)
|
|
606
|
+
except FileNotFoundError as error:
|
|
607
|
+
raise SpawnError(command, "ENOENT", _not_found_hint(command, cwd, _WINDOWS)) from error
|
|
608
|
+
except PermissionError as error:
|
|
609
|
+
raise SpawnError(command, "EACCES", f'"{command}" is not executable.') from error
|
|
610
|
+
except OSError as error:
|
|
611
|
+
raise SpawnError(command, getattr(error, "errno", None) and str(error.errno), str(error)) from error
|
|
612
|
+
return SupervisedProcess(
|
|
613
|
+
command, proc, stdout_closed, stderr_tail_bytes=stderr_tail_bytes, kill_grace_ms=kill_grace_ms, on_stderr=on_stderr
|
|
614
|
+
)
|
|
File without changes
|