fancy-subprocess 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
from contextlib import AbstractContextManager
|
|
6
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from types import TracebackType
|
|
10
|
+
from typing import Optional
|
|
11
|
+
|
|
12
|
+
import oslex
|
|
13
|
+
import subprocess
|
|
14
|
+
if sys.platform=='win32':
|
|
15
|
+
from ntstatus import NtStatus, NtStatusSeverity, ThirtyTwoBits
|
|
16
|
+
else:
|
|
17
|
+
import signal
|
|
18
|
+
|
|
19
|
+
def which(name: str, *, path: Optional[str | Sequence[str | Path]] = None, cwd: Optional[str | Path] = None) -> Optional[Path]:
|
|
20
|
+
"""
|
|
21
|
+
Wrapper for `shutil.which()` which returns the result as an absolute `Path` (or `None` if it fails to find the executable). It also has a couple extra features, see below.
|
|
22
|
+
|
|
23
|
+
Arguments (all of them except `name` are optional):
|
|
24
|
+
- `name: str` - Executable name to look up.
|
|
25
|
+
- `path: None | str | Sequence[str | Path]` - Directory list to look up `name` in. If set to `None`, or set to a string, then it is passed to `shutil.which()` as-is. If set to a list, concatenates the list items using `os.pathsep`, and passes the result to `shutil.which()`. Defaults to `None`. See `shutil.which()`'s documentation on exact behaviour of this argument.
|
|
26
|
+
- `cwd: Optional[str | Path]` - If specified, then changes the current working directory to `cwd` for the duration of the `shutil.which()` call. Note that since it is changing global state (the current working directory), it is inherently not thread-safe.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
if path is not None and not isinstance(path, str):
|
|
30
|
+
path = os.pathsep.join(str(d) for d in path)
|
|
31
|
+
|
|
32
|
+
old_cwd = Path.cwd()
|
|
33
|
+
if cwd is not None:
|
|
34
|
+
os.chdir(cwd)
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
result = shutil.which(name, path=path)
|
|
38
|
+
finally:
|
|
39
|
+
if cwd is not None:
|
|
40
|
+
os.chdir(old_cwd)
|
|
41
|
+
|
|
42
|
+
if result is not None:
|
|
43
|
+
return Path(result).absolute()
|
|
44
|
+
else:
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
def checked_which(name: str, *, path: Optional[str | Sequence[str | Path]] = None, cwd: Optional[str | Path] = None) -> Path:
|
|
48
|
+
"""
|
|
49
|
+
Same as `fancy_subprocess.which()`, except it raises `ValueError` instead of returning `None` if it cannot find the executable.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
result = which(name, path=path, cwd=cwd)
|
|
53
|
+
if result is not None:
|
|
54
|
+
return result
|
|
55
|
+
else:
|
|
56
|
+
raise ValueError(f'Could not find executable in PATH: "{name}"')
|
|
57
|
+
|
|
58
|
+
def _oslex_join(cmd: Sequence[str | Path]) -> str:
|
|
59
|
+
return oslex.join([str(arg) for arg in cmd])
|
|
60
|
+
|
|
61
|
+
def _stringify_exit_code(exit_code: int) -> Optional[str]:
|
|
62
|
+
if sys.platform=='win32':
|
|
63
|
+
# Windows
|
|
64
|
+
try:
|
|
65
|
+
bits = ThirtyTwoBits(exit_code)
|
|
66
|
+
except ValueError:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
code = NtStatus(bits)
|
|
71
|
+
if code.severity!=NtStatusSeverity.STATUS_SEVERITY_SUCCESS:
|
|
72
|
+
return code.name
|
|
73
|
+
except ValueError:
|
|
74
|
+
pass
|
|
75
|
+
|
|
76
|
+
return f'0x{bits.unsigned_value:08X}'
|
|
77
|
+
else:
|
|
78
|
+
# POSIX
|
|
79
|
+
if exit_code<0:
|
|
80
|
+
try:
|
|
81
|
+
return signal.Signals(-exit_code).name
|
|
82
|
+
except ValueError:
|
|
83
|
+
return 'unknown signal'
|
|
84
|
+
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
class AnyExitCode:
|
|
88
|
+
"""
|
|
89
|
+
Use an instance of this class (eg. fancy_subprocess.ANY_EXIT_CODE) as the 'success' argument to make run() and related functions treat any exit code as success.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
pass
|
|
93
|
+
|
|
94
|
+
ANY_EXIT_CODE = AnyExitCode()
|
|
95
|
+
|
|
96
|
+
@dataclass(kw_only=True, frozen=True)
|
|
97
|
+
class RunProcessResult:
|
|
98
|
+
"""
|
|
99
|
+
`fancy_subprocess.run()` and similar functions return a `RunProcessResult` instance on success.
|
|
100
|
+
|
|
101
|
+
`RunProcessResult` has the following properties:
|
|
102
|
+
- `exit_code: int` - Exit code of the finished process. (On Windows, this is a signed `int32` value, i.e. in the range of \[-2<sup>31</sup>, 2<sup>31</sup>-1\].)
|
|
103
|
+
- `output: str` - Combination of the process's output on stdout and stderr.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
exit_code: int
|
|
107
|
+
output: str
|
|
108
|
+
|
|
109
|
+
@dataclass(kw_only=True, frozen=True)
|
|
110
|
+
class RunProcessError(Exception):
|
|
111
|
+
"""
|
|
112
|
+
`fancy_subprocess.run()` and similar functions raise `RunProcessError` on error. There are two kinds of errors that result in a `RunProcessError`:
|
|
113
|
+
- If the requested command has failed, the `completed` property will be `True`, and the `exit_code` and `output` properties will be set.
|
|
114
|
+
- If the command couldn't be run (eg. because the executable wasn't found), the `completed` property will be `False`, and the `oserror` property will be set to the `OSError` exception instance originally raised by the underlying `subprocess.Popen()` call.
|
|
115
|
+
|
|
116
|
+
Calling `str()` on a `RunProcessError` object returns a detailed one-line description of the error:
|
|
117
|
+
- The failed command is included in the message.
|
|
118
|
+
- If an `OSError` happened, its message is included in the message.
|
|
119
|
+
- On Windows, if the exit code of the process is recognized as a known `NTSTATUS` error value, its name is included in the message, otherwise its hexadecimal representation is included (to make searching it on the internet easier).
|
|
120
|
+
- On Unix systems, if the exit code represents a signal, its name is included in the message.
|
|
121
|
+
|
|
122
|
+
`RunProcessError` has the following properties:
|
|
123
|
+
- `cmd: Sequence[str | Path]` - Original command passed to `fancy_subprocess.run()`.
|
|
124
|
+
- `completed: bool` - `True` if the process completed (with an error), `False` if the underlying `subprocess.Popen()` call raised an OSError (eg. because it could not start the process).
|
|
125
|
+
- `exit_code: int` - Exit code of the completed process. Raises `ValueError` if `completed` is `False`.
|
|
126
|
+
- `output: str` - Combination of the process's output on stdout and stderr. Raises `ValueError` if `completed` is `False`.
|
|
127
|
+
- `oserror: OSError` - The `OSError` raised by `subprocess.Popen()`. Raises `ValueError` if `completed` is `True`.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
cmd: Sequence[str | Path]
|
|
131
|
+
result: RunProcessResult | OSError
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def completed(self) -> bool:
|
|
135
|
+
return isinstance(self.result, RunProcessResult)
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def exit_code(self) -> int:
|
|
139
|
+
if isinstance(self.result, RunProcessResult):
|
|
140
|
+
return self.result.exit_code
|
|
141
|
+
else:
|
|
142
|
+
raise ValueError('...')
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def output(self) -> str:
|
|
146
|
+
if isinstance(self.result, RunProcessResult):
|
|
147
|
+
return self.result.output
|
|
148
|
+
else:
|
|
149
|
+
raise ValueError('...')
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def oserror(self) -> OSError:
|
|
153
|
+
if isinstance(self.result, OSError):
|
|
154
|
+
return self.result
|
|
155
|
+
else:
|
|
156
|
+
raise ValueError('...')
|
|
157
|
+
|
|
158
|
+
def __str__(self) -> str:
|
|
159
|
+
if isinstance(self.result, RunProcessResult):
|
|
160
|
+
exit_code_str = _stringify_exit_code(self.exit_code)
|
|
161
|
+
if exit_code_str is not None:
|
|
162
|
+
exit_code_comment = f' ({exit_code_str})'
|
|
163
|
+
else:
|
|
164
|
+
exit_code_comment = ''
|
|
165
|
+
return f'Command failed with exit code {self.exit_code}{exit_code_comment}: {_oslex_join(self.cmd)}'
|
|
166
|
+
else:
|
|
167
|
+
return f'Exception {type(self.result).__name__} with message "{str(self.result)}" was raised while trying to run command: {_oslex_join(self.cmd)}'
|
|
168
|
+
|
|
169
|
+
def SILENCE(msg: str) -> None:
|
|
170
|
+
"""
|
|
171
|
+
Helper function that takes a string, and does nothing with it. Meant to be passed as the print_message or print_output argument of run() and related functions to silence the corresponding output stream.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
pass
|
|
175
|
+
|
|
176
|
+
def run(
|
|
177
|
+
cmd: Sequence[str | Path],
|
|
178
|
+
*,
|
|
179
|
+
print_message: Optional[Callable[[str], None]] = None,
|
|
180
|
+
print_output: Optional[Callable[[str], None]] = None,
|
|
181
|
+
description: Optional[str] = None,
|
|
182
|
+
success: Sequence[int] | AnyExitCode | None = None,
|
|
183
|
+
flush_before_subprocess: bool = True,
|
|
184
|
+
max_output_size: int = 10*1000*1000,
|
|
185
|
+
retry: int = 0,
|
|
186
|
+
retry_initial_sleep_seconds: float = 10,
|
|
187
|
+
retry_backoff: float = 2,
|
|
188
|
+
env_overrides: Optional[Mapping[str, str]] = None,
|
|
189
|
+
cwd: Optional[str | Path] = None,
|
|
190
|
+
encoding: Optional[str] = None,
|
|
191
|
+
errors: Optional[str] = None,
|
|
192
|
+
) -> RunProcessResult:
|
|
193
|
+
"""
|
|
194
|
+
An extended (and in some aspects, constrained) version of `subprocess.run()`. It runs a command and prints its output line-by-line using a customizable `print_output` function, while printing informational messages (eg. which command it is running) using a customizable `print_message` function.
|
|
195
|
+
|
|
196
|
+
Key differences compared to `subprocess.run()`:
|
|
197
|
+
- The command must be specified as a list, simply specifying a string is not allowed.
|
|
198
|
+
- The command's stdout and stderr is always combined into a single stream. (Like `subprocess.run(stderr=STDOUT)`.)
|
|
199
|
+
- The output of the command is always assumed to be textual, not binary. (Like `subprocess.run(text=True)`.)
|
|
200
|
+
- The output of the command is always captured, but it is also immediately printed using `print_output`.
|
|
201
|
+
- The exit code of the command is checked, and an exception is raised on failure, like `subprocess.run(check=True)`, but the list of exit codes treated as success is customizable, and the raised exception is `RunProcessError` instead of `CalledProcessError`.
|
|
202
|
+
- `OSError` is never raised, it gets converted to `RunProcessError`.
|
|
203
|
+
- `RunProcessResult` is returned instead of `CompletedProcess` on success.
|
|
204
|
+
|
|
205
|
+
Arguments (all of them except `cmd` are optional):
|
|
206
|
+
- `cmd: Sequence[str | Path]` - Command to run. See `subprocess.run()`'s documentation for the interpretation of `cmd[0]`. It is recommended to use `fancy_subprocess.which()` to produce `cmd[0]`.
|
|
207
|
+
- `print_message: Optional[Callable[[str], None]]` - Function used to print informational messages. If not set or `None`, defaults to `print(flush=True)`. Use `print_message=fancy_subprocess.SILENCE` to disable printing informational messages.
|
|
208
|
+
- `print_output: Optional[Callable[[str], None]]` - Function used to print a line of the output of the command. If not set or `None`, defaults to `print(flush=True)`. Use `print_message=fancy_subprocess.SILENCE` to disable printing the command's output.
|
|
209
|
+
- `description: Optional[str]` - Description printed before running the command. If not set or `None`, defaults to `Running command: ...`.
|
|
210
|
+
- `success: Sequence[int] | AnyExitCode | None` - List of exit codes that should be considered successful. If set to `fancy_subprocess.ANY_EXIT_CODE`, then all exit codes are considered successful. If not set or `None`, defaults to `[0]`. Note that 0 is not automatically included in the list of successful exit codes, so if a list without 0 is specified, then the function will consider 0 a failure.
|
|
211
|
+
- `flush_before_subprocess: bool` - If `True`, flushes both the standard output and error streams before running the command. Defaults to `True`.
|
|
212
|
+
- `max_output_size: int` - Maximum number of characters to be recorded in the `output` field of `RunProcessResult`. If the command produces more than `max_output_size` characters, only the last `max_output_size` will be recorded. Defaults to 10,000,000.
|
|
213
|
+
- `retry: int` - Number of times to retry running the command on failure. Note that the total number of attempts is one greater than what's specified. (I.e. `retry=2` attempts to run the command 3 times.) Defaults to 0.
|
|
214
|
+
- `retry_initial_sleep_seconds: float` - Number of seconds to wait before retrying for the first time. Defaults to 10.
|
|
215
|
+
- `retry_backoff: float` - Factor used to increase wait times before subsequent retries. Defaults to 2.
|
|
216
|
+
- `env_overrides: Optional[Mapping[str, str]]` - Dictionary used to set environment variables. Note that unline the `env` argument of `subprocess.run()`, `env_overrides` does not need to contain all environment variables, only the ones you want to add/modify compared to os.environ.
|
|
217
|
+
- `cwd: Optional[str | Path]` - If not `None`, change current working directory to `cwd` before running the command.
|
|
218
|
+
- `encoding: Optional[str]` - This encoding will be used to open stdout and stderr of the command. If not set or `None`, see default behaviour in `io.TextIOWrapper`'s documentation.
|
|
219
|
+
- `errors: Optional[str]` - This specifies how text decoding errors will be handled. See details in `io.TextIOWrapper`'s documentation.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
if print_message is None:
|
|
223
|
+
print_message = lambda msg: print(msg, flush=True)
|
|
224
|
+
|
|
225
|
+
if print_output is None:
|
|
226
|
+
print_output = lambda line: print(line, flush=True)
|
|
227
|
+
|
|
228
|
+
if description is None:
|
|
229
|
+
description = f'Running command: {_oslex_join(cmd)}'
|
|
230
|
+
|
|
231
|
+
if success is None:
|
|
232
|
+
success = [0]
|
|
233
|
+
|
|
234
|
+
env = dict(os.environ)
|
|
235
|
+
if env_overrides is not None:
|
|
236
|
+
if sys.platform=='win32':
|
|
237
|
+
env.update((key.upper(), value) for key,value in env_overrides.items())
|
|
238
|
+
else:
|
|
239
|
+
env.update(env_overrides)
|
|
240
|
+
|
|
241
|
+
def run_with_params() -> RunProcessResult:
|
|
242
|
+
return _run_internal(
|
|
243
|
+
cmd,
|
|
244
|
+
print_message=print_message,
|
|
245
|
+
print_output=print_output,
|
|
246
|
+
description=description,
|
|
247
|
+
success=success,
|
|
248
|
+
flush_before_subprocess=flush_before_subprocess,
|
|
249
|
+
max_output_size=max_output_size,
|
|
250
|
+
env=env,
|
|
251
|
+
cwd=cwd,
|
|
252
|
+
encoding=encoding,
|
|
253
|
+
errors=errors)
|
|
254
|
+
|
|
255
|
+
sleep_seconds = retry_initial_sleep_seconds
|
|
256
|
+
for attempts_left in range(retry, 0, -1):
|
|
257
|
+
try:
|
|
258
|
+
return run_with_params()
|
|
259
|
+
except RunProcessError as e:
|
|
260
|
+
print_message(str(e))
|
|
261
|
+
if attempts_left!=1:
|
|
262
|
+
plural = 's'
|
|
263
|
+
else:
|
|
264
|
+
plural = ''
|
|
265
|
+
print_message(f'Retrying in {sleep_seconds} seconds ({attempts_left} attempt{plural} left)...')
|
|
266
|
+
time.sleep(sleep_seconds)
|
|
267
|
+
sleep_seconds *= retry_backoff
|
|
268
|
+
|
|
269
|
+
return run_with_params()
|
|
270
|
+
|
|
271
|
+
def _run_internal(
|
|
272
|
+
cmd: Sequence[str | Path],
|
|
273
|
+
*,
|
|
274
|
+
print_message: Callable[[str], None],
|
|
275
|
+
print_output: Callable[[str], None],
|
|
276
|
+
description: str,
|
|
277
|
+
success: Sequence[int] | AnyExitCode,
|
|
278
|
+
flush_before_subprocess: bool,
|
|
279
|
+
max_output_size: int,
|
|
280
|
+
env: dict[str, str],
|
|
281
|
+
cwd: Optional[str | Path],
|
|
282
|
+
encoding: Optional[str],
|
|
283
|
+
errors: Optional[str],
|
|
284
|
+
) -> RunProcessResult:
|
|
285
|
+
print_message(description)
|
|
286
|
+
|
|
287
|
+
if flush_before_subprocess:
|
|
288
|
+
sys.stdout.flush()
|
|
289
|
+
sys.stderr.flush()
|
|
290
|
+
|
|
291
|
+
output = ''
|
|
292
|
+
try:
|
|
293
|
+
with subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, cwd=cwd, env=env, encoding=encoding, errors=errors) as proc:
|
|
294
|
+
assert proc.stdout is not None # passing stdout=subprocess.PIPE guarantees this
|
|
295
|
+
|
|
296
|
+
for line in iter(proc.stdout.readline, ''):
|
|
297
|
+
line = line.removesuffix('\n')
|
|
298
|
+
print_output(line)
|
|
299
|
+
|
|
300
|
+
output += line + '\n'
|
|
301
|
+
if len(output)>max_output_size+1:
|
|
302
|
+
output = output[-max_output_size-1:] # drop the beginning of the string
|
|
303
|
+
|
|
304
|
+
proc.wait()
|
|
305
|
+
result = RunProcessResult(exit_code=proc.returncode, output=output.removesuffix('\n'))
|
|
306
|
+
except OSError as e:
|
|
307
|
+
raise RunProcessError(cmd=cmd, result=e) from e
|
|
308
|
+
|
|
309
|
+
if isinstance(success, AnyExitCode) or result.exit_code in success:
|
|
310
|
+
return result
|
|
311
|
+
else:
|
|
312
|
+
raise RunProcessError(cmd=cmd, result=result)
|
|
313
|
+
|
|
314
|
+
def run_indented(
|
|
315
|
+
cmd: Sequence[str | Path],
|
|
316
|
+
*,
|
|
317
|
+
print_message: Optional[Callable[[str], None]] = None,
|
|
318
|
+
indent: str | int = 4,
|
|
319
|
+
description: Optional[str] = None,
|
|
320
|
+
success: Sequence[int] | AnyExitCode | None = None,
|
|
321
|
+
flush_before_subprocess: bool = True,
|
|
322
|
+
max_output_size: int = 10*1000*1000*1000,
|
|
323
|
+
retry: int = 0,
|
|
324
|
+
retry_initial_sleep_seconds: float = 10,
|
|
325
|
+
retry_backoff: float = 2,
|
|
326
|
+
env_overrides: Optional[Mapping[str, str]] = None,
|
|
327
|
+
cwd: Optional[str | Path] = None,
|
|
328
|
+
encoding: Optional[str] = None,
|
|
329
|
+
errors: Optional[str] = None,
|
|
330
|
+
) -> RunProcessResult:
|
|
331
|
+
"""
|
|
332
|
+
Specialized version of `fancy_subprocess.run()` which prints the command's output indented by a user-defined amount.
|
|
333
|
+
|
|
334
|
+
The `print_output` argument is replaced by `indent`, which can be set to either the number of spaces to use for indentation or any custom indentation string (eg. `\t`).
|
|
335
|
+
|
|
336
|
+
All other `fancy_subprocess.run()` arguments are available and behave the same.
|
|
337
|
+
"""
|
|
338
|
+
|
|
339
|
+
if isinstance(indent, int):
|
|
340
|
+
indent = indent*' '
|
|
341
|
+
|
|
342
|
+
return run(
|
|
343
|
+
cmd,
|
|
344
|
+
print_message=print_message,
|
|
345
|
+
print_output=lambda line: print(f'{indent}{line}', flush=True),
|
|
346
|
+
description=description,
|
|
347
|
+
success=success,
|
|
348
|
+
flush_before_subprocess=flush_before_subprocess,
|
|
349
|
+
max_output_size=max_output_size,
|
|
350
|
+
retry=retry,
|
|
351
|
+
retry_initial_sleep_seconds=retry_initial_sleep_seconds,
|
|
352
|
+
retry_backoff=retry_backoff,
|
|
353
|
+
env_overrides=env_overrides,
|
|
354
|
+
cwd=cwd,
|
|
355
|
+
encoding=encoding,
|
|
356
|
+
errors=errors,
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
def run_silenced(
|
|
360
|
+
cmd: Sequence[str | Path],
|
|
361
|
+
*,
|
|
362
|
+
print_message: Optional[Callable[[str], None]] = None,
|
|
363
|
+
description: Optional[str] = None,
|
|
364
|
+
success: Sequence[int] | AnyExitCode | None = None,
|
|
365
|
+
flush_before_subprocess: bool = True,
|
|
366
|
+
max_output_size: int = 10*1000*1000*1000,
|
|
367
|
+
retry: int = 0,
|
|
368
|
+
retry_initial_sleep_seconds: float = 10,
|
|
369
|
+
retry_backoff: float = 2,
|
|
370
|
+
env_overrides: Optional[Mapping[str, str]] = None,
|
|
371
|
+
cwd: Optional[str | Path] = None,
|
|
372
|
+
encoding: Optional[str] = None,
|
|
373
|
+
errors: Optional[str] = None,
|
|
374
|
+
) -> RunProcessResult:
|
|
375
|
+
"""
|
|
376
|
+
Specialized version of `fancy_subprocess.run()`, primarily used to run a command and later process its output.
|
|
377
|
+
|
|
378
|
+
Differences from `fancy_subprocess.run()`:
|
|
379
|
+
- `print_output` is not customizable, it is always set to `fancy_subprocess.SILENCE`, which disables printing the command's output.
|
|
380
|
+
- `description` is customizable, but its default value (used when it is either not specified or set to `None`) changes to `Running command (output silenced): ...`.
|
|
381
|
+
|
|
382
|
+
All other `fancy_subprocess.run()` arguments are available and behave the same.
|
|
383
|
+
"""
|
|
384
|
+
|
|
385
|
+
if description is None:
|
|
386
|
+
description = f'Running command (output silenced): {_oslex_join(cmd)}'
|
|
387
|
+
|
|
388
|
+
return run(
|
|
389
|
+
cmd,
|
|
390
|
+
print_message=print_message,
|
|
391
|
+
print_output=SILENCE,
|
|
392
|
+
description=description,
|
|
393
|
+
success=success,
|
|
394
|
+
flush_before_subprocess=flush_before_subprocess,
|
|
395
|
+
max_output_size=max_output_size,
|
|
396
|
+
retry=retry,
|
|
397
|
+
retry_initial_sleep_seconds=retry_initial_sleep_seconds,
|
|
398
|
+
retry_backoff=retry_backoff,
|
|
399
|
+
env_overrides=env_overrides,
|
|
400
|
+
cwd=cwd,
|
|
401
|
+
encoding=encoding,
|
|
402
|
+
errors=errors,
|
|
403
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fancy_subprocess
|
|
3
|
+
Version: 1.0
|
|
4
|
+
Summary: subprocess.run() with formatted output, detailed error messages and retry capabilities
|
|
5
|
+
Project-URL: Homepage, https://github.com/petamas/fancy-subprocess
|
|
6
|
+
Project-URL: Bug Tracker, https://github.com/petamas/fancy-subprocess/issues
|
|
7
|
+
Author-email: Tamás PEREGI <petamas@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: MacOS
|
|
12
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Operating System :: POSIX
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: ntstatus<2
|
|
22
|
+
Requires-Dist: oslex<2
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# fancy-subprocess
|
|
26
|
+
|
|
27
|
+
`fancy-subprocess` provides variants of `subprocess.run()` with formatted output, detailed error messages and retry capabilities.
|
|
28
|
+
|
|
29
|
+
## Package contents
|
|
30
|
+
|
|
31
|
+
### `fancy_subprocess.run()`
|
|
32
|
+
|
|
33
|
+
An extended (and in some aspects, constrained) version of `subprocess.run()`. It runs a command and prints its output line-by-line using a customizable `print_output` function, while printing informational messages (eg. which command it is running) using a customizable `print_message` function.
|
|
34
|
+
|
|
35
|
+
Key differences compared to `subprocess.run()`:
|
|
36
|
+
- The command must be specified as a list, simply specifying a string is not allowed.
|
|
37
|
+
- The command's stdout and stderr is always combined into a single stream. (Like `subprocess.run(stderr=STDOUT)`.)
|
|
38
|
+
- The output of the command is always assumed to be textual, not binary. (Like `subprocess.run(text=True)`.)
|
|
39
|
+
- The output of the command is always captured, but it is also immediately printed using `print_output`.
|
|
40
|
+
- The exit code of the command is checked, and an exception is raised on failure, like `subprocess.run(check=True)`, but the list of exit codes treated as success is customizable, and the raised exception is `RunProcessError` instead of `CalledProcessError`.
|
|
41
|
+
- `OSError` is never raised, it gets converted to `RunProcessError`.
|
|
42
|
+
- `RunProcessResult` is returned instead of `CompletedProcess` on success.
|
|
43
|
+
|
|
44
|
+
Arguments (all of them except `cmd` are optional):
|
|
45
|
+
- `cmd: Sequence[str | Path]` - Command to run. See `subprocess.run()`'s documentation for the interpretation of `cmd[0]`. It is recommended to use `fancy_subprocess.which()` to produce `cmd[0]`.
|
|
46
|
+
- `print_message: Optional[Callable[[str], None]]` - Function used to print informational messages. If not set or `None`, defaults to `print(flush=True)`. Use `print_message=fancy_subprocess.SILENCE` to disable printing informational messages.
|
|
47
|
+
- `print_output: Optional[Callable[[str], None]]` - Function used to print a line of the output of the command. If not set or `None`, defaults to `print(flush=True)`. Use `print_message=fancy_subprocess.SILENCE` to disable printing the command's output.
|
|
48
|
+
- `description: Optional[str]` - Description printed before running the command. If not set or `None`, defaults to `Running command: ...`.
|
|
49
|
+
- `success: Sequence[int] | AnyExitCode | None` - List of exit codes that should be considered successful. If set to `fancy_subprocess.ANY_EXIT_CODE`, then all exit codes are considered successful. If not set or `None`, defaults to `[0]`. Note that 0 is not automatically included in the list of successful exit codes, so if a list without 0 is specified, then the function will consider 0 a failure.
|
|
50
|
+
- `flush_before_subprocess: bool` - If `True`, flushes both the standard output and error streams before running the command. Defaults to `True`.
|
|
51
|
+
- `max_output_size: int` - Maximum number of characters to be recorded in the `output` field of `RunProcessResult`. If the command produces more than `max_output_size` characters, only the last `max_output_size` will be recorded. Defaults to 10,000,000.
|
|
52
|
+
- `retry: int` - Number of times to retry running the command on failure. Note that the total number of attempts is one greater than what's specified. (I.e. `retry=2` attempts to run the command 3 times.) Defaults to 0.
|
|
53
|
+
- `retry_initial_sleep_seconds: float` - Number of seconds to wait before retrying for the first time. Defaults to 10.
|
|
54
|
+
- `retry_backoff: float` - Factor used to increase wait times before subsequent retries. Defaults to 2.
|
|
55
|
+
- `env_overrides: Optional[Mapping[str, str]]` - Dictionary used to set environment variables. Note that unline the `env` argument of `subprocess.run()`, `env_overrides` does not need to contain all environment variables, only the ones you want to add/modify compared to os.environ.
|
|
56
|
+
- `cwd: Optional[str | Path]` - If not `None`, change current working directory to `cwd` before running the command.
|
|
57
|
+
- `encoding: Optional[str]` - This encoding will be used to open stdout and stderr of the command. If not set or `None`, see default behaviour in `io.TextIOWrapper`'s documentation.
|
|
58
|
+
- `errors: Optional[str]` - This specifies how text decoding errors will be handled. See details in `io.TextIOWrapper`'s documentation.
|
|
59
|
+
|
|
60
|
+
#### Return value: `fancy_subprocess.RunProcessResult`
|
|
61
|
+
|
|
62
|
+
`fancy_subprocess.run()` and similar functions return a `RunProcessResult` instance on success.
|
|
63
|
+
|
|
64
|
+
`RunProcessResult` has the following properties:
|
|
65
|
+
- `exit_code: int` - Exit code of the finished process. (On Windows, this is a signed `int32` value, i.e. in the range of \[-2<sup>31</sup>, 2<sup>31</sup>-1\].)
|
|
66
|
+
- `output: str` - Combination of the process's output on stdout and stderr.
|
|
67
|
+
|
|
68
|
+
#### Exception: `fancy_subprocess.RunProcessError`
|
|
69
|
+
|
|
70
|
+
`fancy_subprocess.run()` and similar functions raise `RunProcessError` on error. There are two kinds of errors that result in a `RunProcessError`:
|
|
71
|
+
- If the requested command has failed, the `completed` property will be `True`, and the `exit_code` and `output` properties will be set.
|
|
72
|
+
- If the command couldn't be run (eg. because the executable wasn't found), the `completed` property will be `False`, and the `oserror` property will be set to the `OSError` exception instance originally raised by the underlying `subprocess.Popen()` call.
|
|
73
|
+
|
|
74
|
+
Calling `str()` on a `RunProcessError` object returns a detailed one-line description of the error:
|
|
75
|
+
- The failed command is included in the message.
|
|
76
|
+
- If an `OSError` happened, its message is included in the message.
|
|
77
|
+
- On Windows, if the exit code of the process is recognized as a known `NTSTATUS` error value, its name is included in the message, otherwise its hexadecimal representation is included (to make searching it on the internet easier).
|
|
78
|
+
- On Unix systems, if the exit code represents a signal, its name is included in the message.
|
|
79
|
+
|
|
80
|
+
`RunProcessError` has the following properties:
|
|
81
|
+
- `cmd: Sequence[str | Path]` - Original command passed to `fancy_subprocess.run()`.
|
|
82
|
+
- `completed: bool` - `True` if the process completed (with an error), `False` if the underlying `subprocess.Popen()` call raised an OSError (eg. because it could not start the process).
|
|
83
|
+
- `exit_code: int` - Exit code of the completed process. Raises `ValueError` if `completed` is `False`.
|
|
84
|
+
- `output: str` - Combination of the process's output on stdout and stderr. Raises `ValueError` if `completed` is `False`.
|
|
85
|
+
- `oserror: OSError` - The `OSError` raised by `subprocess.Popen()`. Raises `ValueError` if `completed` is `True`.
|
|
86
|
+
|
|
87
|
+
### `fancy_subprocess.run_silenced()`
|
|
88
|
+
|
|
89
|
+
Specialized version of `fancy_subprocess.run()`, primarily used to run a command and later process its output.
|
|
90
|
+
|
|
91
|
+
Differences from `fancy_subprocess.run()`:
|
|
92
|
+
- `print_output` is not customizable, it is always set to `fancy_subprocess.SILENCE`, which disables printing the command's output.
|
|
93
|
+
- `description` is customizable, but its default value (used when it is either not specified or set to `None`) changes to `Running command (output silenced): ...`.
|
|
94
|
+
|
|
95
|
+
All other `fancy_subprocess.run()` arguments are available and behave the same.
|
|
96
|
+
|
|
97
|
+
### `fancy_subprocess.run_indented()`
|
|
98
|
+
|
|
99
|
+
Specialized version of `fancy_subprocess.run()` which prints the command's output indented by a user-defined amount.
|
|
100
|
+
|
|
101
|
+
The `print_output` argument is replaced by `indent`, which can be set to either the number of spaces to use for indentation or any custom indentation string (eg. `\t`).
|
|
102
|
+
|
|
103
|
+
All other `fancy_subprocess.run()` arguments are available and behave the same.
|
|
104
|
+
|
|
105
|
+
### `fancy_subprocess.which()`
|
|
106
|
+
|
|
107
|
+
Wrapper for `shutil.which()` which returns the result as an absolute `Path` (or `None` if it fails to find the executable). It also has a couple extra features, see below.
|
|
108
|
+
|
|
109
|
+
Arguments (all of them except `name` are optional):
|
|
110
|
+
- `name: str` - Executable name to look up.
|
|
111
|
+
- `path: None | str | Sequence[str | Path]` - Directory list to look up `name` in. If set to `None`, or set to a string, then it is passed to `shutil.which()` as-is. If set to a list, concatenates the list items using `os.pathsep`, and passes the result to `shutil.which()`. Defaults to `None`. See `shutil.which()`'s documentation on exact behaviour of this argument.
|
|
112
|
+
- `cwd: Optional[str | Path]` - If specified, then changes the current working directory to `cwd` for the duration of the `shutil.which()` call. Note that since it is changing global state (the current working directory), it is inherently not thread-safe.
|
|
113
|
+
|
|
114
|
+
### `fancy_subprocess.checked_which()`
|
|
115
|
+
|
|
116
|
+
Same as `fancy_subprocess.which()`, except it raises `ValueError` instead of returning `None` if it cannot find the executable.
|
|
117
|
+
|
|
118
|
+
## Examples
|
|
119
|
+
|
|
120
|
+
### Success
|
|
121
|
+
|
|
122
|
+
Take this script:
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
import fancy_subprocess
|
|
126
|
+
import sys
|
|
127
|
+
|
|
128
|
+
fancy_subprocess.run_indented(
|
|
129
|
+
[sys.executable, '-m', 'venv', '--help'],
|
|
130
|
+
print_message=lambda msg: print(f'[script-name] {msg}'),
|
|
131
|
+
success=fancy_subprocess.ANY_EXIT_CODE)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Running the script will produce the following output (on Windows):
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
[script-name] Running command: d:\projects\python-libs\fancy_subprocess\.venv\Scripts\python.exe -m venv --help
|
|
138
|
+
usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear]
|
|
139
|
+
[--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps]
|
|
140
|
+
ENV_DIR [ENV_DIR ...]
|
|
141
|
+
|
|
142
|
+
Creates virtual Python environments in one or more target directories.
|
|
143
|
+
|
|
144
|
+
positional arguments:
|
|
145
|
+
ENV_DIR A directory to create the environment in.
|
|
146
|
+
|
|
147
|
+
options:
|
|
148
|
+
-h, --help show this help message and exit
|
|
149
|
+
--system-site-packages
|
|
150
|
+
Give the virtual environment access to the system
|
|
151
|
+
site-packages dir.
|
|
152
|
+
--symlinks Try to use symlinks rather than copies, when symlinks
|
|
153
|
+
are not the default for the platform.
|
|
154
|
+
--copies Try to use copies rather than symlinks, even when
|
|
155
|
+
symlinks are the default for the platform.
|
|
156
|
+
--clear Delete the contents of the environment directory if it
|
|
157
|
+
already exists, before environment creation.
|
|
158
|
+
--upgrade Upgrade the environment directory to use this version
|
|
159
|
+
of Python, assuming Python has been upgraded in-place.
|
|
160
|
+
--without-pip Skips installing or upgrading pip in the virtual
|
|
161
|
+
environment (pip is bootstrapped by default)
|
|
162
|
+
--prompt PROMPT Provides an alternative prompt prefix for this
|
|
163
|
+
environment.
|
|
164
|
+
--upgrade-deps Upgrade core dependencies: pip setuptools to the
|
|
165
|
+
latest version in PyPI
|
|
166
|
+
|
|
167
|
+
Once an environment has been created, you may wish to activate it, e.g. by
|
|
168
|
+
sourcing an activate script in its bin directory.
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
### Failed command on Windows
|
|
173
|
+
|
|
174
|
+
Take this script:
|
|
175
|
+
|
|
176
|
+
```
|
|
177
|
+
import fancy_subprocess
|
|
178
|
+
import sys
|
|
179
|
+
|
|
180
|
+
try:
|
|
181
|
+
fancy_subprocess.run(
|
|
182
|
+
[sys.executable, '-c', 'import sys; print("Noooooo!"); sys.exit(-1072103376)'],
|
|
183
|
+
description='Demonstrating failure...',
|
|
184
|
+
)
|
|
185
|
+
except fancy_subprocess.RunProcessError as e:
|
|
186
|
+
print(e)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Running the script on Windows will produce the following output (-1072103376 is the signed integer interpretation of 0xC0190030, i.e. `STATUS_LOG_CORRUPTION_DETECTED`):
|
|
190
|
+
|
|
191
|
+
```
|
|
192
|
+
Demonstrating failure...
|
|
193
|
+
Noooooo!
|
|
194
|
+
Command failed with exit code -1072103376 (STATUS_LOG_CORRUPTION_DETECTED): d:\projects\python-libs\fancy_subprocess\.venv\Scripts\python.exe -c "import sys; print("\^"Noooooo^!\^""); sys.exit(-1072103376)"
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Killed command on Linux
|
|
198
|
+
|
|
199
|
+
Take this script:
|
|
200
|
+
|
|
201
|
+
```
|
|
202
|
+
import fancy_subprocess
|
|
203
|
+
import sys
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
fancy_subprocess.run_silenced(
|
|
207
|
+
[sys.executable, '-c', 'import time; time.sleep(60)'],
|
|
208
|
+
description='Sweet dreams!',
|
|
209
|
+
)
|
|
210
|
+
except fancy_subprocess.RunProcessError as e:
|
|
211
|
+
print(e)
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Running the script on Linux and killing the subprocess using `kill -9` before the 60 seconds are up will result in the following output:
|
|
215
|
+
|
|
216
|
+
```
|
|
217
|
+
Sweet dreams!
|
|
218
|
+
Command failed with exit code -9 (SIGKILL): /home/petamas/.venv/bin/python -c 'import time; time.sleep(60)'
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
### Failure to find executable
|
|
222
|
+
|
|
223
|
+
Take this script:
|
|
224
|
+
|
|
225
|
+
```
|
|
226
|
+
import fancy_subprocess
|
|
227
|
+
|
|
228
|
+
try:
|
|
229
|
+
fancy_subprocess.run(['foo', '--bar', 'baz'])
|
|
230
|
+
except fancy_subprocess.RunProcessError as e:
|
|
231
|
+
print(e)
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Running the script will produce the following output (exact error message may depend on OS):
|
|
235
|
+
|
|
236
|
+
```
|
|
237
|
+
Running command: foo --bar baz
|
|
238
|
+
Exception FileNotFoundError with message "[Errno 2] No such file or directory: 'foo'" was raised while trying to run command: foo --bar baz
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## Licensing
|
|
242
|
+
|
|
243
|
+
This library is licensed under the MIT license.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
fancy_subprocess/__init__.py,sha256=VN2qrBSnk74KNSe98hCmrr0RRuUAedYU-MCxLs-mU8E,18925
|
|
2
|
+
fancy_subprocess/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
fancy_subprocess-1.0.dist-info/METADATA,sha256=JdNVQBABMrGO42uxgGL3guO2owFaPiKCUZLlZzSoBMU,13575
|
|
4
|
+
fancy_subprocess-1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
5
|
+
fancy_subprocess-1.0.dist-info/licenses/LICENSE,sha256=ZXB8tGHQGN6AIY-xxMUcN60DM-WA5XusmM6JlArTpm0,1091
|
|
6
|
+
fancy_subprocess-1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Tamás PEREGI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|