dycw-utilities 0.174.12__py3-none-any.whl → 0.175.31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dycw-utilities might be problematic. Click here for more details.
- {dycw_utilities-0.174.12.dist-info → dycw_utilities-0.175.31.dist-info}/METADATA +4 -11
- {dycw_utilities-0.174.12.dist-info → dycw_utilities-0.175.31.dist-info}/RECORD +18 -19
- {dycw_utilities-0.174.12.dist-info → dycw_utilities-0.175.31.dist-info}/WHEEL +1 -1
- {dycw_utilities-0.174.12.dist-info → dycw_utilities-0.175.31.dist-info}/entry_points.txt +0 -2
- utilities/__init__.py +1 -1
- utilities/altair.py +3 -1
- utilities/click.py +1 -2
- utilities/docker.py +117 -28
- utilities/functions.py +1 -1
- utilities/hypothesis.py +1 -1
- utilities/importlib.py +17 -1
- utilities/pathlib.py +1 -1
- utilities/permissions.py +1 -0
- utilities/platform.py +1 -1
- utilities/polars.py +2 -0
- utilities/pytest_regressions.py +25 -5
- utilities/subprocess.py +1191 -108
- utilities/tempfile.py +49 -16
- utilities/aeventkit.py +0 -389
utilities/subprocess.py
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import shutil
|
|
3
4
|
import sys
|
|
4
5
|
from contextlib import contextmanager
|
|
5
6
|
from dataclasses import dataclass
|
|
6
7
|
from io import StringIO
|
|
8
|
+
from itertools import repeat
|
|
7
9
|
from pathlib import Path
|
|
10
|
+
from re import MULTILINE, search
|
|
8
11
|
from shlex import join
|
|
12
|
+
from shutil import copyfile, copytree, move, rmtree
|
|
9
13
|
from string import Template
|
|
10
14
|
from subprocess import PIPE, CalledProcessError, Popen
|
|
11
15
|
from threading import Thread
|
|
@@ -15,13 +19,18 @@ from typing import IO, TYPE_CHECKING, Literal, assert_never, overload, override
|
|
|
15
19
|
from utilities.errors import ImpossibleCaseError
|
|
16
20
|
from utilities.iterables import always_iterable
|
|
17
21
|
from utilities.logging import to_logger
|
|
22
|
+
from utilities.pathlib import PWD
|
|
23
|
+
from utilities.permissions import Permissions, ensure_perms
|
|
24
|
+
from utilities.tempfile import TemporaryDirectory
|
|
18
25
|
from utilities.text import strip_and_dedent
|
|
19
|
-
from utilities.whenever import to_seconds
|
|
26
|
+
from utilities.whenever import SECOND, to_seconds
|
|
20
27
|
|
|
21
28
|
if TYPE_CHECKING:
|
|
22
|
-
from collections.abc import Iterator
|
|
29
|
+
from collections.abc import Callable, Iterator
|
|
23
30
|
|
|
31
|
+
from utilities.permissions import PermissionsLike
|
|
24
32
|
from utilities.types import (
|
|
33
|
+
Delta,
|
|
25
34
|
LoggerLike,
|
|
26
35
|
MaybeIterable,
|
|
27
36
|
PathLike,
|
|
@@ -35,6 +44,9 @@ _HOST_KEY_ALGORITHMS = ["ssh-ed25519"]
|
|
|
35
44
|
APT_UPDATE = ["apt", "update", "-y"]
|
|
36
45
|
BASH_LC = ["bash", "-lc"]
|
|
37
46
|
BASH_LS = ["bash", "-ls"]
|
|
47
|
+
CHPASSWD = "chpasswd"
|
|
48
|
+
GIT_BRANCH_SHOW_CURRENT = ["git", "branch", "--show-current"]
|
|
49
|
+
KNOWN_HOSTS = Path.home() / ".ssh/known_hosts"
|
|
38
50
|
MKTEMP_DIR_CMD = ["mktemp", "-d"]
|
|
39
51
|
RESTART_SSHD = ["systemctl", "restart", "sshd"]
|
|
40
52
|
UPDATE_CA_CERTIFICATES: str = "update-ca-certificates"
|
|
@@ -43,45 +55,181 @@ UPDATE_CA_CERTIFICATES: str = "update-ca-certificates"
|
|
|
43
55
|
##
|
|
44
56
|
|
|
45
57
|
|
|
46
|
-
def
|
|
47
|
-
|
|
58
|
+
def append_text(
|
|
59
|
+
path: PathLike,
|
|
60
|
+
text: str,
|
|
61
|
+
/,
|
|
62
|
+
*,
|
|
63
|
+
sudo: bool = False,
|
|
64
|
+
skip_if_present: bool = False,
|
|
65
|
+
flags: int = 0,
|
|
66
|
+
blank_lines: int = 1,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Append text to a file."""
|
|
69
|
+
try:
|
|
70
|
+
existing = cat(path, sudo=sudo)
|
|
71
|
+
except (CalledProcessError, FileNotFoundError):
|
|
72
|
+
tee(path, text, sudo=sudo, append=True)
|
|
73
|
+
return
|
|
74
|
+
if skip_if_present and (search(text, existing, flags=flags) is not None):
|
|
75
|
+
return
|
|
76
|
+
full = "".join([*repeat("\n", times=blank_lines), text])
|
|
77
|
+
tee(path, full, sudo=sudo, append=True)
|
|
48
78
|
|
|
49
79
|
|
|
50
80
|
##
|
|
51
81
|
|
|
52
82
|
|
|
53
|
-
def
|
|
54
|
-
|
|
83
|
+
def apt_install(
|
|
84
|
+
package: str, /, *packages: str, update: bool = False, sudo: bool = False
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Install packages."""
|
|
87
|
+
if update: # pragma: no cover
|
|
88
|
+
apt_update(sudo=sudo)
|
|
89
|
+
args = maybe_sudo_cmd( # pragma: no cover
|
|
90
|
+
*apt_install_cmd(package, *packages), sudo=sudo
|
|
91
|
+
)
|
|
92
|
+
run(*args) # pragma: no cover
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def apt_install_cmd(package: str, /, *packages: str) -> list[str]:
|
|
96
|
+
"""Command to use 'apt' to install packages."""
|
|
97
|
+
return ["apt", "install", "-y", package, *packages]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
##
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def apt_remove(package: str, /, *packages: str, sudo: bool = False) -> None:
|
|
104
|
+
"""Remove a package."""
|
|
105
|
+
args = maybe_sudo_cmd( # pragma: no cover
|
|
106
|
+
*apt_remove_cmd(package, *packages), sudo=sudo
|
|
107
|
+
)
|
|
108
|
+
run(*args) # pragma: no cover
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def apt_remove_cmd(package: str, /, *packages: str) -> list[str]:
|
|
112
|
+
"""Command to use 'apt' to remove packages."""
|
|
113
|
+
return ["apt", "remove", "-y", package, *packages]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
##
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def apt_update(*, sudo: bool = False) -> None:
|
|
120
|
+
"""Update 'apt'."""
|
|
121
|
+
run(*maybe_sudo_cmd(*APT_UPDATE, sudo=sudo))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
##
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def cat(path: PathLike, /, *paths: PathLike, sudo: bool = False) -> str:
|
|
128
|
+
"""Concatenate and print files."""
|
|
129
|
+
if sudo: # pragma: no cover
|
|
130
|
+
return run(*sudo_cmd(*cat_cmd(path, *paths)), return_=True)
|
|
131
|
+
all_paths = list(map(Path, [path, *paths]))
|
|
132
|
+
return "\n".join(p.read_text() for p in all_paths)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def cat_cmd(path: PathLike, /, *paths: PathLike) -> list[str]:
|
|
136
|
+
"""Command to use 'cat' to concatenate and print files."""
|
|
137
|
+
return ["cat", str(path), *map(str, paths)]
|
|
55
138
|
|
|
56
139
|
|
|
57
140
|
##
|
|
58
141
|
|
|
59
142
|
|
|
60
143
|
def cd_cmd(path: PathLike, /) -> list[str]:
|
|
144
|
+
"""Command to use 'cd' to change working directory."""
|
|
61
145
|
return ["cd", str(path)]
|
|
62
146
|
|
|
63
147
|
|
|
64
148
|
##
|
|
65
149
|
|
|
66
150
|
|
|
67
|
-
def
|
|
68
|
-
|
|
151
|
+
def chattr(
|
|
152
|
+
path: PathLike, /, *, immutable: bool | None = None, sudo: bool = False
|
|
153
|
+
) -> None:
|
|
154
|
+
"""Change file attributes."""
|
|
155
|
+
args = maybe_sudo_cmd( # pragma: no cover
|
|
156
|
+
*chattr_cmd(path, immutable=immutable), sudo=sudo
|
|
157
|
+
)
|
|
158
|
+
run(*args) # pragma: no cover
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def chattr_cmd(path: PathLike, /, *, immutable: bool | None = None) -> list[str]:
|
|
162
|
+
"""Command to use 'chattr' to change file attributes."""
|
|
163
|
+
args: list[str] = ["chattr"]
|
|
164
|
+
if immutable is True:
|
|
165
|
+
args.append("+i")
|
|
166
|
+
elif immutable is False:
|
|
167
|
+
args.append("-i")
|
|
168
|
+
return [*args, str(path)]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
##
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def chmod(path: PathLike, perms: PermissionsLike, /, *, sudo: bool = False) -> None:
|
|
175
|
+
"""Change file mode."""
|
|
176
|
+
if sudo: # pragma: no cover
|
|
177
|
+
run(*sudo_cmd(*chmod_cmd(path, perms)))
|
|
178
|
+
else:
|
|
179
|
+
Path(path).chmod(int(ensure_perms(perms)))
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def chmod_cmd(path: PathLike, perms: PermissionsLike, /) -> list[str]:
|
|
183
|
+
"""Command to use 'chmod' to change file mode."""
|
|
184
|
+
return ["chmod", str(ensure_perms(perms)), str(path)]
|
|
69
185
|
|
|
70
186
|
|
|
71
187
|
##
|
|
72
188
|
|
|
73
189
|
|
|
190
|
+
def chown(
|
|
191
|
+
path: PathLike,
|
|
192
|
+
/,
|
|
193
|
+
*,
|
|
194
|
+
sudo: bool = False,
|
|
195
|
+
user: str | int | None = None,
|
|
196
|
+
group: str | int | None = None,
|
|
197
|
+
) -> None:
|
|
198
|
+
"""Change file owner and/or group."""
|
|
199
|
+
if sudo: # pragma: no cover
|
|
200
|
+
match user, group:
|
|
201
|
+
case None, None:
|
|
202
|
+
...
|
|
203
|
+
case str() | int() | None, str() | int() | None:
|
|
204
|
+
run(*sudo_cmd(*chown_cmd(path, user=user, group=group)))
|
|
205
|
+
case never:
|
|
206
|
+
assert_never(never)
|
|
207
|
+
else:
|
|
208
|
+
match user, group:
|
|
209
|
+
case None, None:
|
|
210
|
+
...
|
|
211
|
+
case str() | int(), None:
|
|
212
|
+
shutil.chown(path, user, group)
|
|
213
|
+
case None, str() | int():
|
|
214
|
+
shutil.chown(path, user, group)
|
|
215
|
+
case str() | int(), str() | int():
|
|
216
|
+
shutil.chown(path, user, group)
|
|
217
|
+
case never:
|
|
218
|
+
assert_never(never)
|
|
219
|
+
|
|
220
|
+
|
|
74
221
|
def chown_cmd(
|
|
75
|
-
path: PathLike, /, *, user: str | None = None, group: str | None = None
|
|
222
|
+
path: PathLike, /, *, user: str | int | None = None, group: str | int | None = None
|
|
76
223
|
) -> list[str]:
|
|
224
|
+
"""Command to use 'chown' to change file owner and/or group."""
|
|
77
225
|
match user, group:
|
|
78
226
|
case None, None:
|
|
79
227
|
raise ChownCmdError
|
|
80
|
-
case str(), None:
|
|
228
|
+
case str() | int(), None:
|
|
81
229
|
ownership = "user"
|
|
82
|
-
case None, str():
|
|
230
|
+
case None, str() | int():
|
|
83
231
|
ownership = f":{group}"
|
|
84
|
-
case str(), str():
|
|
232
|
+
case str() | int(), str() | int():
|
|
85
233
|
ownership = f"{user}:{group}"
|
|
86
234
|
case never:
|
|
87
235
|
assert_never(never)
|
|
@@ -98,23 +246,277 @@ class ChownCmdError(Exception):
|
|
|
98
246
|
##
|
|
99
247
|
|
|
100
248
|
|
|
249
|
+
def chpasswd(user_name: str, password: str, /, *, sudo: bool = False) -> None:
|
|
250
|
+
"""Update passwords."""
|
|
251
|
+
args = maybe_sudo_cmd(CHPASSWD, sudo=sudo) # pragma: no cover
|
|
252
|
+
run(*args, input=f"{user_name}:{password}") # pragma: no cover
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
##
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def copy_text(
|
|
259
|
+
src: PathLike,
|
|
260
|
+
dest: PathLike,
|
|
261
|
+
/,
|
|
262
|
+
*,
|
|
263
|
+
sudo: bool = False,
|
|
264
|
+
substitutions: StrMapping | None = None,
|
|
265
|
+
) -> None:
|
|
266
|
+
"""Copy the text contents of a file."""
|
|
267
|
+
text = cat(src, sudo=sudo)
|
|
268
|
+
if substitutions is not None:
|
|
269
|
+
text = Template(text).substitute(**substitutions)
|
|
270
|
+
tee(dest, text, sudo=sudo)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
##
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def cp(
|
|
277
|
+
src: PathLike,
|
|
278
|
+
dest: PathLike,
|
|
279
|
+
/,
|
|
280
|
+
*,
|
|
281
|
+
sudo: bool = False,
|
|
282
|
+
perms: PermissionsLike | None = None,
|
|
283
|
+
owner: str | int | None = None,
|
|
284
|
+
group: str | int | None = None,
|
|
285
|
+
) -> None:
|
|
286
|
+
"""Copy a file/directory."""
|
|
287
|
+
mkdir(dest, sudo=sudo, parent=True)
|
|
288
|
+
if sudo: # pragma: no cover
|
|
289
|
+
run(*sudo_cmd(*cp_cmd(src, dest)))
|
|
290
|
+
else:
|
|
291
|
+
src, dest = map(Path, [src, dest])
|
|
292
|
+
if src.is_file():
|
|
293
|
+
_ = copyfile(src, dest)
|
|
294
|
+
elif src.is_dir():
|
|
295
|
+
_ = copytree(src, dest, dirs_exist_ok=True)
|
|
296
|
+
else:
|
|
297
|
+
raise CpError(src=src, dest=dest)
|
|
298
|
+
if perms is not None:
|
|
299
|
+
chmod(dest, perms, sudo=sudo)
|
|
300
|
+
if (owner is not None) or (group is not None):
|
|
301
|
+
chown(dest, sudo=sudo, user=owner, group=group)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@dataclass(kw_only=True, slots=True)
|
|
305
|
+
class CpError(Exception):
|
|
306
|
+
src: Path
|
|
307
|
+
dest: Path
|
|
308
|
+
|
|
309
|
+
@override
|
|
310
|
+
def __str__(self) -> str:
|
|
311
|
+
return f"Unable to copy {str(self.src)!r} to {str(self.dest)!r}; source does not exist"
|
|
312
|
+
|
|
313
|
+
|
|
101
314
|
def cp_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
|
|
315
|
+
"""Command to use 'cp' to copy a file/directory."""
|
|
102
316
|
return ["cp", "-r", str(src), str(dest)]
|
|
103
317
|
|
|
104
318
|
|
|
105
319
|
##
|
|
106
320
|
|
|
107
321
|
|
|
322
|
+
@overload
|
|
323
|
+
def curl(
|
|
324
|
+
url: str,
|
|
325
|
+
/,
|
|
326
|
+
*,
|
|
327
|
+
fail: bool = True,
|
|
328
|
+
location: bool = True,
|
|
329
|
+
output: PathLike | None = None,
|
|
330
|
+
show_error: bool = True,
|
|
331
|
+
silent: bool = True,
|
|
332
|
+
sudo: bool = False,
|
|
333
|
+
print: bool = False,
|
|
334
|
+
print_stdout: bool = False,
|
|
335
|
+
print_stderr: bool = False,
|
|
336
|
+
return_: Literal[True],
|
|
337
|
+
return_stdout: Literal[False] = False,
|
|
338
|
+
return_stderr: Literal[False] = False,
|
|
339
|
+
retry: Retry | None = None,
|
|
340
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
341
|
+
logger: LoggerLike | None = None,
|
|
342
|
+
) -> str: ...
|
|
343
|
+
@overload
|
|
344
|
+
def curl(
|
|
345
|
+
url: str,
|
|
346
|
+
/,
|
|
347
|
+
*,
|
|
348
|
+
fail: bool = True,
|
|
349
|
+
location: bool = True,
|
|
350
|
+
output: PathLike | None = None,
|
|
351
|
+
show_error: bool = True,
|
|
352
|
+
silent: bool = True,
|
|
353
|
+
sudo: bool = False,
|
|
354
|
+
print: bool = False,
|
|
355
|
+
print_stdout: bool = False,
|
|
356
|
+
print_stderr: bool = False,
|
|
357
|
+
return_: Literal[False] = False,
|
|
358
|
+
return_stdout: Literal[True],
|
|
359
|
+
return_stderr: Literal[False] = False,
|
|
360
|
+
retry: Retry | None = None,
|
|
361
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
362
|
+
logger: LoggerLike | None = None,
|
|
363
|
+
) -> str: ...
|
|
364
|
+
@overload
|
|
365
|
+
def curl(
|
|
366
|
+
url: str,
|
|
367
|
+
/,
|
|
368
|
+
*,
|
|
369
|
+
fail: bool = True,
|
|
370
|
+
location: bool = True,
|
|
371
|
+
output: PathLike | None = None,
|
|
372
|
+
show_error: bool = True,
|
|
373
|
+
silent: bool = True,
|
|
374
|
+
sudo: bool = False,
|
|
375
|
+
print: bool = False,
|
|
376
|
+
print_stdout: bool = False,
|
|
377
|
+
print_stderr: bool = False,
|
|
378
|
+
return_: Literal[False] = False,
|
|
379
|
+
return_stdout: Literal[False] = False,
|
|
380
|
+
return_stderr: Literal[True],
|
|
381
|
+
retry: Retry | None = None,
|
|
382
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
383
|
+
logger: LoggerLike | None = None,
|
|
384
|
+
) -> str: ...
|
|
385
|
+
@overload
|
|
386
|
+
def curl(
|
|
387
|
+
url: str,
|
|
388
|
+
/,
|
|
389
|
+
*,
|
|
390
|
+
fail: bool = True,
|
|
391
|
+
location: bool = True,
|
|
392
|
+
output: PathLike | None = None,
|
|
393
|
+
show_error: bool = True,
|
|
394
|
+
silent: bool = True,
|
|
395
|
+
sudo: bool = False,
|
|
396
|
+
print: bool = False,
|
|
397
|
+
print_stdout: bool = False,
|
|
398
|
+
print_stderr: bool = False,
|
|
399
|
+
return_: Literal[False] = False,
|
|
400
|
+
return_stdout: Literal[False] = False,
|
|
401
|
+
return_stderr: Literal[False] = False,
|
|
402
|
+
retry: Retry | None = None,
|
|
403
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
404
|
+
logger: LoggerLike | None = None,
|
|
405
|
+
) -> None: ...
|
|
406
|
+
@overload
|
|
407
|
+
def curl(
|
|
408
|
+
url: str,
|
|
409
|
+
/,
|
|
410
|
+
*,
|
|
411
|
+
fail: bool = True,
|
|
412
|
+
location: bool = True,
|
|
413
|
+
output: PathLike | None = None,
|
|
414
|
+
show_error: bool = True,
|
|
415
|
+
silent: bool = True,
|
|
416
|
+
sudo: bool = False,
|
|
417
|
+
print: bool = False,
|
|
418
|
+
print_stdout: bool = False,
|
|
419
|
+
print_stderr: bool = False,
|
|
420
|
+
return_: bool = False,
|
|
421
|
+
return_stdout: bool = False,
|
|
422
|
+
return_stderr: bool = False,
|
|
423
|
+
retry: Retry | None = None,
|
|
424
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
425
|
+
logger: LoggerLike | None = None,
|
|
426
|
+
) -> str | None: ...
|
|
427
|
+
def curl(
|
|
428
|
+
url: str,
|
|
429
|
+
/,
|
|
430
|
+
*,
|
|
431
|
+
fail: bool = True,
|
|
432
|
+
location: bool = True,
|
|
433
|
+
output: PathLike | None = None,
|
|
434
|
+
show_error: bool = True,
|
|
435
|
+
silent: bool = True,
|
|
436
|
+
sudo: bool = False,
|
|
437
|
+
print: bool = False, # noqa: A002
|
|
438
|
+
print_stdout: bool = False,
|
|
439
|
+
print_stderr: bool = False,
|
|
440
|
+
return_: bool = False,
|
|
441
|
+
return_stdout: bool = False,
|
|
442
|
+
return_stderr: bool = False,
|
|
443
|
+
retry: Retry | None = None,
|
|
444
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
445
|
+
logger: LoggerLike | None = None,
|
|
446
|
+
) -> str | None:
|
|
447
|
+
"""Transfer a URL."""
|
|
448
|
+
args = maybe_sudo_cmd( # skipif-ci
|
|
449
|
+
*curl_cmd(
|
|
450
|
+
url,
|
|
451
|
+
fail=fail,
|
|
452
|
+
location=location,
|
|
453
|
+
output=output,
|
|
454
|
+
show_error=show_error,
|
|
455
|
+
silent=silent,
|
|
456
|
+
),
|
|
457
|
+
sudo=sudo,
|
|
458
|
+
)
|
|
459
|
+
return run( # skipif-ci
|
|
460
|
+
*args,
|
|
461
|
+
print=print,
|
|
462
|
+
print_stdout=print_stdout,
|
|
463
|
+
print_stderr=print_stderr,
|
|
464
|
+
return_=return_,
|
|
465
|
+
return_stdout=return_stdout,
|
|
466
|
+
return_stderr=return_stderr,
|
|
467
|
+
retry=retry,
|
|
468
|
+
retry_skip=retry_skip,
|
|
469
|
+
logger=logger,
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def curl_cmd(
|
|
474
|
+
url: str,
|
|
475
|
+
/,
|
|
476
|
+
*,
|
|
477
|
+
fail: bool = True,
|
|
478
|
+
location: bool = True,
|
|
479
|
+
output: PathLike | None = None,
|
|
480
|
+
show_error: bool = True,
|
|
481
|
+
silent: bool = True,
|
|
482
|
+
) -> list[str]:
|
|
483
|
+
"""Command to use 'curl' to transfer a URL."""
|
|
484
|
+
args: list[str] = ["curl"]
|
|
485
|
+
if fail:
|
|
486
|
+
args.append("--fail")
|
|
487
|
+
if location:
|
|
488
|
+
args.append("--location")
|
|
489
|
+
if output is not None:
|
|
490
|
+
args.extend(["--create-dirs", "--output", str(output)])
|
|
491
|
+
if show_error:
|
|
492
|
+
args.append("--show-error")
|
|
493
|
+
if silent:
|
|
494
|
+
args.append("--silent")
|
|
495
|
+
return [*args, url]
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
##
|
|
499
|
+
|
|
500
|
+
|
|
108
501
|
def echo_cmd(text: str, /) -> list[str]:
|
|
502
|
+
"""Command to use 'echo' to write arguments to the standard output."""
|
|
109
503
|
return ["echo", text]
|
|
110
504
|
|
|
111
505
|
|
|
112
506
|
##
|
|
113
507
|
|
|
114
508
|
|
|
509
|
+
def env_cmds(env: StrStrMapping, /) -> list[str]:
|
|
510
|
+
return [f"{key}={value}" for key, value in env.items()]
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
##
|
|
514
|
+
|
|
515
|
+
|
|
115
516
|
def expand_path(
|
|
116
517
|
path: PathLike, /, *, subs: StrMapping | None = None, sudo: bool = False
|
|
117
518
|
) -> Path:
|
|
519
|
+
"""Expand a path using `subprocess`."""
|
|
118
520
|
if subs is not None:
|
|
119
521
|
path = Template(str(path)).substitute(**subs)
|
|
120
522
|
if sudo: # pragma: no cover
|
|
@@ -125,38 +527,103 @@ def expand_path(
|
|
|
125
527
|
##
|
|
126
528
|
|
|
127
529
|
|
|
128
|
-
def
|
|
129
|
-
|
|
530
|
+
def git_branch_current(path: PathLike, /) -> str:
|
|
531
|
+
"""Show the current a branch."""
|
|
532
|
+
return run(*GIT_BRANCH_SHOW_CURRENT, cwd=path, return_=True)
|
|
130
533
|
|
|
131
534
|
|
|
132
535
|
##
|
|
133
536
|
|
|
134
537
|
|
|
135
|
-
def
|
|
136
|
-
|
|
137
|
-
|
|
538
|
+
def git_checkout(branch: str, path: PathLike, /) -> None:
|
|
539
|
+
"""Switch a branch."""
|
|
540
|
+
run(*git_checkout_cmd(branch), cwd=path)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def git_checkout_cmd(branch: str, /) -> list[str]:
|
|
544
|
+
"""Command to use 'git checkout' to switch a branch."""
|
|
545
|
+
return ["git", "checkout", branch]
|
|
138
546
|
|
|
139
547
|
|
|
140
548
|
##
|
|
141
549
|
|
|
142
550
|
|
|
143
|
-
def
|
|
144
|
-
path =
|
|
145
|
-
|
|
551
|
+
def git_clone(
|
|
552
|
+
url: str, path: PathLike, /, *, sudo: bool = False, branch: str | None = None
|
|
553
|
+
) -> None:
|
|
554
|
+
"""Clone a repository."""
|
|
555
|
+
rm(path, sudo=sudo)
|
|
556
|
+
run(*maybe_sudo_cmd(*git_clone_cmd(url, path), sudo=sudo))
|
|
557
|
+
if branch is not None:
|
|
558
|
+
git_checkout(branch, path)
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def git_clone_cmd(url: str, path: PathLike, /) -> list[str]:
|
|
562
|
+
"""Command to use 'git clone' to clone a repository."""
|
|
563
|
+
return ["git", "clone", "--recurse-submodules", url, str(path)]
|
|
146
564
|
|
|
147
565
|
|
|
148
566
|
##
|
|
149
567
|
|
|
150
568
|
|
|
151
|
-
def
|
|
152
|
-
|
|
153
|
-
|
|
569
|
+
def install(
|
|
570
|
+
path: PathLike,
|
|
571
|
+
/,
|
|
572
|
+
*,
|
|
573
|
+
directory: bool = False,
|
|
574
|
+
mode: PermissionsLike | None = None,
|
|
575
|
+
owner: str | int | None = None,
|
|
576
|
+
group: str | int | None = None,
|
|
577
|
+
sudo: bool = False,
|
|
578
|
+
) -> None:
|
|
579
|
+
"""Install a binary."""
|
|
580
|
+
args = maybe_sudo_cmd(
|
|
581
|
+
*install_cmd(path, directory=directory, mode=mode, owner=owner, group=group),
|
|
582
|
+
sudo=sudo,
|
|
583
|
+
)
|
|
584
|
+
run(*args)
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def install_cmd(
|
|
588
|
+
path: PathLike,
|
|
589
|
+
/,
|
|
590
|
+
*,
|
|
591
|
+
directory: bool = False,
|
|
592
|
+
mode: PermissionsLike | None = None,
|
|
593
|
+
owner: str | int | None = None,
|
|
594
|
+
group: str | int | None = None,
|
|
595
|
+
) -> list[str]:
|
|
596
|
+
"""Command to use 'install' to install a binary."""
|
|
597
|
+
args: list[str] = ["install"]
|
|
598
|
+
if directory:
|
|
599
|
+
args.append("-d")
|
|
600
|
+
if mode is not None:
|
|
601
|
+
args.extend(["-m", str(ensure_perms(mode))])
|
|
602
|
+
if owner is not None:
|
|
603
|
+
args.extend(["-o", str(owner)])
|
|
604
|
+
if group is not None:
|
|
605
|
+
args.extend(["-g", str(group)])
|
|
606
|
+
if directory:
|
|
607
|
+
args.append(str(path))
|
|
608
|
+
else:
|
|
609
|
+
args.extend(["/dev/null", str(path)])
|
|
610
|
+
return args
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
##
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def maybe_parent(path: PathLike, /, *, parent: bool = False) -> Path:
|
|
617
|
+
"""Get the parent of a path, if required."""
|
|
618
|
+
path = Path(path)
|
|
619
|
+
return path.parent if parent else path
|
|
154
620
|
|
|
155
621
|
|
|
156
622
|
##
|
|
157
623
|
|
|
158
624
|
|
|
159
625
|
def mkdir(path: PathLike, /, *, sudo: bool = False, parent: bool = False) -> None:
|
|
626
|
+
"""Make a directory."""
|
|
160
627
|
if sudo: # pragma: no cover
|
|
161
628
|
run(*sudo_cmd(*mkdir_cmd(path, parent=parent)))
|
|
162
629
|
else:
|
|
@@ -167,21 +634,105 @@ def mkdir(path: PathLike, /, *, sudo: bool = False, parent: bool = False) -> Non
|
|
|
167
634
|
|
|
168
635
|
|
|
169
636
|
def mkdir_cmd(path: PathLike, /, *, parent: bool = False) -> list[str]:
|
|
637
|
+
"""Command to use 'mv' to make a directory."""
|
|
170
638
|
return ["mkdir", "-p", str(maybe_parent(path, parent=parent))]
|
|
171
639
|
|
|
172
640
|
|
|
173
641
|
##
|
|
174
642
|
|
|
175
643
|
|
|
644
|
+
def mv(
|
|
645
|
+
src: PathLike,
|
|
646
|
+
dest: PathLike,
|
|
647
|
+
/,
|
|
648
|
+
*,
|
|
649
|
+
sudo: bool = False,
|
|
650
|
+
perms: PermissionsLike | None = None,
|
|
651
|
+
owner: str | int | None = None,
|
|
652
|
+
group: str | int | None = None,
|
|
653
|
+
) -> None:
|
|
654
|
+
"""Move a file/directory."""
|
|
655
|
+
mkdir(dest, sudo=sudo, parent=True)
|
|
656
|
+
if sudo: # pragma: no cover
|
|
657
|
+
run(*sudo_cmd(*cp_cmd(src, dest)))
|
|
658
|
+
else:
|
|
659
|
+
src, dest = map(Path, [src, dest])
|
|
660
|
+
if src.exists():
|
|
661
|
+
_ = move(src, dest)
|
|
662
|
+
else:
|
|
663
|
+
raise MvFileError(src=src, dest=dest)
|
|
664
|
+
if perms is not None:
|
|
665
|
+
chmod(dest, perms, sudo=sudo)
|
|
666
|
+
if (owner is not None) or (group is not None):
|
|
667
|
+
chown(dest, sudo=sudo, user=owner, group=group)
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
@dataclass(kw_only=True, slots=True)
|
|
671
|
+
class MvFileError(Exception):
|
|
672
|
+
src: Path
|
|
673
|
+
dest: Path
|
|
674
|
+
|
|
675
|
+
@override
|
|
676
|
+
def __str__(self) -> str:
|
|
677
|
+
return f"Unable to move {str(self.src)!r} to {str(self.dest)!r}; source does not exist"
|
|
678
|
+
|
|
679
|
+
|
|
176
680
|
def mv_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
|
|
681
|
+
"""Command to use 'mv' to move a file/directory."""
|
|
177
682
|
return ["mv", str(src), str(dest)]
|
|
178
683
|
|
|
179
684
|
|
|
180
685
|
##
|
|
181
686
|
|
|
182
687
|
|
|
183
|
-
def
|
|
184
|
-
|
|
688
|
+
def replace_text(
|
|
689
|
+
path: PathLike, /, *replacements: tuple[str, str], sudo: bool = False
|
|
690
|
+
) -> None:
|
|
691
|
+
"""Replace the text in a file."""
|
|
692
|
+
path = Path(path)
|
|
693
|
+
text = cat(path, sudo=sudo)
|
|
694
|
+
for old, new in replacements:
|
|
695
|
+
text = text.replace(old, new)
|
|
696
|
+
tee(path, text, sudo=sudo)
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
##
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def ripgrep(*args: str, path: PathLike = PWD) -> str | None:
|
|
703
|
+
"""Search for lines."""
|
|
704
|
+
try: # skipif-ci
|
|
705
|
+
return run(*ripgrep_cmd(*args, path=path), return_=True)
|
|
706
|
+
except CalledProcessError as error: # skipif-ci
|
|
707
|
+
if error.returncode == 1:
|
|
708
|
+
return None
|
|
709
|
+
raise
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def ripgrep_cmd(*args: str, path: PathLike = PWD) -> list[str]:
|
|
713
|
+
"""Command to use 'ripgrep' to search for lines."""
|
|
714
|
+
return ["rg", *args, str(path)]
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
##
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def rm(path: PathLike, /, *paths: PathLike, sudo: bool = False) -> None:
|
|
721
|
+
"""Remove a file/directory."""
|
|
722
|
+
if sudo: # pragma: no cover
|
|
723
|
+
run(*sudo_cmd(*rm_cmd(path, *paths)))
|
|
724
|
+
else:
|
|
725
|
+
all_paths = list(map(Path, [path, *paths]))
|
|
726
|
+
for p in all_paths:
|
|
727
|
+
if p.is_file():
|
|
728
|
+
p.unlink(missing_ok=True)
|
|
729
|
+
elif p.is_dir():
|
|
730
|
+
rmtree(p, ignore_errors=True)
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def rm_cmd(path: PathLike, /, *paths: PathLike) -> list[str]:
|
|
734
|
+
"""Command to use 'rm' to remove a file/directory."""
|
|
735
|
+
return ["rm", "-rf", str(path), *map(str, paths)]
|
|
185
736
|
|
|
186
737
|
|
|
187
738
|
##
|
|
@@ -204,8 +755,9 @@ def rsync(
|
|
|
204
755
|
chown_user: str | None = None,
|
|
205
756
|
chown_group: str | None = None,
|
|
206
757
|
exclude: MaybeIterable[str] | None = None,
|
|
207
|
-
chmod:
|
|
758
|
+
chmod: PermissionsLike | None = None,
|
|
208
759
|
) -> None:
|
|
760
|
+
"""Remote & local file copying."""
|
|
209
761
|
mkdir_args = maybe_sudo_cmd(*mkdir_cmd(dest, parent=True), sudo=sudo) # skipif-ci
|
|
210
762
|
ssh( # skipif-ci
|
|
211
763
|
user,
|
|
@@ -218,13 +770,13 @@ def rsync(
|
|
|
218
770
|
retry=retry,
|
|
219
771
|
logger=logger,
|
|
220
772
|
)
|
|
221
|
-
|
|
773
|
+
srcs = list(always_iterable(src_or_srcs)) # skipif-ci
|
|
222
774
|
rsync_args = rsync_cmd( # skipif-ci
|
|
223
|
-
|
|
775
|
+
srcs,
|
|
224
776
|
user,
|
|
225
777
|
hostname,
|
|
226
778
|
dest,
|
|
227
|
-
archive=is_dir,
|
|
779
|
+
archive=any(Path(s).is_dir() for s in srcs),
|
|
228
780
|
chown_user=chown_user,
|
|
229
781
|
chown_group=chown_group,
|
|
230
782
|
exclude=exclude,
|
|
@@ -232,7 +784,6 @@ def rsync(
|
|
|
232
784
|
host_key_algorithms=host_key_algorithms,
|
|
233
785
|
strict_host_key_checking=strict_host_key_checking,
|
|
234
786
|
sudo=sudo,
|
|
235
|
-
parent=is_dir,
|
|
236
787
|
)
|
|
237
788
|
run(*rsync_args, print=print, retry=retry, logger=logger) # skipif-ci
|
|
238
789
|
if chmod is not None: # skipif-ci
|
|
@@ -250,59 +801,188 @@ def rsync(
|
|
|
250
801
|
)
|
|
251
802
|
|
|
252
803
|
|
|
804
|
+
def rsync_cmd(
|
|
805
|
+
src_or_srcs: MaybeIterable[PathLike],
|
|
806
|
+
user: str,
|
|
807
|
+
hostname: str,
|
|
808
|
+
dest: PathLike,
|
|
809
|
+
/,
|
|
810
|
+
*,
|
|
811
|
+
archive: bool = False,
|
|
812
|
+
chown_user: str | None = None,
|
|
813
|
+
chown_group: str | None = None,
|
|
814
|
+
exclude: MaybeIterable[str] | None = None,
|
|
815
|
+
batch_mode: bool = True,
|
|
816
|
+
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
817
|
+
strict_host_key_checking: bool = True,
|
|
818
|
+
sudo: bool = False,
|
|
819
|
+
) -> list[str]:
|
|
820
|
+
"""Command to use 'rsync' to do remote & local file copying."""
|
|
821
|
+
args: list[str] = ["rsync"]
|
|
822
|
+
if archive:
|
|
823
|
+
args.append("--archive")
|
|
824
|
+
args.append("--checksum")
|
|
825
|
+
match chown_user, chown_group:
|
|
826
|
+
case None, None:
|
|
827
|
+
...
|
|
828
|
+
case str(), None:
|
|
829
|
+
args.extend(["--chown", chown_user])
|
|
830
|
+
case None, str():
|
|
831
|
+
args.extend(["--chown", f":{chown_group}"])
|
|
832
|
+
case str(), str():
|
|
833
|
+
args.extend(["--chown", f"{chown_user}:{chown_group}"])
|
|
834
|
+
case never:
|
|
835
|
+
assert_never(never)
|
|
836
|
+
args.append("--compress")
|
|
837
|
+
if exclude is not None:
|
|
838
|
+
for exclude_i in always_iterable(exclude):
|
|
839
|
+
args.extend(["--exclude", exclude_i])
|
|
840
|
+
rsh_args: list[str] = ssh_opts_cmd(
|
|
841
|
+
batch_mode=batch_mode,
|
|
842
|
+
host_key_algorithms=host_key_algorithms,
|
|
843
|
+
strict_host_key_checking=strict_host_key_checking,
|
|
844
|
+
)
|
|
845
|
+
args.extend(["--rsh", join(rsh_args)])
|
|
846
|
+
if sudo:
|
|
847
|
+
args.extend(["--rsync-path", join(sudo_cmd("rsync"))])
|
|
848
|
+
srcs = list(always_iterable(src_or_srcs)) # do not Path()
|
|
849
|
+
if len(srcs) == 0:
|
|
850
|
+
raise RsyncCmdNoSourcesError(user=user, hostname=hostname, dest=dest)
|
|
851
|
+
missing = [s for s in srcs if not Path(s).exists()]
|
|
852
|
+
if len(missing) >= 1:
|
|
853
|
+
raise RsyncCmdSourcesNotFoundError(
|
|
854
|
+
sources=missing, user=user, hostname=hostname, dest=dest
|
|
855
|
+
)
|
|
856
|
+
return [*args, *map(str, srcs), f"{user}@{hostname}:{dest}"]
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
@dataclass(kw_only=True, slots=True)
|
|
860
|
+
class RsyncCmdError(Exception):
|
|
861
|
+
user: str
|
|
862
|
+
hostname: str
|
|
863
|
+
dest: PathLike
|
|
864
|
+
|
|
865
|
+
@override
|
|
866
|
+
def __str__(self) -> str:
|
|
867
|
+
return f"No sources selected to send to {self.user}@{self.hostname}:{self.dest}"
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
@dataclass(kw_only=True, slots=True)
|
|
871
|
+
class RsyncCmdNoSourcesError(RsyncCmdError):
|
|
872
|
+
@override
|
|
873
|
+
def __str__(self) -> str:
|
|
874
|
+
return f"No sources selected to send to {self.user}@{self.hostname}:{self.dest}"
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
@dataclass(kw_only=True, slots=True)
|
|
878
|
+
class RsyncCmdSourcesNotFoundError(RsyncCmdError):
|
|
879
|
+
sources: list[PathLike]
|
|
880
|
+
|
|
881
|
+
@override
|
|
882
|
+
def __str__(self) -> str:
|
|
883
|
+
desc = ", ".join(map(repr, map(str, self.sources)))
|
|
884
|
+
return f"Sources selected to send to {self.user}@{self.hostname}:{self.dest} but not found: {desc}"
|
|
885
|
+
|
|
886
|
+
|
|
253
887
|
##
|
|
254
888
|
|
|
255
889
|
|
|
256
|
-
def
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
890
|
+
def rsync_many(
|
|
891
|
+
user: str,
|
|
892
|
+
hostname: str,
|
|
893
|
+
/,
|
|
894
|
+
*items: tuple[PathLike, PathLike]
|
|
895
|
+
| tuple[Literal["sudo"], PathLike, PathLike]
|
|
896
|
+
| tuple[PathLike, PathLike, PermissionsLike],
|
|
897
|
+
retry: Retry | None = None,
|
|
898
|
+
logger: LoggerLike | None = None,
|
|
899
|
+
keep: bool = False,
|
|
900
|
+
batch_mode: bool = True,
|
|
901
|
+
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
902
|
+
strict_host_key_checking: bool = True,
|
|
903
|
+
print: bool = False, # noqa: A002
|
|
904
|
+
exclude: MaybeIterable[str] | None = None,
|
|
905
|
+
) -> None:
|
|
906
|
+
cmds: list[list[str]] = [] # skipif-ci
|
|
907
|
+
with ( # skipif-ci
|
|
908
|
+
TemporaryDirectory() as temp_src,
|
|
909
|
+
yield_ssh_temp_dir(
|
|
910
|
+
user, hostname, retry=retry, logger=logger, keep=keep
|
|
911
|
+
) as temp_dest,
|
|
912
|
+
):
|
|
913
|
+
for item in items:
|
|
914
|
+
match item:
|
|
915
|
+
case Path() | str() as src, Path() | str() as dest:
|
|
916
|
+
cmds.extend(_rsync_many_prepare(src, dest, temp_src, temp_dest))
|
|
917
|
+
case "sudo", Path() | str() as src, Path() | str() as dest:
|
|
918
|
+
cmds.extend(
|
|
919
|
+
_rsync_many_prepare(src, dest, temp_src, temp_dest, sudo=True)
|
|
920
|
+
)
|
|
921
|
+
case (
|
|
922
|
+
Path() | str() as src,
|
|
923
|
+
Path() | str() as dest,
|
|
924
|
+
Permissions() | int() | str() as perms,
|
|
925
|
+
):
|
|
926
|
+
cmds.extend(
|
|
927
|
+
_rsync_many_prepare(src, dest, temp_src, temp_dest, perms=perms)
|
|
928
|
+
)
|
|
929
|
+
case never:
|
|
930
|
+
assert_never(never)
|
|
931
|
+
rsync(
|
|
932
|
+
f"{temp_src}/",
|
|
933
|
+
user,
|
|
934
|
+
hostname,
|
|
935
|
+
temp_dest,
|
|
936
|
+
batch_mode=batch_mode,
|
|
937
|
+
host_key_algorithms=host_key_algorithms,
|
|
938
|
+
strict_host_key_checking=strict_host_key_checking,
|
|
939
|
+
print=print,
|
|
940
|
+
retry=retry,
|
|
941
|
+
logger=logger,
|
|
942
|
+
exclude=exclude,
|
|
943
|
+
)
|
|
944
|
+
ssh(
|
|
945
|
+
user,
|
|
946
|
+
hostname,
|
|
947
|
+
*BASH_LS,
|
|
948
|
+
input="\n".join(map(join, cmds)),
|
|
949
|
+
print=print,
|
|
950
|
+
retry=retry,
|
|
951
|
+
logger=logger,
|
|
952
|
+
)
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def _rsync_many_prepare(
|
|
956
|
+
src: PathLike,
|
|
260
957
|
dest: PathLike,
|
|
958
|
+
temp_src: PathLike,
|
|
959
|
+
temp_dest: PathLike,
|
|
261
960
|
/,
|
|
262
961
|
*,
|
|
263
|
-
archive: bool = False,
|
|
264
|
-
chown_user: str | None = None,
|
|
265
|
-
chown_group: str | None = None,
|
|
266
|
-
exclude: MaybeIterable[str] | None = None,
|
|
267
|
-
batch_mode: bool = True,
|
|
268
|
-
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
269
|
-
strict_host_key_checking: bool = True,
|
|
270
962
|
sudo: bool = False,
|
|
271
|
-
|
|
272
|
-
) -> list[str]:
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
case str(), str():
|
|
285
|
-
args.extend(["--chown", f"{chown_user}:{chown_group}"])
|
|
963
|
+
perms: PermissionsLike | None = None,
|
|
964
|
+
) -> list[list[str]]:
|
|
965
|
+
dest, temp_src, temp_dest = map(Path, [dest, temp_src, temp_dest])
|
|
966
|
+
n = len(list(temp_src.iterdir()))
|
|
967
|
+
name = str(n)
|
|
968
|
+
match src:
|
|
969
|
+
case Path():
|
|
970
|
+
cp(src, temp_src / name)
|
|
971
|
+
case str():
|
|
972
|
+
if Path(src).exists():
|
|
973
|
+
cp(src, temp_src / name)
|
|
974
|
+
else:
|
|
975
|
+
tee(temp_src / name, src)
|
|
286
976
|
case never:
|
|
287
977
|
assert_never(never)
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
rsh_args: list[str] = ssh_opts_cmd(
|
|
293
|
-
batch_mode=batch_mode,
|
|
294
|
-
host_key_algorithms=host_key_algorithms,
|
|
295
|
-
strict_host_key_checking=strict_host_key_checking,
|
|
296
|
-
)
|
|
297
|
-
args.extend(["--rsh", join(rsh_args)])
|
|
298
|
-
if sudo:
|
|
299
|
-
args.extend(["--rsync-path", join(sudo_cmd("rsync"))])
|
|
300
|
-
dest_use = maybe_parent(dest, parent=parent)
|
|
301
|
-
return [
|
|
302
|
-
*args,
|
|
303
|
-
*map(str, always_iterable(src_or_srcs)),
|
|
304
|
-
f"{user}@{hostname}:{dest_use}",
|
|
978
|
+
cmds: list[list[str]] = [
|
|
979
|
+
maybe_sudo_cmd(*rm_cmd(dest), sudo=sudo),
|
|
980
|
+
maybe_sudo_cmd(*mkdir_cmd(dest, parent=True), sudo=sudo),
|
|
981
|
+
maybe_sudo_cmd(*cp_cmd(temp_dest / name, dest), sudo=sudo),
|
|
305
982
|
]
|
|
983
|
+
if perms is not None:
|
|
984
|
+
cmds.append(maybe_sudo_cmd(*chmod_cmd(dest, perms), sudo=sudo))
|
|
985
|
+
return cmds
|
|
306
986
|
|
|
307
987
|
|
|
308
988
|
##
|
|
@@ -326,6 +1006,7 @@ def run(
|
|
|
326
1006
|
return_stdout: bool = False,
|
|
327
1007
|
return_stderr: bool = False,
|
|
328
1008
|
retry: Retry | None = None,
|
|
1009
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
329
1010
|
logger: LoggerLike | None = None,
|
|
330
1011
|
) -> str: ...
|
|
331
1012
|
@overload
|
|
@@ -346,6 +1027,7 @@ def run(
|
|
|
346
1027
|
return_stdout: Literal[True],
|
|
347
1028
|
return_stderr: bool = False,
|
|
348
1029
|
retry: Retry | None = None,
|
|
1030
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
349
1031
|
logger: LoggerLike | None = None,
|
|
350
1032
|
) -> str: ...
|
|
351
1033
|
@overload
|
|
@@ -366,6 +1048,7 @@ def run(
|
|
|
366
1048
|
return_stdout: bool = False,
|
|
367
1049
|
return_stderr: Literal[True],
|
|
368
1050
|
retry: Retry | None = None,
|
|
1051
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
369
1052
|
logger: LoggerLike | None = None,
|
|
370
1053
|
) -> str: ...
|
|
371
1054
|
@overload
|
|
@@ -386,6 +1069,7 @@ def run(
|
|
|
386
1069
|
return_stdout: Literal[False] = False,
|
|
387
1070
|
return_stderr: Literal[False] = False,
|
|
388
1071
|
retry: Retry | None = None,
|
|
1072
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
389
1073
|
logger: LoggerLike | None = None,
|
|
390
1074
|
) -> None: ...
|
|
391
1075
|
@overload
|
|
@@ -406,6 +1090,7 @@ def run(
|
|
|
406
1090
|
return_stdout: bool = False,
|
|
407
1091
|
return_stderr: bool = False,
|
|
408
1092
|
retry: Retry | None = None,
|
|
1093
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
409
1094
|
logger: LoggerLike | None = None,
|
|
410
1095
|
) -> str | None: ...
|
|
411
1096
|
def run(
|
|
@@ -425,8 +1110,10 @@ def run(
|
|
|
425
1110
|
return_stdout: bool = False,
|
|
426
1111
|
return_stderr: bool = False,
|
|
427
1112
|
retry: Retry | None = None,
|
|
1113
|
+
retry_skip: Callable[[int, str, str], bool] | None = None,
|
|
428
1114
|
logger: LoggerLike | None = None,
|
|
429
1115
|
) -> str | None:
|
|
1116
|
+
"""Run a command in a subprocess."""
|
|
430
1117
|
args: list[str] = []
|
|
431
1118
|
if user is not None: # pragma: no cover
|
|
432
1119
|
args.extend(["su", "-", str(user)])
|
|
@@ -481,14 +1168,18 @@ def run(
|
|
|
481
1168
|
case 0, False, False:
|
|
482
1169
|
return None
|
|
483
1170
|
case _, _, _:
|
|
484
|
-
if retry is None:
|
|
485
|
-
attempts = delta = None
|
|
486
|
-
else:
|
|
487
|
-
attempts, delta = retry
|
|
488
1171
|
_ = stdout.seek(0)
|
|
489
1172
|
stdout_text = stdout.read()
|
|
490
1173
|
_ = stderr.seek(0)
|
|
491
1174
|
stderr_text = stderr.read()
|
|
1175
|
+
if (retry is None) or (
|
|
1176
|
+
(retry is not None)
|
|
1177
|
+
and (retry_skip is not None)
|
|
1178
|
+
and retry_skip(return_code, stdout_text, stderr_text)
|
|
1179
|
+
):
|
|
1180
|
+
attempts = delta = None
|
|
1181
|
+
else:
|
|
1182
|
+
attempts, delta = retry
|
|
492
1183
|
if logger is not None:
|
|
493
1184
|
msg = strip_and_dedent(f"""
|
|
494
1185
|
'run' failed with:
|
|
@@ -567,6 +1258,7 @@ def _run_write_to_streams(text: str, /, *outputs: IO[str]) -> None:
|
|
|
567
1258
|
|
|
568
1259
|
|
|
569
1260
|
def set_hostname_cmd(hostname: str, /) -> list[str]:
|
|
1261
|
+
"""Command to set the system hostname."""
|
|
570
1262
|
return ["hostnamectl", "set-hostname", hostname]
|
|
571
1263
|
|
|
572
1264
|
|
|
@@ -578,10 +1270,12 @@ def ssh(
|
|
|
578
1270
|
user: str,
|
|
579
1271
|
hostname: str,
|
|
580
1272
|
/,
|
|
581
|
-
*
|
|
1273
|
+
*cmd_and_args: str,
|
|
582
1274
|
batch_mode: bool = True,
|
|
583
1275
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
584
1276
|
strict_host_key_checking: bool = True,
|
|
1277
|
+
port: int | None = None,
|
|
1278
|
+
env: StrStrMapping | None = None,
|
|
585
1279
|
input: str | None = None,
|
|
586
1280
|
print: bool = False,
|
|
587
1281
|
print_stdout: bool = False,
|
|
@@ -597,10 +1291,12 @@ def ssh(
|
|
|
597
1291
|
user: str,
|
|
598
1292
|
hostname: str,
|
|
599
1293
|
/,
|
|
600
|
-
*
|
|
1294
|
+
*cmd_and_args: str,
|
|
601
1295
|
batch_mode: bool = True,
|
|
602
1296
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
603
1297
|
strict_host_key_checking: bool = True,
|
|
1298
|
+
port: int | None = None,
|
|
1299
|
+
env: StrStrMapping | None = None,
|
|
604
1300
|
input: str | None = None,
|
|
605
1301
|
print: bool = False,
|
|
606
1302
|
print_stdout: bool = False,
|
|
@@ -616,10 +1312,12 @@ def ssh(
|
|
|
616
1312
|
user: str,
|
|
617
1313
|
hostname: str,
|
|
618
1314
|
/,
|
|
619
|
-
*
|
|
1315
|
+
*cmd_and_args: str,
|
|
620
1316
|
batch_mode: bool = True,
|
|
621
1317
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
622
1318
|
strict_host_key_checking: bool = True,
|
|
1319
|
+
port: int | None = None,
|
|
1320
|
+
env: StrStrMapping | None = None,
|
|
623
1321
|
input: str | None = None,
|
|
624
1322
|
print: bool = False,
|
|
625
1323
|
print_stdout: bool = False,
|
|
@@ -635,10 +1333,12 @@ def ssh(
|
|
|
635
1333
|
user: str,
|
|
636
1334
|
hostname: str,
|
|
637
1335
|
/,
|
|
638
|
-
*
|
|
1336
|
+
*cmd_and_args: str,
|
|
639
1337
|
batch_mode: bool = True,
|
|
640
1338
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
641
1339
|
strict_host_key_checking: bool = True,
|
|
1340
|
+
port: int | None = None,
|
|
1341
|
+
env: StrStrMapping | None = None,
|
|
642
1342
|
input: str | None = None,
|
|
643
1343
|
print: bool = False,
|
|
644
1344
|
print_stdout: bool = False,
|
|
@@ -654,10 +1354,12 @@ def ssh(
|
|
|
654
1354
|
user: str,
|
|
655
1355
|
hostname: str,
|
|
656
1356
|
/,
|
|
657
|
-
*
|
|
1357
|
+
*cmd_and_args: str,
|
|
658
1358
|
batch_mode: bool = True,
|
|
659
1359
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
660
1360
|
strict_host_key_checking: bool = True,
|
|
1361
|
+
port: int | None = None,
|
|
1362
|
+
env: StrStrMapping | None = None,
|
|
661
1363
|
input: str | None = None,
|
|
662
1364
|
print: bool = False,
|
|
663
1365
|
print_stdout: bool = False,
|
|
@@ -672,10 +1374,12 @@ def ssh(
|
|
|
672
1374
|
user: str,
|
|
673
1375
|
hostname: str,
|
|
674
1376
|
/,
|
|
675
|
-
*
|
|
1377
|
+
*cmd_and_args: str,
|
|
676
1378
|
batch_mode: bool = True,
|
|
677
1379
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
678
1380
|
strict_host_key_checking: bool = True,
|
|
1381
|
+
port: int | None = None,
|
|
1382
|
+
env: StrStrMapping | None = None,
|
|
679
1383
|
input: str | None = None, # noqa: A002
|
|
680
1384
|
print: bool = False, # noqa: A002
|
|
681
1385
|
print_stdout: bool = False,
|
|
@@ -686,49 +1390,91 @@ def ssh(
|
|
|
686
1390
|
retry: Retry | None = None,
|
|
687
1391
|
logger: LoggerLike | None = None,
|
|
688
1392
|
) -> str | None:
|
|
689
|
-
|
|
1393
|
+
"""Execute a command on a remote machine."""
|
|
1394
|
+
run_cmd_and_args = ssh_cmd( # skipif-ci
|
|
690
1395
|
user,
|
|
691
1396
|
hostname,
|
|
692
|
-
*
|
|
1397
|
+
*cmd_and_args,
|
|
693
1398
|
batch_mode=batch_mode,
|
|
694
1399
|
host_key_algorithms=host_key_algorithms,
|
|
695
1400
|
strict_host_key_checking=strict_host_key_checking,
|
|
1401
|
+
port=port,
|
|
1402
|
+
env=env,
|
|
696
1403
|
)
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
1404
|
+
try: # skipif-ci
|
|
1405
|
+
return run(
|
|
1406
|
+
*run_cmd_and_args,
|
|
1407
|
+
input=input,
|
|
1408
|
+
print=print,
|
|
1409
|
+
print_stdout=print_stdout,
|
|
1410
|
+
print_stderr=print_stderr,
|
|
1411
|
+
return_=return_,
|
|
1412
|
+
return_stdout=return_stdout,
|
|
1413
|
+
return_stderr=return_stderr,
|
|
1414
|
+
retry=retry,
|
|
1415
|
+
retry_skip=_ssh_retry_skip,
|
|
1416
|
+
logger=logger,
|
|
1417
|
+
)
|
|
1418
|
+
except CalledProcessError as error: # skipif-ci
|
|
1419
|
+
if not _ssh_is_strict_checking_error(error.stderr):
|
|
1420
|
+
raise
|
|
1421
|
+
ssh_keyscan(hostname, port=port)
|
|
1422
|
+
return ssh(
|
|
1423
|
+
user,
|
|
1424
|
+
hostname,
|
|
1425
|
+
*cmd_and_args,
|
|
1426
|
+
batch_mode=batch_mode,
|
|
1427
|
+
host_key_algorithms=host_key_algorithms,
|
|
1428
|
+
strict_host_key_checking=strict_host_key_checking,
|
|
1429
|
+
port=port,
|
|
1430
|
+
input=input,
|
|
1431
|
+
print=print,
|
|
1432
|
+
print_stdout=print_stdout,
|
|
1433
|
+
print_stderr=print_stderr,
|
|
1434
|
+
return_=return_,
|
|
1435
|
+
return_stdout=return_stdout,
|
|
1436
|
+
return_stderr=return_stderr,
|
|
1437
|
+
retry=retry,
|
|
1438
|
+
logger=logger,
|
|
1439
|
+
)
|
|
709
1440
|
|
|
710
1441
|
|
|
711
|
-
|
|
1442
|
+
def _ssh_retry_skip(return_code: int, stdout: str, stderr: str, /) -> bool:
|
|
1443
|
+
_ = (return_code, stdout)
|
|
1444
|
+
return _ssh_is_strict_checking_error(stderr)
|
|
1445
|
+
|
|
1446
|
+
|
|
1447
|
+
def _ssh_is_strict_checking_error(text: str, /) -> bool:
|
|
1448
|
+
match = search(
|
|
1449
|
+
"(Host key for .* has changed|No ED25519 host key is known for .*) and you have requested strict checking",
|
|
1450
|
+
text,
|
|
1451
|
+
flags=MULTILINE,
|
|
1452
|
+
)
|
|
1453
|
+
return match is not None
|
|
712
1454
|
|
|
713
1455
|
|
|
714
1456
|
def ssh_cmd(
|
|
715
1457
|
user: str,
|
|
716
1458
|
hostname: str,
|
|
717
1459
|
/,
|
|
718
|
-
*
|
|
1460
|
+
*cmd_and_args: str,
|
|
719
1461
|
batch_mode: bool = True,
|
|
720
1462
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
721
1463
|
strict_host_key_checking: bool = True,
|
|
1464
|
+
port: int | None = None,
|
|
1465
|
+
env: StrStrMapping | None = None,
|
|
722
1466
|
) -> list[str]:
|
|
1467
|
+
"""Command to use 'ssh' to execute a command on a remote machine."""
|
|
723
1468
|
args: list[str] = ssh_opts_cmd(
|
|
724
1469
|
batch_mode=batch_mode,
|
|
725
1470
|
host_key_algorithms=host_key_algorithms,
|
|
726
1471
|
strict_host_key_checking=strict_host_key_checking,
|
|
1472
|
+
port=port,
|
|
727
1473
|
)
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
1474
|
+
args.append(f"{user}@{hostname}")
|
|
1475
|
+
if env is not None:
|
|
1476
|
+
args.extend(env_cmds(env))
|
|
1477
|
+
return [*args, *cmd_and_args]
|
|
732
1478
|
|
|
733
1479
|
|
|
734
1480
|
def ssh_opts_cmd(
|
|
@@ -736,55 +1482,333 @@ def ssh_opts_cmd(
|
|
|
736
1482
|
batch_mode: bool = True,
|
|
737
1483
|
host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
|
|
738
1484
|
strict_host_key_checking: bool = True,
|
|
1485
|
+
port: int | None = None,
|
|
739
1486
|
) -> list[str]:
|
|
1487
|
+
"""Command to use prepare 'ssh' to execute a command on a remote machine."""
|
|
740
1488
|
args: list[str] = ["ssh"]
|
|
741
1489
|
if batch_mode:
|
|
742
1490
|
args.extend(["-o", "BatchMode=yes"])
|
|
743
1491
|
args.extend(["-o", f"HostKeyAlgorithms={','.join(host_key_algorithms)}"])
|
|
744
1492
|
if strict_host_key_checking:
|
|
745
1493
|
args.extend(["-o", "StrictHostKeyChecking=yes"])
|
|
1494
|
+
if port is not None:
|
|
1495
|
+
args.extend(["-p", str(port)])
|
|
746
1496
|
return [*args, "-T"]
|
|
747
1497
|
|
|
748
1498
|
|
|
749
1499
|
##
|
|
750
1500
|
|
|
751
1501
|
|
|
752
|
-
def
|
|
753
|
-
|
|
1502
|
+
def ssh_await(
|
|
1503
|
+
user: str,
|
|
1504
|
+
hostname: str,
|
|
1505
|
+
/,
|
|
1506
|
+
*,
|
|
1507
|
+
logger: LoggerLike | None = None,
|
|
1508
|
+
delta: Delta = SECOND,
|
|
1509
|
+
) -> None:
|
|
1510
|
+
while True: # skipif-ci
|
|
1511
|
+
if logger is not None:
|
|
1512
|
+
to_logger(logger).info("Waiting for '%s'...", hostname)
|
|
1513
|
+
try:
|
|
1514
|
+
ssh(user, hostname, "true")
|
|
1515
|
+
except CalledProcessError:
|
|
1516
|
+
sleep(to_seconds(delta))
|
|
1517
|
+
else:
|
|
1518
|
+
if logger is not None:
|
|
1519
|
+
to_logger(logger).info("'%s' is up", hostname)
|
|
1520
|
+
return
|
|
1521
|
+
|
|
1522
|
+
|
|
1523
|
+
##
|
|
1524
|
+
|
|
1525
|
+
|
|
1526
|
+
def ssh_keyscan(
|
|
1527
|
+
hostname: str,
|
|
1528
|
+
/,
|
|
1529
|
+
*,
|
|
1530
|
+
path: PathLike = KNOWN_HOSTS,
|
|
1531
|
+
retry: Retry | None = None,
|
|
1532
|
+
port: int | None = None,
|
|
1533
|
+
) -> None:
|
|
1534
|
+
"""Add a known host."""
|
|
1535
|
+
ssh_keygen_remove(hostname, path=path, retry=retry) # skipif-ci
|
|
1536
|
+
result = run( # skipif-ci
|
|
1537
|
+
*ssh_keyscan_cmd(hostname, port=port), return_=True, retry=retry
|
|
1538
|
+
)
|
|
1539
|
+
tee(path, result, append=True) # skipif-ci
|
|
1540
|
+
|
|
1541
|
+
|
|
1542
|
+
def ssh_keyscan_cmd(hostname: str, /, *, port: int | None = None) -> list[str]:
|
|
1543
|
+
"""Command to use 'ssh-keyscan' to add a known host."""
|
|
1544
|
+
args: list[str] = ["ssh-keyscan"]
|
|
1545
|
+
if port is not None:
|
|
1546
|
+
args.extend(["-p", str(port)])
|
|
1547
|
+
return [*args, "-q", "-t", "ed25519", hostname]
|
|
1548
|
+
|
|
1549
|
+
|
|
1550
|
+
##
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
def ssh_keygen_remove(
|
|
1554
|
+
hostname: str, /, *, path: PathLike = KNOWN_HOSTS, retry: Retry | None = None
|
|
1555
|
+
) -> None:
|
|
1556
|
+
"""Remove a known host."""
|
|
1557
|
+
path = Path(path)
|
|
1558
|
+
if path.exists():
|
|
1559
|
+
run(*ssh_keygen_remove_cmd(hostname, path=path), retry=retry)
|
|
1560
|
+
|
|
1561
|
+
|
|
1562
|
+
def ssh_keygen_remove_cmd(
|
|
1563
|
+
hostname: str, /, *, path: PathLike = KNOWN_HOSTS
|
|
1564
|
+
) -> list[str]:
|
|
1565
|
+
"""Command to use 'ssh-keygen' to remove a known host."""
|
|
1566
|
+
return ["ssh-keygen", "-f", str(path), "-R", hostname]
|
|
754
1567
|
|
|
755
1568
|
|
|
756
1569
|
##
|
|
757
1570
|
|
|
758
1571
|
|
|
759
1572
|
def sudo_cmd(cmd: str, /, *args: str) -> list[str]:
|
|
1573
|
+
"""Command to use 'sudo' to execute a command as another user."""
|
|
760
1574
|
return ["sudo", cmd, *args]
|
|
761
1575
|
|
|
762
1576
|
|
|
1577
|
+
def maybe_sudo_cmd(cmd: str, /, *args: str, sudo: bool = False) -> list[str]:
|
|
1578
|
+
"""Command to use 'sudo' to execute a command as another user, if required."""
|
|
1579
|
+
parts: list[str] = [cmd, *args]
|
|
1580
|
+
return sudo_cmd(*parts) if sudo else parts
|
|
1581
|
+
|
|
1582
|
+
|
|
763
1583
|
##
|
|
764
1584
|
|
|
765
1585
|
|
|
766
1586
|
def sudo_nopasswd_cmd(user: str, /) -> str:
|
|
1587
|
+
"""Command to allow a user to use password-free `sudo`."""
|
|
767
1588
|
return f"{user} ALL=(ALL) NOPASSWD: ALL"
|
|
768
1589
|
|
|
769
1590
|
|
|
770
1591
|
##
|
|
771
1592
|
|
|
772
1593
|
|
|
773
|
-
def
|
|
774
|
-
|
|
1594
|
+
def symlink(target: PathLike, link: PathLike, /, *, sudo: bool = False) -> None:
|
|
1595
|
+
"""Make a symbolic link."""
|
|
1596
|
+
rm(link, sudo=sudo)
|
|
1597
|
+
mkdir(link, sudo=sudo, parent=True)
|
|
1598
|
+
if sudo: # pragma: no cover
|
|
1599
|
+
run(*sudo_cmd(*symlink_cmd(target, link)))
|
|
1600
|
+
else:
|
|
1601
|
+
target, link = map(Path, [target, link])
|
|
1602
|
+
link.symlink_to(target)
|
|
1603
|
+
|
|
1604
|
+
|
|
1605
|
+
def symlink_cmd(target: PathLike, link: PathLike, /) -> list[str]:
|
|
1606
|
+
"""Command to use 'symlink' to make a symbolic link."""
|
|
1607
|
+
return ["ln", "-s", str(target), str(link)]
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
##
|
|
1611
|
+
|
|
1612
|
+
|
|
1613
|
+
def tee(
|
|
1614
|
+
path: PathLike, text: str, /, *, sudo: bool = False, append: bool = False
|
|
1615
|
+
) -> None:
|
|
1616
|
+
"""Duplicate standard input."""
|
|
1617
|
+
mkdir(path, sudo=sudo, parent=True)
|
|
1618
|
+
if sudo: # pragma: no cover
|
|
1619
|
+
run(*sudo_cmd(*tee_cmd(path, append=append)), input=text)
|
|
1620
|
+
else:
|
|
1621
|
+
path = Path(path)
|
|
1622
|
+
with path.open(mode="a" if append else "w") as fh:
|
|
1623
|
+
_ = fh.write(text)
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
def tee_cmd(path: PathLike, /, *, append: bool = False) -> list[str]:
|
|
1627
|
+
"""Command to use 'tee' to duplicate standard input."""
|
|
1628
|
+
args: list[str] = ["tee"]
|
|
1629
|
+
if append:
|
|
1630
|
+
args.append("-a")
|
|
1631
|
+
return [*args, str(path)]
|
|
775
1632
|
|
|
776
1633
|
|
|
777
1634
|
##
|
|
778
1635
|
|
|
779
1636
|
|
|
1637
|
+
def touch(path: PathLike, /, *, sudo: bool = False) -> None:
|
|
1638
|
+
"""Change file access and modification times."""
|
|
1639
|
+
run(*maybe_sudo_cmd(*touch_cmd(path), sudo=sudo))
|
|
1640
|
+
|
|
1641
|
+
|
|
780
1642
|
def touch_cmd(path: PathLike, /) -> list[str]:
|
|
1643
|
+
"""Command to use 'touch' to change file access and modification times."""
|
|
781
1644
|
return ["touch", str(path)]
|
|
782
1645
|
|
|
783
1646
|
|
|
784
1647
|
##
|
|
785
1648
|
|
|
786
1649
|
|
|
1650
|
+
def update_ca_certificates(*, sudo: bool = False) -> None:
|
|
1651
|
+
"""Update the system CA certificates."""
|
|
1652
|
+
run(*maybe_sudo_cmd(UPDATE_CA_CERTIFICATES, sudo=sudo)) # pragma: no cover
|
|
1653
|
+
|
|
1654
|
+
|
|
1655
|
+
##
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def useradd(
|
|
1659
|
+
login: str,
|
|
1660
|
+
/,
|
|
1661
|
+
*,
|
|
1662
|
+
create_home: bool = True,
|
|
1663
|
+
groups: MaybeIterable[str] | None = None,
|
|
1664
|
+
shell: PathLike | None = None,
|
|
1665
|
+
sudo: bool = False,
|
|
1666
|
+
password: str | None = None,
|
|
1667
|
+
) -> None:
|
|
1668
|
+
"""Create a new user."""
|
|
1669
|
+
args = maybe_sudo_cmd( # pragma: no cover
|
|
1670
|
+
*useradd_cmd(login, create_home=create_home, groups=groups, shell=shell)
|
|
1671
|
+
)
|
|
1672
|
+
run(*args) # pragma: no cover
|
|
1673
|
+
if password is not None: # pragma: no cover
|
|
1674
|
+
chpasswd(login, password, sudo=sudo)
|
|
1675
|
+
|
|
1676
|
+
|
|
1677
|
+
def useradd_cmd(
|
|
1678
|
+
login: str,
|
|
1679
|
+
/,
|
|
1680
|
+
*,
|
|
1681
|
+
create_home: bool = True,
|
|
1682
|
+
groups: MaybeIterable[str] | None = None,
|
|
1683
|
+
shell: PathLike | None = None,
|
|
1684
|
+
) -> list[str]:
|
|
1685
|
+
"""Command to use 'useradd' to create a new user."""
|
|
1686
|
+
args: list[str] = ["useradd"]
|
|
1687
|
+
if create_home:
|
|
1688
|
+
args.append("--create-home")
|
|
1689
|
+
if groups is not None:
|
|
1690
|
+
args.extend(["--groups", *always_iterable(groups)])
|
|
1691
|
+
if shell is not None:
|
|
1692
|
+
args.extend(["--shell", str(shell)])
|
|
1693
|
+
return [*args, login]
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
##
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
@overload
|
|
1700
|
+
def uv_run(
|
|
1701
|
+
module: str,
|
|
1702
|
+
/,
|
|
1703
|
+
*args: str,
|
|
1704
|
+
env: StrStrMapping | None = None,
|
|
1705
|
+
cwd: PathLike | None = None,
|
|
1706
|
+
print: bool = False,
|
|
1707
|
+
print_stdout: bool = False,
|
|
1708
|
+
print_stderr: bool = False,
|
|
1709
|
+
return_: Literal[True],
|
|
1710
|
+
return_stdout: bool = False,
|
|
1711
|
+
return_stderr: bool = False,
|
|
1712
|
+
retry: Retry | None = None,
|
|
1713
|
+
logger: LoggerLike | None = None,
|
|
1714
|
+
) -> str: ...
|
|
1715
|
+
@overload
|
|
1716
|
+
def uv_run(
|
|
1717
|
+
module: str,
|
|
1718
|
+
/,
|
|
1719
|
+
*args: str,
|
|
1720
|
+
env: StrStrMapping | None = None,
|
|
1721
|
+
cwd: PathLike | None = None,
|
|
1722
|
+
print: bool = False,
|
|
1723
|
+
print_stdout: bool = False,
|
|
1724
|
+
print_stderr: bool = False,
|
|
1725
|
+
return_: bool = False,
|
|
1726
|
+
return_stdout: Literal[True],
|
|
1727
|
+
return_stderr: bool = False,
|
|
1728
|
+
retry: Retry | None = None,
|
|
1729
|
+
logger: LoggerLike | None = None,
|
|
1730
|
+
) -> str: ...
|
|
1731
|
+
@overload
|
|
1732
|
+
def uv_run(
|
|
1733
|
+
module: str,
|
|
1734
|
+
/,
|
|
1735
|
+
*args: str,
|
|
1736
|
+
env: StrStrMapping | None = None,
|
|
1737
|
+
cwd: PathLike | None = None,
|
|
1738
|
+
print: bool = False,
|
|
1739
|
+
print_stdout: bool = False,
|
|
1740
|
+
print_stderr: bool = False,
|
|
1741
|
+
return_: bool = False,
|
|
1742
|
+
return_stdout: bool = False,
|
|
1743
|
+
return_stderr: Literal[True],
|
|
1744
|
+
retry: Retry | None = None,
|
|
1745
|
+
logger: LoggerLike | None = None,
|
|
1746
|
+
) -> str: ...
|
|
1747
|
+
@overload
|
|
1748
|
+
def uv_run(
|
|
1749
|
+
module: str,
|
|
1750
|
+
/,
|
|
1751
|
+
*args: str,
|
|
1752
|
+
env: StrStrMapping | None = None,
|
|
1753
|
+
cwd: PathLike | None = None,
|
|
1754
|
+
print: bool = False,
|
|
1755
|
+
print_stdout: bool = False,
|
|
1756
|
+
print_stderr: bool = False,
|
|
1757
|
+
return_: Literal[False] = False,
|
|
1758
|
+
return_stdout: Literal[False] = False,
|
|
1759
|
+
return_stderr: Literal[False] = False,
|
|
1760
|
+
retry: Retry | None = None,
|
|
1761
|
+
logger: LoggerLike | None = None,
|
|
1762
|
+
) -> None: ...
|
|
1763
|
+
@overload
|
|
1764
|
+
def uv_run(
|
|
1765
|
+
module: str,
|
|
1766
|
+
/,
|
|
1767
|
+
*args: str,
|
|
1768
|
+
env: StrStrMapping | None = None,
|
|
1769
|
+
cwd: PathLike | None = None,
|
|
1770
|
+
print: bool = False,
|
|
1771
|
+
print_stdout: bool = False,
|
|
1772
|
+
print_stderr: bool = False,
|
|
1773
|
+
return_: bool = False,
|
|
1774
|
+
return_stdout: bool = False,
|
|
1775
|
+
return_stderr: bool = False,
|
|
1776
|
+
retry: Retry | None = None,
|
|
1777
|
+
logger: LoggerLike | None = None,
|
|
1778
|
+
) -> str | None: ...
|
|
1779
|
+
def uv_run(
|
|
1780
|
+
module: str,
|
|
1781
|
+
/,
|
|
1782
|
+
*args: str,
|
|
1783
|
+
cwd: PathLike | None = None,
|
|
1784
|
+
env: StrStrMapping | None = None,
|
|
1785
|
+
print: bool = False, # noqa: A002
|
|
1786
|
+
print_stdout: bool = False,
|
|
1787
|
+
print_stderr: bool = False,
|
|
1788
|
+
return_: bool = False,
|
|
1789
|
+
return_stdout: bool = False,
|
|
1790
|
+
return_stderr: bool = False,
|
|
1791
|
+
retry: Retry | None = None,
|
|
1792
|
+
logger: LoggerLike | None = None,
|
|
1793
|
+
) -> str | None:
|
|
1794
|
+
"""Run a command or script."""
|
|
1795
|
+
return run( # pragma: no cover
|
|
1796
|
+
*uv_run_cmd(module, *args),
|
|
1797
|
+
cwd=cwd,
|
|
1798
|
+
env=env,
|
|
1799
|
+
print=print,
|
|
1800
|
+
print_stdout=print_stdout,
|
|
1801
|
+
print_stderr=print_stderr,
|
|
1802
|
+
return_=return_,
|
|
1803
|
+
return_stdout=return_stdout,
|
|
1804
|
+
return_stderr=return_stderr,
|
|
1805
|
+
retry=retry,
|
|
1806
|
+
logger=logger,
|
|
1807
|
+
)
|
|
1808
|
+
|
|
1809
|
+
|
|
787
1810
|
def uv_run_cmd(module: str, /, *args: str) -> list[str]:
|
|
1811
|
+
"""Command to use 'uv' to run a command or script."""
|
|
788
1812
|
return [
|
|
789
1813
|
"uv",
|
|
790
1814
|
"run",
|
|
@@ -802,6 +1826,17 @@ def uv_run_cmd(module: str, /, *args: str) -> list[str]:
|
|
|
802
1826
|
##
|
|
803
1827
|
|
|
804
1828
|
|
|
1829
|
+
@contextmanager
|
|
1830
|
+
def yield_git_repo(url: str, /, *, branch: str | None = None) -> Iterator[Path]:
|
|
1831
|
+
"""Yield a temporary git repository."""
|
|
1832
|
+
with TemporaryDirectory() as temp_dir:
|
|
1833
|
+
git_clone(url, temp_dir, branch=branch)
|
|
1834
|
+
yield temp_dir
|
|
1835
|
+
|
|
1836
|
+
|
|
1837
|
+
##
|
|
1838
|
+
|
|
1839
|
+
|
|
805
1840
|
@contextmanager
|
|
806
1841
|
def yield_ssh_temp_dir(
|
|
807
1842
|
user: str,
|
|
@@ -812,6 +1847,7 @@ def yield_ssh_temp_dir(
|
|
|
812
1847
|
logger: LoggerLike | None = None,
|
|
813
1848
|
keep: bool = False,
|
|
814
1849
|
) -> Iterator[Path]:
|
|
1850
|
+
"""Yield a temporary directory on a remote machine."""
|
|
815
1851
|
path = Path( # skipif-ci
|
|
816
1852
|
ssh(user, hostname, *MKTEMP_DIR_CMD, return_=True, retry=retry, logger=logger)
|
|
817
1853
|
)
|
|
@@ -829,36 +1865,83 @@ __all__ = [
|
|
|
829
1865
|
"APT_UPDATE",
|
|
830
1866
|
"BASH_LC",
|
|
831
1867
|
"BASH_LS",
|
|
1868
|
+
"CHPASSWD",
|
|
1869
|
+
"GIT_BRANCH_SHOW_CURRENT",
|
|
832
1870
|
"MKTEMP_DIR_CMD",
|
|
833
1871
|
"RESTART_SSHD",
|
|
834
1872
|
"UPDATE_CA_CERTIFICATES",
|
|
835
1873
|
"ChownCmdError",
|
|
1874
|
+
"CpError",
|
|
1875
|
+
"MvFileError",
|
|
1876
|
+
"RsyncCmdError",
|
|
1877
|
+
"RsyncCmdNoSourcesError",
|
|
1878
|
+
"RsyncCmdSourcesNotFoundError",
|
|
1879
|
+
"append_text",
|
|
1880
|
+
"apt_install",
|
|
836
1881
|
"apt_install_cmd",
|
|
1882
|
+
"apt_remove",
|
|
1883
|
+
"apt_remove_cmd",
|
|
1884
|
+
"apt_update",
|
|
1885
|
+
"cat",
|
|
837
1886
|
"cd_cmd",
|
|
1887
|
+
"chattr",
|
|
1888
|
+
"chattr_cmd",
|
|
1889
|
+
"chmod",
|
|
838
1890
|
"chmod_cmd",
|
|
1891
|
+
"chown",
|
|
839
1892
|
"chown_cmd",
|
|
1893
|
+
"chpasswd",
|
|
1894
|
+
"copy_text",
|
|
1895
|
+
"cp",
|
|
840
1896
|
"cp_cmd",
|
|
1897
|
+
"curl",
|
|
1898
|
+
"curl_cmd",
|
|
841
1899
|
"echo_cmd",
|
|
1900
|
+
"env_cmds",
|
|
842
1901
|
"expand_path",
|
|
1902
|
+
"git_branch_current",
|
|
1903
|
+
"git_checkout",
|
|
1904
|
+
"git_checkout_cmd",
|
|
1905
|
+
"git_clone",
|
|
843
1906
|
"git_clone_cmd",
|
|
844
|
-
"
|
|
1907
|
+
"install",
|
|
1908
|
+
"install_cmd",
|
|
845
1909
|
"maybe_parent",
|
|
846
1910
|
"maybe_sudo_cmd",
|
|
847
1911
|
"mkdir",
|
|
848
1912
|
"mkdir_cmd",
|
|
1913
|
+
"mv",
|
|
849
1914
|
"mv_cmd",
|
|
1915
|
+
"replace_text",
|
|
1916
|
+
"ripgrep",
|
|
1917
|
+
"ripgrep_cmd",
|
|
1918
|
+
"rm",
|
|
850
1919
|
"rm_cmd",
|
|
851
1920
|
"rsync",
|
|
852
1921
|
"rsync_cmd",
|
|
1922
|
+
"rsync_many",
|
|
853
1923
|
"run",
|
|
854
1924
|
"set_hostname_cmd",
|
|
855
1925
|
"ssh",
|
|
1926
|
+
"ssh_await",
|
|
856
1927
|
"ssh_cmd",
|
|
1928
|
+
"ssh_keygen_remove",
|
|
1929
|
+
"ssh_keygen_remove_cmd",
|
|
1930
|
+
"ssh_keyscan",
|
|
1931
|
+
"ssh_keyscan_cmd",
|
|
857
1932
|
"ssh_opts_cmd",
|
|
858
1933
|
"sudo_cmd",
|
|
859
1934
|
"sudo_nopasswd_cmd",
|
|
1935
|
+
"symlink",
|
|
860
1936
|
"symlink_cmd",
|
|
1937
|
+
"tee_cmd",
|
|
1938
|
+
"touch",
|
|
861
1939
|
"touch_cmd",
|
|
1940
|
+
"update_ca_certificates",
|
|
1941
|
+
"useradd",
|
|
1942
|
+
"useradd_cmd",
|
|
1943
|
+
"uv_run",
|
|
862
1944
|
"uv_run_cmd",
|
|
1945
|
+
"yield_git_repo",
|
|
863
1946
|
"yield_ssh_temp_dir",
|
|
864
1947
|
]
|