dycw-utilities 0.174.12__py3-none-any.whl → 0.175.31__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dycw-utilities might be problematic. Click here for more details.

utilities/subprocess.py CHANGED
@@ -1,11 +1,15 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import shutil
3
4
  import sys
4
5
  from contextlib import contextmanager
5
6
  from dataclasses import dataclass
6
7
  from io import StringIO
8
+ from itertools import repeat
7
9
  from pathlib import Path
10
+ from re import MULTILINE, search
8
11
  from shlex import join
12
+ from shutil import copyfile, copytree, move, rmtree
9
13
  from string import Template
10
14
  from subprocess import PIPE, CalledProcessError, Popen
11
15
  from threading import Thread
@@ -15,13 +19,18 @@ from typing import IO, TYPE_CHECKING, Literal, assert_never, overload, override
15
19
  from utilities.errors import ImpossibleCaseError
16
20
  from utilities.iterables import always_iterable
17
21
  from utilities.logging import to_logger
22
+ from utilities.pathlib import PWD
23
+ from utilities.permissions import Permissions, ensure_perms
24
+ from utilities.tempfile import TemporaryDirectory
18
25
  from utilities.text import strip_and_dedent
19
- from utilities.whenever import to_seconds
26
+ from utilities.whenever import SECOND, to_seconds
20
27
 
21
28
  if TYPE_CHECKING:
22
- from collections.abc import Iterator
29
+ from collections.abc import Callable, Iterator
23
30
 
31
+ from utilities.permissions import PermissionsLike
24
32
  from utilities.types import (
33
+ Delta,
25
34
  LoggerLike,
26
35
  MaybeIterable,
27
36
  PathLike,
@@ -35,6 +44,9 @@ _HOST_KEY_ALGORITHMS = ["ssh-ed25519"]
35
44
  APT_UPDATE = ["apt", "update", "-y"]
36
45
  BASH_LC = ["bash", "-lc"]
37
46
  BASH_LS = ["bash", "-ls"]
47
+ CHPASSWD = "chpasswd"
48
+ GIT_BRANCH_SHOW_CURRENT = ["git", "branch", "--show-current"]
49
+ KNOWN_HOSTS = Path.home() / ".ssh/known_hosts"
38
50
  MKTEMP_DIR_CMD = ["mktemp", "-d"]
39
51
  RESTART_SSHD = ["systemctl", "restart", "sshd"]
40
52
  UPDATE_CA_CERTIFICATES: str = "update-ca-certificates"
@@ -43,45 +55,181 @@ UPDATE_CA_CERTIFICATES: str = "update-ca-certificates"
43
55
  ##
44
56
 
45
57
 
46
- def apt_install_cmd(package: str, /) -> list[str]:
47
- return ["apt", "install", "-y", package]
58
+ def append_text(
59
+ path: PathLike,
60
+ text: str,
61
+ /,
62
+ *,
63
+ sudo: bool = False,
64
+ skip_if_present: bool = False,
65
+ flags: int = 0,
66
+ blank_lines: int = 1,
67
+ ) -> None:
68
+ """Append text to a file."""
69
+ try:
70
+ existing = cat(path, sudo=sudo)
71
+ except (CalledProcessError, FileNotFoundError):
72
+ tee(path, text, sudo=sudo, append=True)
73
+ return
74
+ if skip_if_present and (search(text, existing, flags=flags) is not None):
75
+ return
76
+ full = "".join([*repeat("\n", times=blank_lines), text])
77
+ tee(path, full, sudo=sudo, append=True)
48
78
 
49
79
 
50
80
  ##
51
81
 
52
82
 
53
- def cat_cmd(path: PathLike, /) -> list[str]:
54
- return ["cat", str(path)]
83
+ def apt_install(
84
+ package: str, /, *packages: str, update: bool = False, sudo: bool = False
85
+ ) -> None:
86
+ """Install packages."""
87
+ if update: # pragma: no cover
88
+ apt_update(sudo=sudo)
89
+ args = maybe_sudo_cmd( # pragma: no cover
90
+ *apt_install_cmd(package, *packages), sudo=sudo
91
+ )
92
+ run(*args) # pragma: no cover
93
+
94
+
95
+ def apt_install_cmd(package: str, /, *packages: str) -> list[str]:
96
+ """Command to use 'apt' to install packages."""
97
+ return ["apt", "install", "-y", package, *packages]
98
+
99
+
100
+ ##
101
+
102
+
103
+ def apt_remove(package: str, /, *packages: str, sudo: bool = False) -> None:
104
+ """Remove a package."""
105
+ args = maybe_sudo_cmd( # pragma: no cover
106
+ *apt_remove_cmd(package, *packages), sudo=sudo
107
+ )
108
+ run(*args) # pragma: no cover
109
+
110
+
111
+ def apt_remove_cmd(package: str, /, *packages: str) -> list[str]:
112
+ """Command to use 'apt' to remove packages."""
113
+ return ["apt", "remove", "-y", package, *packages]
114
+
115
+
116
+ ##
117
+
118
+
119
+ def apt_update(*, sudo: bool = False) -> None:
120
+ """Update 'apt'."""
121
+ run(*maybe_sudo_cmd(*APT_UPDATE, sudo=sudo))
122
+
123
+
124
+ ##
125
+
126
+
127
+ def cat(path: PathLike, /, *paths: PathLike, sudo: bool = False) -> str:
128
+ """Concatenate and print files."""
129
+ if sudo: # pragma: no cover
130
+ return run(*sudo_cmd(*cat_cmd(path, *paths)), return_=True)
131
+ all_paths = list(map(Path, [path, *paths]))
132
+ return "\n".join(p.read_text() for p in all_paths)
133
+
134
+
135
+ def cat_cmd(path: PathLike, /, *paths: PathLike) -> list[str]:
136
+ """Command to use 'cat' to concatenate and print files."""
137
+ return ["cat", str(path), *map(str, paths)]
55
138
 
56
139
 
57
140
  ##
58
141
 
59
142
 
60
143
  def cd_cmd(path: PathLike, /) -> list[str]:
144
+ """Command to use 'cd' to change working directory."""
61
145
  return ["cd", str(path)]
62
146
 
63
147
 
64
148
  ##
65
149
 
66
150
 
67
- def chmod_cmd(path: PathLike, mode: str, /) -> list[str]:
68
- return ["chmod", mode, str(path)]
151
+ def chattr(
152
+ path: PathLike, /, *, immutable: bool | None = None, sudo: bool = False
153
+ ) -> None:
154
+ """Change file attributes."""
155
+ args = maybe_sudo_cmd( # pragma: no cover
156
+ *chattr_cmd(path, immutable=immutable), sudo=sudo
157
+ )
158
+ run(*args) # pragma: no cover
159
+
160
+
161
+ def chattr_cmd(path: PathLike, /, *, immutable: bool | None = None) -> list[str]:
162
+ """Command to use 'chattr' to change file attributes."""
163
+ args: list[str] = ["chattr"]
164
+ if immutable is True:
165
+ args.append("+i")
166
+ elif immutable is False:
167
+ args.append("-i")
168
+ return [*args, str(path)]
169
+
170
+
171
+ ##
172
+
173
+
174
+ def chmod(path: PathLike, perms: PermissionsLike, /, *, sudo: bool = False) -> None:
175
+ """Change file mode."""
176
+ if sudo: # pragma: no cover
177
+ run(*sudo_cmd(*chmod_cmd(path, perms)))
178
+ else:
179
+ Path(path).chmod(int(ensure_perms(perms)))
180
+
181
+
182
+ def chmod_cmd(path: PathLike, perms: PermissionsLike, /) -> list[str]:
183
+ """Command to use 'chmod' to change file mode."""
184
+ return ["chmod", str(ensure_perms(perms)), str(path)]
69
185
 
70
186
 
71
187
  ##
72
188
 
73
189
 
190
+ def chown(
191
+ path: PathLike,
192
+ /,
193
+ *,
194
+ sudo: bool = False,
195
+ user: str | int | None = None,
196
+ group: str | int | None = None,
197
+ ) -> None:
198
+ """Change file owner and/or group."""
199
+ if sudo: # pragma: no cover
200
+ match user, group:
201
+ case None, None:
202
+ ...
203
+ case str() | int() | None, str() | int() | None:
204
+ run(*sudo_cmd(*chown_cmd(path, user=user, group=group)))
205
+ case never:
206
+ assert_never(never)
207
+ else:
208
+ match user, group:
209
+ case None, None:
210
+ ...
211
+ case str() | int(), None:
212
+ shutil.chown(path, user, group)
213
+ case None, str() | int():
214
+ shutil.chown(path, user, group)
215
+ case str() | int(), str() | int():
216
+ shutil.chown(path, user, group)
217
+ case never:
218
+ assert_never(never)
219
+
220
+
74
221
  def chown_cmd(
75
- path: PathLike, /, *, user: str | None = None, group: str | None = None
222
+ path: PathLike, /, *, user: str | int | None = None, group: str | int | None = None
76
223
  ) -> list[str]:
224
+ """Command to use 'chown' to change file owner and/or group."""
77
225
  match user, group:
78
226
  case None, None:
79
227
  raise ChownCmdError
80
- case str(), None:
228
+ case str() | int(), None:
81
229
  ownership = "user"
82
- case None, str():
230
+ case None, str() | int():
83
231
  ownership = f":{group}"
84
- case str(), str():
232
+ case str() | int(), str() | int():
85
233
  ownership = f"{user}:{group}"
86
234
  case never:
87
235
  assert_never(never)
@@ -98,23 +246,277 @@ class ChownCmdError(Exception):
98
246
  ##
99
247
 
100
248
 
249
+ def chpasswd(user_name: str, password: str, /, *, sudo: bool = False) -> None:
250
+ """Update passwords."""
251
+ args = maybe_sudo_cmd(CHPASSWD, sudo=sudo) # pragma: no cover
252
+ run(*args, input=f"{user_name}:{password}") # pragma: no cover
253
+
254
+
255
+ ##
256
+
257
+
258
+ def copy_text(
259
+ src: PathLike,
260
+ dest: PathLike,
261
+ /,
262
+ *,
263
+ sudo: bool = False,
264
+ substitutions: StrMapping | None = None,
265
+ ) -> None:
266
+ """Copy the text contents of a file."""
267
+ text = cat(src, sudo=sudo)
268
+ if substitutions is not None:
269
+ text = Template(text).substitute(**substitutions)
270
+ tee(dest, text, sudo=sudo)
271
+
272
+
273
+ ##
274
+
275
+
276
+ def cp(
277
+ src: PathLike,
278
+ dest: PathLike,
279
+ /,
280
+ *,
281
+ sudo: bool = False,
282
+ perms: PermissionsLike | None = None,
283
+ owner: str | int | None = None,
284
+ group: str | int | None = None,
285
+ ) -> None:
286
+ """Copy a file/directory."""
287
+ mkdir(dest, sudo=sudo, parent=True)
288
+ if sudo: # pragma: no cover
289
+ run(*sudo_cmd(*cp_cmd(src, dest)))
290
+ else:
291
+ src, dest = map(Path, [src, dest])
292
+ if src.is_file():
293
+ _ = copyfile(src, dest)
294
+ elif src.is_dir():
295
+ _ = copytree(src, dest, dirs_exist_ok=True)
296
+ else:
297
+ raise CpError(src=src, dest=dest)
298
+ if perms is not None:
299
+ chmod(dest, perms, sudo=sudo)
300
+ if (owner is not None) or (group is not None):
301
+ chown(dest, sudo=sudo, user=owner, group=group)
302
+
303
+
304
+ @dataclass(kw_only=True, slots=True)
305
+ class CpError(Exception):
306
+ src: Path
307
+ dest: Path
308
+
309
+ @override
310
+ def __str__(self) -> str:
311
+ return f"Unable to copy {str(self.src)!r} to {str(self.dest)!r}; source does not exist"
312
+
313
+
101
314
  def cp_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
315
+ """Command to use 'cp' to copy a file/directory."""
102
316
  return ["cp", "-r", str(src), str(dest)]
103
317
 
104
318
 
105
319
  ##
106
320
 
107
321
 
322
+ @overload
323
+ def curl(
324
+ url: str,
325
+ /,
326
+ *,
327
+ fail: bool = True,
328
+ location: bool = True,
329
+ output: PathLike | None = None,
330
+ show_error: bool = True,
331
+ silent: bool = True,
332
+ sudo: bool = False,
333
+ print: bool = False,
334
+ print_stdout: bool = False,
335
+ print_stderr: bool = False,
336
+ return_: Literal[True],
337
+ return_stdout: Literal[False] = False,
338
+ return_stderr: Literal[False] = False,
339
+ retry: Retry | None = None,
340
+ retry_skip: Callable[[int, str, str], bool] | None = None,
341
+ logger: LoggerLike | None = None,
342
+ ) -> str: ...
343
+ @overload
344
+ def curl(
345
+ url: str,
346
+ /,
347
+ *,
348
+ fail: bool = True,
349
+ location: bool = True,
350
+ output: PathLike | None = None,
351
+ show_error: bool = True,
352
+ silent: bool = True,
353
+ sudo: bool = False,
354
+ print: bool = False,
355
+ print_stdout: bool = False,
356
+ print_stderr: bool = False,
357
+ return_: Literal[False] = False,
358
+ return_stdout: Literal[True],
359
+ return_stderr: Literal[False] = False,
360
+ retry: Retry | None = None,
361
+ retry_skip: Callable[[int, str, str], bool] | None = None,
362
+ logger: LoggerLike | None = None,
363
+ ) -> str: ...
364
+ @overload
365
+ def curl(
366
+ url: str,
367
+ /,
368
+ *,
369
+ fail: bool = True,
370
+ location: bool = True,
371
+ output: PathLike | None = None,
372
+ show_error: bool = True,
373
+ silent: bool = True,
374
+ sudo: bool = False,
375
+ print: bool = False,
376
+ print_stdout: bool = False,
377
+ print_stderr: bool = False,
378
+ return_: Literal[False] = False,
379
+ return_stdout: Literal[False] = False,
380
+ return_stderr: Literal[True],
381
+ retry: Retry | None = None,
382
+ retry_skip: Callable[[int, str, str], bool] | None = None,
383
+ logger: LoggerLike | None = None,
384
+ ) -> str: ...
385
+ @overload
386
+ def curl(
387
+ url: str,
388
+ /,
389
+ *,
390
+ fail: bool = True,
391
+ location: bool = True,
392
+ output: PathLike | None = None,
393
+ show_error: bool = True,
394
+ silent: bool = True,
395
+ sudo: bool = False,
396
+ print: bool = False,
397
+ print_stdout: bool = False,
398
+ print_stderr: bool = False,
399
+ return_: Literal[False] = False,
400
+ return_stdout: Literal[False] = False,
401
+ return_stderr: Literal[False] = False,
402
+ retry: Retry | None = None,
403
+ retry_skip: Callable[[int, str, str], bool] | None = None,
404
+ logger: LoggerLike | None = None,
405
+ ) -> None: ...
406
+ @overload
407
+ def curl(
408
+ url: str,
409
+ /,
410
+ *,
411
+ fail: bool = True,
412
+ location: bool = True,
413
+ output: PathLike | None = None,
414
+ show_error: bool = True,
415
+ silent: bool = True,
416
+ sudo: bool = False,
417
+ print: bool = False,
418
+ print_stdout: bool = False,
419
+ print_stderr: bool = False,
420
+ return_: bool = False,
421
+ return_stdout: bool = False,
422
+ return_stderr: bool = False,
423
+ retry: Retry | None = None,
424
+ retry_skip: Callable[[int, str, str], bool] | None = None,
425
+ logger: LoggerLike | None = None,
426
+ ) -> str | None: ...
427
+ def curl(
428
+ url: str,
429
+ /,
430
+ *,
431
+ fail: bool = True,
432
+ location: bool = True,
433
+ output: PathLike | None = None,
434
+ show_error: bool = True,
435
+ silent: bool = True,
436
+ sudo: bool = False,
437
+ print: bool = False, # noqa: A002
438
+ print_stdout: bool = False,
439
+ print_stderr: bool = False,
440
+ return_: bool = False,
441
+ return_stdout: bool = False,
442
+ return_stderr: bool = False,
443
+ retry: Retry | None = None,
444
+ retry_skip: Callable[[int, str, str], bool] | None = None,
445
+ logger: LoggerLike | None = None,
446
+ ) -> str | None:
447
+ """Transfer a URL."""
448
+ args = maybe_sudo_cmd( # skipif-ci
449
+ *curl_cmd(
450
+ url,
451
+ fail=fail,
452
+ location=location,
453
+ output=output,
454
+ show_error=show_error,
455
+ silent=silent,
456
+ ),
457
+ sudo=sudo,
458
+ )
459
+ return run( # skipif-ci
460
+ *args,
461
+ print=print,
462
+ print_stdout=print_stdout,
463
+ print_stderr=print_stderr,
464
+ return_=return_,
465
+ return_stdout=return_stdout,
466
+ return_stderr=return_stderr,
467
+ retry=retry,
468
+ retry_skip=retry_skip,
469
+ logger=logger,
470
+ )
471
+
472
+
473
+ def curl_cmd(
474
+ url: str,
475
+ /,
476
+ *,
477
+ fail: bool = True,
478
+ location: bool = True,
479
+ output: PathLike | None = None,
480
+ show_error: bool = True,
481
+ silent: bool = True,
482
+ ) -> list[str]:
483
+ """Command to use 'curl' to transfer a URL."""
484
+ args: list[str] = ["curl"]
485
+ if fail:
486
+ args.append("--fail")
487
+ if location:
488
+ args.append("--location")
489
+ if output is not None:
490
+ args.extend(["--create-dirs", "--output", str(output)])
491
+ if show_error:
492
+ args.append("--show-error")
493
+ if silent:
494
+ args.append("--silent")
495
+ return [*args, url]
496
+
497
+
498
+ ##
499
+
500
+
108
501
  def echo_cmd(text: str, /) -> list[str]:
502
+ """Command to use 'echo' to write arguments to the standard output."""
109
503
  return ["echo", text]
110
504
 
111
505
 
112
506
  ##
113
507
 
114
508
 
509
+ def env_cmds(env: StrStrMapping, /) -> list[str]:
510
+ return [f"{key}={value}" for key, value in env.items()]
511
+
512
+
513
+ ##
514
+
515
+
115
516
  def expand_path(
116
517
  path: PathLike, /, *, subs: StrMapping | None = None, sudo: bool = False
117
518
  ) -> Path:
519
+ """Expand a path using `subprocess`."""
118
520
  if subs is not None:
119
521
  path = Template(str(path)).substitute(**subs)
120
522
  if sudo: # pragma: no cover
@@ -125,38 +527,103 @@ def expand_path(
125
527
  ##
126
528
 
127
529
 
128
- def git_clone_cmd(url: str, path: PathLike, /) -> list[str]:
129
- return ["git", "clone", "--recurse-submodules", url, str(path)]
530
+ def git_branch_current(path: PathLike, /) -> str:
531
+ """Show the current a branch."""
532
+ return run(*GIT_BRANCH_SHOW_CURRENT, cwd=path, return_=True)
130
533
 
131
534
 
132
535
  ##
133
536
 
134
537
 
135
- def git_hard_reset_cmd(*, branch: str | None = None) -> list[str]:
136
- branch_use = "master" if branch is None else branch
137
- return ["git", "hard-reset", branch_use]
538
+ def git_checkout(branch: str, path: PathLike, /) -> None:
539
+ """Switch a branch."""
540
+ run(*git_checkout_cmd(branch), cwd=path)
541
+
542
+
543
+ def git_checkout_cmd(branch: str, /) -> list[str]:
544
+ """Command to use 'git checkout' to switch a branch."""
545
+ return ["git", "checkout", branch]
138
546
 
139
547
 
140
548
  ##
141
549
 
142
550
 
143
- def maybe_parent(path: PathLike, /, *, parent: bool = False) -> Path:
144
- path = Path(path)
145
- return path.parent if parent else path
551
+ def git_clone(
552
+ url: str, path: PathLike, /, *, sudo: bool = False, branch: str | None = None
553
+ ) -> None:
554
+ """Clone a repository."""
555
+ rm(path, sudo=sudo)
556
+ run(*maybe_sudo_cmd(*git_clone_cmd(url, path), sudo=sudo))
557
+ if branch is not None:
558
+ git_checkout(branch, path)
559
+
560
+
561
+ def git_clone_cmd(url: str, path: PathLike, /) -> list[str]:
562
+ """Command to use 'git clone' to clone a repository."""
563
+ return ["git", "clone", "--recurse-submodules", url, str(path)]
146
564
 
147
565
 
148
566
  ##
149
567
 
150
568
 
151
- def maybe_sudo_cmd(cmd: str, /, *args: str, sudo: bool = False) -> list[str]:
152
- parts: list[str] = [cmd, *args]
153
- return sudo_cmd(*parts) if sudo else parts
569
+ def install(
570
+ path: PathLike,
571
+ /,
572
+ *,
573
+ directory: bool = False,
574
+ mode: PermissionsLike | None = None,
575
+ owner: str | int | None = None,
576
+ group: str | int | None = None,
577
+ sudo: bool = False,
578
+ ) -> None:
579
+ """Install a binary."""
580
+ args = maybe_sudo_cmd(
581
+ *install_cmd(path, directory=directory, mode=mode, owner=owner, group=group),
582
+ sudo=sudo,
583
+ )
584
+ run(*args)
585
+
586
+
587
+ def install_cmd(
588
+ path: PathLike,
589
+ /,
590
+ *,
591
+ directory: bool = False,
592
+ mode: PermissionsLike | None = None,
593
+ owner: str | int | None = None,
594
+ group: str | int | None = None,
595
+ ) -> list[str]:
596
+ """Command to use 'install' to install a binary."""
597
+ args: list[str] = ["install"]
598
+ if directory:
599
+ args.append("-d")
600
+ if mode is not None:
601
+ args.extend(["-m", str(ensure_perms(mode))])
602
+ if owner is not None:
603
+ args.extend(["-o", str(owner)])
604
+ if group is not None:
605
+ args.extend(["-g", str(group)])
606
+ if directory:
607
+ args.append(str(path))
608
+ else:
609
+ args.extend(["/dev/null", str(path)])
610
+ return args
611
+
612
+
613
+ ##
614
+
615
+
616
+ def maybe_parent(path: PathLike, /, *, parent: bool = False) -> Path:
617
+ """Get the parent of a path, if required."""
618
+ path = Path(path)
619
+ return path.parent if parent else path
154
620
 
155
621
 
156
622
  ##
157
623
 
158
624
 
159
625
  def mkdir(path: PathLike, /, *, sudo: bool = False, parent: bool = False) -> None:
626
+ """Make a directory."""
160
627
  if sudo: # pragma: no cover
161
628
  run(*sudo_cmd(*mkdir_cmd(path, parent=parent)))
162
629
  else:
@@ -167,21 +634,105 @@ def mkdir(path: PathLike, /, *, sudo: bool = False, parent: bool = False) -> Non
167
634
 
168
635
 
169
636
  def mkdir_cmd(path: PathLike, /, *, parent: bool = False) -> list[str]:
637
+ """Command to use 'mv' to make a directory."""
170
638
  return ["mkdir", "-p", str(maybe_parent(path, parent=parent))]
171
639
 
172
640
 
173
641
  ##
174
642
 
175
643
 
644
+ def mv(
645
+ src: PathLike,
646
+ dest: PathLike,
647
+ /,
648
+ *,
649
+ sudo: bool = False,
650
+ perms: PermissionsLike | None = None,
651
+ owner: str | int | None = None,
652
+ group: str | int | None = None,
653
+ ) -> None:
654
+ """Move a file/directory."""
655
+ mkdir(dest, sudo=sudo, parent=True)
656
+ if sudo: # pragma: no cover
657
+ run(*sudo_cmd(*cp_cmd(src, dest)))
658
+ else:
659
+ src, dest = map(Path, [src, dest])
660
+ if src.exists():
661
+ _ = move(src, dest)
662
+ else:
663
+ raise MvFileError(src=src, dest=dest)
664
+ if perms is not None:
665
+ chmod(dest, perms, sudo=sudo)
666
+ if (owner is not None) or (group is not None):
667
+ chown(dest, sudo=sudo, user=owner, group=group)
668
+
669
+
670
+ @dataclass(kw_only=True, slots=True)
671
+ class MvFileError(Exception):
672
+ src: Path
673
+ dest: Path
674
+
675
+ @override
676
+ def __str__(self) -> str:
677
+ return f"Unable to move {str(self.src)!r} to {str(self.dest)!r}; source does not exist"
678
+
679
+
176
680
  def mv_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
681
+ """Command to use 'mv' to move a file/directory."""
177
682
  return ["mv", str(src), str(dest)]
178
683
 
179
684
 
180
685
  ##
181
686
 
182
687
 
183
- def rm_cmd(path: PathLike, /) -> list[str]:
184
- return ["rm", "-rf", str(path)]
688
+ def replace_text(
689
+ path: PathLike, /, *replacements: tuple[str, str], sudo: bool = False
690
+ ) -> None:
691
+ """Replace the text in a file."""
692
+ path = Path(path)
693
+ text = cat(path, sudo=sudo)
694
+ for old, new in replacements:
695
+ text = text.replace(old, new)
696
+ tee(path, text, sudo=sudo)
697
+
698
+
699
+ ##
700
+
701
+
702
+ def ripgrep(*args: str, path: PathLike = PWD) -> str | None:
703
+ """Search for lines."""
704
+ try: # skipif-ci
705
+ return run(*ripgrep_cmd(*args, path=path), return_=True)
706
+ except CalledProcessError as error: # skipif-ci
707
+ if error.returncode == 1:
708
+ return None
709
+ raise
710
+
711
+
712
+ def ripgrep_cmd(*args: str, path: PathLike = PWD) -> list[str]:
713
+ """Command to use 'ripgrep' to search for lines."""
714
+ return ["rg", *args, str(path)]
715
+
716
+
717
+ ##
718
+
719
+
720
+ def rm(path: PathLike, /, *paths: PathLike, sudo: bool = False) -> None:
721
+ """Remove a file/directory."""
722
+ if sudo: # pragma: no cover
723
+ run(*sudo_cmd(*rm_cmd(path, *paths)))
724
+ else:
725
+ all_paths = list(map(Path, [path, *paths]))
726
+ for p in all_paths:
727
+ if p.is_file():
728
+ p.unlink(missing_ok=True)
729
+ elif p.is_dir():
730
+ rmtree(p, ignore_errors=True)
731
+
732
+
733
+ def rm_cmd(path: PathLike, /, *paths: PathLike) -> list[str]:
734
+ """Command to use 'rm' to remove a file/directory."""
735
+ return ["rm", "-rf", str(path), *map(str, paths)]
185
736
 
186
737
 
187
738
  ##
@@ -204,8 +755,9 @@ def rsync(
204
755
  chown_user: str | None = None,
205
756
  chown_group: str | None = None,
206
757
  exclude: MaybeIterable[str] | None = None,
207
- chmod: str | None = None,
758
+ chmod: PermissionsLike | None = None,
208
759
  ) -> None:
760
+ """Remote & local file copying."""
209
761
  mkdir_args = maybe_sudo_cmd(*mkdir_cmd(dest, parent=True), sudo=sudo) # skipif-ci
210
762
  ssh( # skipif-ci
211
763
  user,
@@ -218,13 +770,13 @@ def rsync(
218
770
  retry=retry,
219
771
  logger=logger,
220
772
  )
221
- is_dir = any(Path(s).is_dir() for s in always_iterable(src_or_srcs)) # skipif-ci
773
+ srcs = list(always_iterable(src_or_srcs)) # skipif-ci
222
774
  rsync_args = rsync_cmd( # skipif-ci
223
- src_or_srcs,
775
+ srcs,
224
776
  user,
225
777
  hostname,
226
778
  dest,
227
- archive=is_dir,
779
+ archive=any(Path(s).is_dir() for s in srcs),
228
780
  chown_user=chown_user,
229
781
  chown_group=chown_group,
230
782
  exclude=exclude,
@@ -232,7 +784,6 @@ def rsync(
232
784
  host_key_algorithms=host_key_algorithms,
233
785
  strict_host_key_checking=strict_host_key_checking,
234
786
  sudo=sudo,
235
- parent=is_dir,
236
787
  )
237
788
  run(*rsync_args, print=print, retry=retry, logger=logger) # skipif-ci
238
789
  if chmod is not None: # skipif-ci
@@ -250,59 +801,188 @@ def rsync(
250
801
  )
251
802
 
252
803
 
804
+ def rsync_cmd(
805
+ src_or_srcs: MaybeIterable[PathLike],
806
+ user: str,
807
+ hostname: str,
808
+ dest: PathLike,
809
+ /,
810
+ *,
811
+ archive: bool = False,
812
+ chown_user: str | None = None,
813
+ chown_group: str | None = None,
814
+ exclude: MaybeIterable[str] | None = None,
815
+ batch_mode: bool = True,
816
+ host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
817
+ strict_host_key_checking: bool = True,
818
+ sudo: bool = False,
819
+ ) -> list[str]:
820
+ """Command to use 'rsync' to do remote & local file copying."""
821
+ args: list[str] = ["rsync"]
822
+ if archive:
823
+ args.append("--archive")
824
+ args.append("--checksum")
825
+ match chown_user, chown_group:
826
+ case None, None:
827
+ ...
828
+ case str(), None:
829
+ args.extend(["--chown", chown_user])
830
+ case None, str():
831
+ args.extend(["--chown", f":{chown_group}"])
832
+ case str(), str():
833
+ args.extend(["--chown", f"{chown_user}:{chown_group}"])
834
+ case never:
835
+ assert_never(never)
836
+ args.append("--compress")
837
+ if exclude is not None:
838
+ for exclude_i in always_iterable(exclude):
839
+ args.extend(["--exclude", exclude_i])
840
+ rsh_args: list[str] = ssh_opts_cmd(
841
+ batch_mode=batch_mode,
842
+ host_key_algorithms=host_key_algorithms,
843
+ strict_host_key_checking=strict_host_key_checking,
844
+ )
845
+ args.extend(["--rsh", join(rsh_args)])
846
+ if sudo:
847
+ args.extend(["--rsync-path", join(sudo_cmd("rsync"))])
848
+ srcs = list(always_iterable(src_or_srcs)) # do not Path()
849
+ if len(srcs) == 0:
850
+ raise RsyncCmdNoSourcesError(user=user, hostname=hostname, dest=dest)
851
+ missing = [s for s in srcs if not Path(s).exists()]
852
+ if len(missing) >= 1:
853
+ raise RsyncCmdSourcesNotFoundError(
854
+ sources=missing, user=user, hostname=hostname, dest=dest
855
+ )
856
+ return [*args, *map(str, srcs), f"{user}@{hostname}:{dest}"]
857
+
858
+
859
+ @dataclass(kw_only=True, slots=True)
860
+ class RsyncCmdError(Exception):
861
+ user: str
862
+ hostname: str
863
+ dest: PathLike
864
+
865
+ @override
866
+ def __str__(self) -> str:
867
+ return f"No sources selected to send to {self.user}@{self.hostname}:{self.dest}"
868
+
869
+
870
+ @dataclass(kw_only=True, slots=True)
871
+ class RsyncCmdNoSourcesError(RsyncCmdError):
872
+ @override
873
+ def __str__(self) -> str:
874
+ return f"No sources selected to send to {self.user}@{self.hostname}:{self.dest}"
875
+
876
+
877
+ @dataclass(kw_only=True, slots=True)
878
+ class RsyncCmdSourcesNotFoundError(RsyncCmdError):
879
+ sources: list[PathLike]
880
+
881
+ @override
882
+ def __str__(self) -> str:
883
+ desc = ", ".join(map(repr, map(str, self.sources)))
884
+ return f"Sources selected to send to {self.user}@{self.hostname}:{self.dest} but not found: {desc}"
885
+
886
+
253
887
  ##
254
888
 
255
889
 
256
- def rsync_cmd(
257
- src_or_srcs: MaybeIterable[PathLike],
258
- user: str,
259
- hostname: str,
890
+ def rsync_many(
891
+ user: str,
892
+ hostname: str,
893
+ /,
894
+ *items: tuple[PathLike, PathLike]
895
+ | tuple[Literal["sudo"], PathLike, PathLike]
896
+ | tuple[PathLike, PathLike, PermissionsLike],
897
+ retry: Retry | None = None,
898
+ logger: LoggerLike | None = None,
899
+ keep: bool = False,
900
+ batch_mode: bool = True,
901
+ host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
902
+ strict_host_key_checking: bool = True,
903
+ print: bool = False, # noqa: A002
904
+ exclude: MaybeIterable[str] | None = None,
905
+ ) -> None:
906
+ cmds: list[list[str]] = [] # skipif-ci
907
+ with ( # skipif-ci
908
+ TemporaryDirectory() as temp_src,
909
+ yield_ssh_temp_dir(
910
+ user, hostname, retry=retry, logger=logger, keep=keep
911
+ ) as temp_dest,
912
+ ):
913
+ for item in items:
914
+ match item:
915
+ case Path() | str() as src, Path() | str() as dest:
916
+ cmds.extend(_rsync_many_prepare(src, dest, temp_src, temp_dest))
917
+ case "sudo", Path() | str() as src, Path() | str() as dest:
918
+ cmds.extend(
919
+ _rsync_many_prepare(src, dest, temp_src, temp_dest, sudo=True)
920
+ )
921
+ case (
922
+ Path() | str() as src,
923
+ Path() | str() as dest,
924
+ Permissions() | int() | str() as perms,
925
+ ):
926
+ cmds.extend(
927
+ _rsync_many_prepare(src, dest, temp_src, temp_dest, perms=perms)
928
+ )
929
+ case never:
930
+ assert_never(never)
931
+ rsync(
932
+ f"{temp_src}/",
933
+ user,
934
+ hostname,
935
+ temp_dest,
936
+ batch_mode=batch_mode,
937
+ host_key_algorithms=host_key_algorithms,
938
+ strict_host_key_checking=strict_host_key_checking,
939
+ print=print,
940
+ retry=retry,
941
+ logger=logger,
942
+ exclude=exclude,
943
+ )
944
+ ssh(
945
+ user,
946
+ hostname,
947
+ *BASH_LS,
948
+ input="\n".join(map(join, cmds)),
949
+ print=print,
950
+ retry=retry,
951
+ logger=logger,
952
+ )
953
+
954
+
955
+ def _rsync_many_prepare(
956
+ src: PathLike,
260
957
  dest: PathLike,
958
+ temp_src: PathLike,
959
+ temp_dest: PathLike,
261
960
  /,
262
961
  *,
263
- archive: bool = False,
264
- chown_user: str | None = None,
265
- chown_group: str | None = None,
266
- exclude: MaybeIterable[str] | None = None,
267
- batch_mode: bool = True,
268
- host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
269
- strict_host_key_checking: bool = True,
270
962
  sudo: bool = False,
271
- parent: bool = False,
272
- ) -> list[str]:
273
- args: list[str] = ["rsync"]
274
- if archive:
275
- args.append("--archive")
276
- args.append("--checksum")
277
- match chown_user, chown_group:
278
- case None, None:
279
- ...
280
- case str(), None:
281
- args.extend(["--chown", chown_user])
282
- case None, str():
283
- args.extend(["--chown", f":{chown_group}"])
284
- case str(), str():
285
- args.extend(["--chown", f"{chown_user}:{chown_group}"])
963
+ perms: PermissionsLike | None = None,
964
+ ) -> list[list[str]]:
965
+ dest, temp_src, temp_dest = map(Path, [dest, temp_src, temp_dest])
966
+ n = len(list(temp_src.iterdir()))
967
+ name = str(n)
968
+ match src:
969
+ case Path():
970
+ cp(src, temp_src / name)
971
+ case str():
972
+ if Path(src).exists():
973
+ cp(src, temp_src / name)
974
+ else:
975
+ tee(temp_src / name, src)
286
976
  case never:
287
977
  assert_never(never)
288
- args.append("--compress")
289
- if exclude is not None:
290
- for exclude_i in always_iterable(exclude):
291
- args.extend(["--exclude", exclude_i])
292
- rsh_args: list[str] = ssh_opts_cmd(
293
- batch_mode=batch_mode,
294
- host_key_algorithms=host_key_algorithms,
295
- strict_host_key_checking=strict_host_key_checking,
296
- )
297
- args.extend(["--rsh", join(rsh_args)])
298
- if sudo:
299
- args.extend(["--rsync-path", join(sudo_cmd("rsync"))])
300
- dest_use = maybe_parent(dest, parent=parent)
301
- return [
302
- *args,
303
- *map(str, always_iterable(src_or_srcs)),
304
- f"{user}@{hostname}:{dest_use}",
978
+ cmds: list[list[str]] = [
979
+ maybe_sudo_cmd(*rm_cmd(dest), sudo=sudo),
980
+ maybe_sudo_cmd(*mkdir_cmd(dest, parent=True), sudo=sudo),
981
+ maybe_sudo_cmd(*cp_cmd(temp_dest / name, dest), sudo=sudo),
305
982
  ]
983
+ if perms is not None:
984
+ cmds.append(maybe_sudo_cmd(*chmod_cmd(dest, perms), sudo=sudo))
985
+ return cmds
306
986
 
307
987
 
308
988
  ##
@@ -326,6 +1006,7 @@ def run(
326
1006
  return_stdout: bool = False,
327
1007
  return_stderr: bool = False,
328
1008
  retry: Retry | None = None,
1009
+ retry_skip: Callable[[int, str, str], bool] | None = None,
329
1010
  logger: LoggerLike | None = None,
330
1011
  ) -> str: ...
331
1012
  @overload
@@ -346,6 +1027,7 @@ def run(
346
1027
  return_stdout: Literal[True],
347
1028
  return_stderr: bool = False,
348
1029
  retry: Retry | None = None,
1030
+ retry_skip: Callable[[int, str, str], bool] | None = None,
349
1031
  logger: LoggerLike | None = None,
350
1032
  ) -> str: ...
351
1033
  @overload
@@ -366,6 +1048,7 @@ def run(
366
1048
  return_stdout: bool = False,
367
1049
  return_stderr: Literal[True],
368
1050
  retry: Retry | None = None,
1051
+ retry_skip: Callable[[int, str, str], bool] | None = None,
369
1052
  logger: LoggerLike | None = None,
370
1053
  ) -> str: ...
371
1054
  @overload
@@ -386,6 +1069,7 @@ def run(
386
1069
  return_stdout: Literal[False] = False,
387
1070
  return_stderr: Literal[False] = False,
388
1071
  retry: Retry | None = None,
1072
+ retry_skip: Callable[[int, str, str], bool] | None = None,
389
1073
  logger: LoggerLike | None = None,
390
1074
  ) -> None: ...
391
1075
  @overload
@@ -406,6 +1090,7 @@ def run(
406
1090
  return_stdout: bool = False,
407
1091
  return_stderr: bool = False,
408
1092
  retry: Retry | None = None,
1093
+ retry_skip: Callable[[int, str, str], bool] | None = None,
409
1094
  logger: LoggerLike | None = None,
410
1095
  ) -> str | None: ...
411
1096
  def run(
@@ -425,8 +1110,10 @@ def run(
425
1110
  return_stdout: bool = False,
426
1111
  return_stderr: bool = False,
427
1112
  retry: Retry | None = None,
1113
+ retry_skip: Callable[[int, str, str], bool] | None = None,
428
1114
  logger: LoggerLike | None = None,
429
1115
  ) -> str | None:
1116
+ """Run a command in a subprocess."""
430
1117
  args: list[str] = []
431
1118
  if user is not None: # pragma: no cover
432
1119
  args.extend(["su", "-", str(user)])
@@ -481,14 +1168,18 @@ def run(
481
1168
  case 0, False, False:
482
1169
  return None
483
1170
  case _, _, _:
484
- if retry is None:
485
- attempts = delta = None
486
- else:
487
- attempts, delta = retry
488
1171
  _ = stdout.seek(0)
489
1172
  stdout_text = stdout.read()
490
1173
  _ = stderr.seek(0)
491
1174
  stderr_text = stderr.read()
1175
+ if (retry is None) or (
1176
+ (retry is not None)
1177
+ and (retry_skip is not None)
1178
+ and retry_skip(return_code, stdout_text, stderr_text)
1179
+ ):
1180
+ attempts = delta = None
1181
+ else:
1182
+ attempts, delta = retry
492
1183
  if logger is not None:
493
1184
  msg = strip_and_dedent(f"""
494
1185
  'run' failed with:
@@ -567,6 +1258,7 @@ def _run_write_to_streams(text: str, /, *outputs: IO[str]) -> None:
567
1258
 
568
1259
 
569
1260
  def set_hostname_cmd(hostname: str, /) -> list[str]:
1261
+ """Command to set the system hostname."""
570
1262
  return ["hostnamectl", "set-hostname", hostname]
571
1263
 
572
1264
 
@@ -578,10 +1270,12 @@ def ssh(
578
1270
  user: str,
579
1271
  hostname: str,
580
1272
  /,
581
- *cmd_and_cmds_or_args: str,
1273
+ *cmd_and_args: str,
582
1274
  batch_mode: bool = True,
583
1275
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
584
1276
  strict_host_key_checking: bool = True,
1277
+ port: int | None = None,
1278
+ env: StrStrMapping | None = None,
585
1279
  input: str | None = None,
586
1280
  print: bool = False,
587
1281
  print_stdout: bool = False,
@@ -597,10 +1291,12 @@ def ssh(
597
1291
  user: str,
598
1292
  hostname: str,
599
1293
  /,
600
- *cmd_and_cmds_or_args: str,
1294
+ *cmd_and_args: str,
601
1295
  batch_mode: bool = True,
602
1296
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
603
1297
  strict_host_key_checking: bool = True,
1298
+ port: int | None = None,
1299
+ env: StrStrMapping | None = None,
604
1300
  input: str | None = None,
605
1301
  print: bool = False,
606
1302
  print_stdout: bool = False,
@@ -616,10 +1312,12 @@ def ssh(
616
1312
  user: str,
617
1313
  hostname: str,
618
1314
  /,
619
- *cmd_and_cmds_or_args: str,
1315
+ *cmd_and_args: str,
620
1316
  batch_mode: bool = True,
621
1317
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
622
1318
  strict_host_key_checking: bool = True,
1319
+ port: int | None = None,
1320
+ env: StrStrMapping | None = None,
623
1321
  input: str | None = None,
624
1322
  print: bool = False,
625
1323
  print_stdout: bool = False,
@@ -635,10 +1333,12 @@ def ssh(
635
1333
  user: str,
636
1334
  hostname: str,
637
1335
  /,
638
- *cmd_and_cmds_or_args: str,
1336
+ *cmd_and_args: str,
639
1337
  batch_mode: bool = True,
640
1338
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
641
1339
  strict_host_key_checking: bool = True,
1340
+ port: int | None = None,
1341
+ env: StrStrMapping | None = None,
642
1342
  input: str | None = None,
643
1343
  print: bool = False,
644
1344
  print_stdout: bool = False,
@@ -654,10 +1354,12 @@ def ssh(
654
1354
  user: str,
655
1355
  hostname: str,
656
1356
  /,
657
- *cmd_and_cmds_or_args: str,
1357
+ *cmd_and_args: str,
658
1358
  batch_mode: bool = True,
659
1359
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
660
1360
  strict_host_key_checking: bool = True,
1361
+ port: int | None = None,
1362
+ env: StrStrMapping | None = None,
661
1363
  input: str | None = None,
662
1364
  print: bool = False,
663
1365
  print_stdout: bool = False,
@@ -672,10 +1374,12 @@ def ssh(
672
1374
  user: str,
673
1375
  hostname: str,
674
1376
  /,
675
- *cmd_and_cmds_or_args: str,
1377
+ *cmd_and_args: str,
676
1378
  batch_mode: bool = True,
677
1379
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
678
1380
  strict_host_key_checking: bool = True,
1381
+ port: int | None = None,
1382
+ env: StrStrMapping | None = None,
679
1383
  input: str | None = None, # noqa: A002
680
1384
  print: bool = False, # noqa: A002
681
1385
  print_stdout: bool = False,
@@ -686,49 +1390,91 @@ def ssh(
686
1390
  retry: Retry | None = None,
687
1391
  logger: LoggerLike | None = None,
688
1392
  ) -> str | None:
689
- cmd_and_args = ssh_cmd( # skipif-ci
1393
+ """Execute a command on a remote machine."""
1394
+ run_cmd_and_args = ssh_cmd( # skipif-ci
690
1395
  user,
691
1396
  hostname,
692
- *cmd_and_cmds_or_args,
1397
+ *cmd_and_args,
693
1398
  batch_mode=batch_mode,
694
1399
  host_key_algorithms=host_key_algorithms,
695
1400
  strict_host_key_checking=strict_host_key_checking,
1401
+ port=port,
1402
+ env=env,
696
1403
  )
697
- return run( # skipif-ci
698
- *cmd_and_args,
699
- input=input,
700
- print=print,
701
- print_stdout=print_stdout,
702
- print_stderr=print_stderr,
703
- return_=return_,
704
- return_stdout=return_stdout,
705
- return_stderr=return_stderr,
706
- retry=retry,
707
- logger=logger,
708
- )
1404
+ try: # skipif-ci
1405
+ return run(
1406
+ *run_cmd_and_args,
1407
+ input=input,
1408
+ print=print,
1409
+ print_stdout=print_stdout,
1410
+ print_stderr=print_stderr,
1411
+ return_=return_,
1412
+ return_stdout=return_stdout,
1413
+ return_stderr=return_stderr,
1414
+ retry=retry,
1415
+ retry_skip=_ssh_retry_skip,
1416
+ logger=logger,
1417
+ )
1418
+ except CalledProcessError as error: # skipif-ci
1419
+ if not _ssh_is_strict_checking_error(error.stderr):
1420
+ raise
1421
+ ssh_keyscan(hostname, port=port)
1422
+ return ssh(
1423
+ user,
1424
+ hostname,
1425
+ *cmd_and_args,
1426
+ batch_mode=batch_mode,
1427
+ host_key_algorithms=host_key_algorithms,
1428
+ strict_host_key_checking=strict_host_key_checking,
1429
+ port=port,
1430
+ input=input,
1431
+ print=print,
1432
+ print_stdout=print_stdout,
1433
+ print_stderr=print_stderr,
1434
+ return_=return_,
1435
+ return_stdout=return_stdout,
1436
+ return_stderr=return_stderr,
1437
+ retry=retry,
1438
+ logger=logger,
1439
+ )
709
1440
 
710
1441
 
711
- ##
1442
+ def _ssh_retry_skip(return_code: int, stdout: str, stderr: str, /) -> bool:
1443
+ _ = (return_code, stdout)
1444
+ return _ssh_is_strict_checking_error(stderr)
1445
+
1446
+
1447
+ def _ssh_is_strict_checking_error(text: str, /) -> bool:
1448
+ match = search(
1449
+ "(Host key for .* has changed|No ED25519 host key is known for .*) and you have requested strict checking",
1450
+ text,
1451
+ flags=MULTILINE,
1452
+ )
1453
+ return match is not None
712
1454
 
713
1455
 
714
1456
  def ssh_cmd(
715
1457
  user: str,
716
1458
  hostname: str,
717
1459
  /,
718
- *cmd_and_cmds_or_args: str,
1460
+ *cmd_and_args: str,
719
1461
  batch_mode: bool = True,
720
1462
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
721
1463
  strict_host_key_checking: bool = True,
1464
+ port: int | None = None,
1465
+ env: StrStrMapping | None = None,
722
1466
  ) -> list[str]:
1467
+ """Command to use 'ssh' to execute a command on a remote machine."""
723
1468
  args: list[str] = ssh_opts_cmd(
724
1469
  batch_mode=batch_mode,
725
1470
  host_key_algorithms=host_key_algorithms,
726
1471
  strict_host_key_checking=strict_host_key_checking,
1472
+ port=port,
727
1473
  )
728
- return [*args, f"{user}@{hostname}", *cmd_and_cmds_or_args]
729
-
730
-
731
- ##
1474
+ args.append(f"{user}@{hostname}")
1475
+ if env is not None:
1476
+ args.extend(env_cmds(env))
1477
+ return [*args, *cmd_and_args]
732
1478
 
733
1479
 
734
1480
  def ssh_opts_cmd(
@@ -736,55 +1482,333 @@ def ssh_opts_cmd(
736
1482
  batch_mode: bool = True,
737
1483
  host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
738
1484
  strict_host_key_checking: bool = True,
1485
+ port: int | None = None,
739
1486
  ) -> list[str]:
1487
+ """Command to use prepare 'ssh' to execute a command on a remote machine."""
740
1488
  args: list[str] = ["ssh"]
741
1489
  if batch_mode:
742
1490
  args.extend(["-o", "BatchMode=yes"])
743
1491
  args.extend(["-o", f"HostKeyAlgorithms={','.join(host_key_algorithms)}"])
744
1492
  if strict_host_key_checking:
745
1493
  args.extend(["-o", "StrictHostKeyChecking=yes"])
1494
+ if port is not None:
1495
+ args.extend(["-p", str(port)])
746
1496
  return [*args, "-T"]
747
1497
 
748
1498
 
749
1499
  ##
750
1500
 
751
1501
 
752
- def ssh_keygen_cmd(hostname: str, /) -> list[str]:
753
- return ["ssh-keygen", "-f", "~/.ssh/known_hosts", "-R", hostname]
1502
+ def ssh_await(
1503
+ user: str,
1504
+ hostname: str,
1505
+ /,
1506
+ *,
1507
+ logger: LoggerLike | None = None,
1508
+ delta: Delta = SECOND,
1509
+ ) -> None:
1510
+ while True: # skipif-ci
1511
+ if logger is not None:
1512
+ to_logger(logger).info("Waiting for '%s'...", hostname)
1513
+ try:
1514
+ ssh(user, hostname, "true")
1515
+ except CalledProcessError:
1516
+ sleep(to_seconds(delta))
1517
+ else:
1518
+ if logger is not None:
1519
+ to_logger(logger).info("'%s' is up", hostname)
1520
+ return
1521
+
1522
+
1523
+ ##
1524
+
1525
+
1526
+ def ssh_keyscan(
1527
+ hostname: str,
1528
+ /,
1529
+ *,
1530
+ path: PathLike = KNOWN_HOSTS,
1531
+ retry: Retry | None = None,
1532
+ port: int | None = None,
1533
+ ) -> None:
1534
+ """Add a known host."""
1535
+ ssh_keygen_remove(hostname, path=path, retry=retry) # skipif-ci
1536
+ result = run( # skipif-ci
1537
+ *ssh_keyscan_cmd(hostname, port=port), return_=True, retry=retry
1538
+ )
1539
+ tee(path, result, append=True) # skipif-ci
1540
+
1541
+
1542
+ def ssh_keyscan_cmd(hostname: str, /, *, port: int | None = None) -> list[str]:
1543
+ """Command to use 'ssh-keyscan' to add a known host."""
1544
+ args: list[str] = ["ssh-keyscan"]
1545
+ if port is not None:
1546
+ args.extend(["-p", str(port)])
1547
+ return [*args, "-q", "-t", "ed25519", hostname]
1548
+
1549
+
1550
+ ##
1551
+
1552
+
1553
+ def ssh_keygen_remove(
1554
+ hostname: str, /, *, path: PathLike = KNOWN_HOSTS, retry: Retry | None = None
1555
+ ) -> None:
1556
+ """Remove a known host."""
1557
+ path = Path(path)
1558
+ if path.exists():
1559
+ run(*ssh_keygen_remove_cmd(hostname, path=path), retry=retry)
1560
+
1561
+
1562
+ def ssh_keygen_remove_cmd(
1563
+ hostname: str, /, *, path: PathLike = KNOWN_HOSTS
1564
+ ) -> list[str]:
1565
+ """Command to use 'ssh-keygen' to remove a known host."""
1566
+ return ["ssh-keygen", "-f", str(path), "-R", hostname]
754
1567
 
755
1568
 
756
1569
  ##
757
1570
 
758
1571
 
759
1572
  def sudo_cmd(cmd: str, /, *args: str) -> list[str]:
1573
+ """Command to use 'sudo' to execute a command as another user."""
760
1574
  return ["sudo", cmd, *args]
761
1575
 
762
1576
 
1577
+ def maybe_sudo_cmd(cmd: str, /, *args: str, sudo: bool = False) -> list[str]:
1578
+ """Command to use 'sudo' to execute a command as another user, if required."""
1579
+ parts: list[str] = [cmd, *args]
1580
+ return sudo_cmd(*parts) if sudo else parts
1581
+
1582
+
763
1583
  ##
764
1584
 
765
1585
 
766
1586
  def sudo_nopasswd_cmd(user: str, /) -> str:
1587
+ """Command to allow a user to use password-free `sudo`."""
767
1588
  return f"{user} ALL=(ALL) NOPASSWD: ALL"
768
1589
 
769
1590
 
770
1591
  ##
771
1592
 
772
1593
 
773
- def symlink_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
774
- return ["ln", "-s", str(src), str(dest)]
1594
+ def symlink(target: PathLike, link: PathLike, /, *, sudo: bool = False) -> None:
1595
+ """Make a symbolic link."""
1596
+ rm(link, sudo=sudo)
1597
+ mkdir(link, sudo=sudo, parent=True)
1598
+ if sudo: # pragma: no cover
1599
+ run(*sudo_cmd(*symlink_cmd(target, link)))
1600
+ else:
1601
+ target, link = map(Path, [target, link])
1602
+ link.symlink_to(target)
1603
+
1604
+
1605
+ def symlink_cmd(target: PathLike, link: PathLike, /) -> list[str]:
1606
+ """Command to use 'symlink' to make a symbolic link."""
1607
+ return ["ln", "-s", str(target), str(link)]
1608
+
1609
+
1610
+ ##
1611
+
1612
+
1613
+ def tee(
1614
+ path: PathLike, text: str, /, *, sudo: bool = False, append: bool = False
1615
+ ) -> None:
1616
+ """Duplicate standard input."""
1617
+ mkdir(path, sudo=sudo, parent=True)
1618
+ if sudo: # pragma: no cover
1619
+ run(*sudo_cmd(*tee_cmd(path, append=append)), input=text)
1620
+ else:
1621
+ path = Path(path)
1622
+ with path.open(mode="a" if append else "w") as fh:
1623
+ _ = fh.write(text)
1624
+
1625
+
1626
+ def tee_cmd(path: PathLike, /, *, append: bool = False) -> list[str]:
1627
+ """Command to use 'tee' to duplicate standard input."""
1628
+ args: list[str] = ["tee"]
1629
+ if append:
1630
+ args.append("-a")
1631
+ return [*args, str(path)]
775
1632
 
776
1633
 
777
1634
  ##
778
1635
 
779
1636
 
1637
+ def touch(path: PathLike, /, *, sudo: bool = False) -> None:
1638
+ """Change file access and modification times."""
1639
+ run(*maybe_sudo_cmd(*touch_cmd(path), sudo=sudo))
1640
+
1641
+
780
1642
  def touch_cmd(path: PathLike, /) -> list[str]:
1643
+ """Command to use 'touch' to change file access and modification times."""
781
1644
  return ["touch", str(path)]
782
1645
 
783
1646
 
784
1647
  ##
785
1648
 
786
1649
 
1650
+ def update_ca_certificates(*, sudo: bool = False) -> None:
1651
+ """Update the system CA certificates."""
1652
+ run(*maybe_sudo_cmd(UPDATE_CA_CERTIFICATES, sudo=sudo)) # pragma: no cover
1653
+
1654
+
1655
+ ##
1656
+
1657
+
1658
+ def useradd(
1659
+ login: str,
1660
+ /,
1661
+ *,
1662
+ create_home: bool = True,
1663
+ groups: MaybeIterable[str] | None = None,
1664
+ shell: PathLike | None = None,
1665
+ sudo: bool = False,
1666
+ password: str | None = None,
1667
+ ) -> None:
1668
+ """Create a new user."""
1669
+ args = maybe_sudo_cmd( # pragma: no cover
1670
+ *useradd_cmd(login, create_home=create_home, groups=groups, shell=shell)
1671
+ )
1672
+ run(*args) # pragma: no cover
1673
+ if password is not None: # pragma: no cover
1674
+ chpasswd(login, password, sudo=sudo)
1675
+
1676
+
1677
+ def useradd_cmd(
1678
+ login: str,
1679
+ /,
1680
+ *,
1681
+ create_home: bool = True,
1682
+ groups: MaybeIterable[str] | None = None,
1683
+ shell: PathLike | None = None,
1684
+ ) -> list[str]:
1685
+ """Command to use 'useradd' to create a new user."""
1686
+ args: list[str] = ["useradd"]
1687
+ if create_home:
1688
+ args.append("--create-home")
1689
+ if groups is not None:
1690
+ args.extend(["--groups", *always_iterable(groups)])
1691
+ if shell is not None:
1692
+ args.extend(["--shell", str(shell)])
1693
+ return [*args, login]
1694
+
1695
+
1696
+ ##
1697
+
1698
+
1699
+ @overload
1700
+ def uv_run(
1701
+ module: str,
1702
+ /,
1703
+ *args: str,
1704
+ env: StrStrMapping | None = None,
1705
+ cwd: PathLike | None = None,
1706
+ print: bool = False,
1707
+ print_stdout: bool = False,
1708
+ print_stderr: bool = False,
1709
+ return_: Literal[True],
1710
+ return_stdout: bool = False,
1711
+ return_stderr: bool = False,
1712
+ retry: Retry | None = None,
1713
+ logger: LoggerLike | None = None,
1714
+ ) -> str: ...
1715
+ @overload
1716
+ def uv_run(
1717
+ module: str,
1718
+ /,
1719
+ *args: str,
1720
+ env: StrStrMapping | None = None,
1721
+ cwd: PathLike | None = None,
1722
+ print: bool = False,
1723
+ print_stdout: bool = False,
1724
+ print_stderr: bool = False,
1725
+ return_: bool = False,
1726
+ return_stdout: Literal[True],
1727
+ return_stderr: bool = False,
1728
+ retry: Retry | None = None,
1729
+ logger: LoggerLike | None = None,
1730
+ ) -> str: ...
1731
+ @overload
1732
+ def uv_run(
1733
+ module: str,
1734
+ /,
1735
+ *args: str,
1736
+ env: StrStrMapping | None = None,
1737
+ cwd: PathLike | None = None,
1738
+ print: bool = False,
1739
+ print_stdout: bool = False,
1740
+ print_stderr: bool = False,
1741
+ return_: bool = False,
1742
+ return_stdout: bool = False,
1743
+ return_stderr: Literal[True],
1744
+ retry: Retry | None = None,
1745
+ logger: LoggerLike | None = None,
1746
+ ) -> str: ...
1747
+ @overload
1748
+ def uv_run(
1749
+ module: str,
1750
+ /,
1751
+ *args: str,
1752
+ env: StrStrMapping | None = None,
1753
+ cwd: PathLike | None = None,
1754
+ print: bool = False,
1755
+ print_stdout: bool = False,
1756
+ print_stderr: bool = False,
1757
+ return_: Literal[False] = False,
1758
+ return_stdout: Literal[False] = False,
1759
+ return_stderr: Literal[False] = False,
1760
+ retry: Retry | None = None,
1761
+ logger: LoggerLike | None = None,
1762
+ ) -> None: ...
1763
+ @overload
1764
+ def uv_run(
1765
+ module: str,
1766
+ /,
1767
+ *args: str,
1768
+ env: StrStrMapping | None = None,
1769
+ cwd: PathLike | None = None,
1770
+ print: bool = False,
1771
+ print_stdout: bool = False,
1772
+ print_stderr: bool = False,
1773
+ return_: bool = False,
1774
+ return_stdout: bool = False,
1775
+ return_stderr: bool = False,
1776
+ retry: Retry | None = None,
1777
+ logger: LoggerLike | None = None,
1778
+ ) -> str | None: ...
1779
+ def uv_run(
1780
+ module: str,
1781
+ /,
1782
+ *args: str,
1783
+ cwd: PathLike | None = None,
1784
+ env: StrStrMapping | None = None,
1785
+ print: bool = False, # noqa: A002
1786
+ print_stdout: bool = False,
1787
+ print_stderr: bool = False,
1788
+ return_: bool = False,
1789
+ return_stdout: bool = False,
1790
+ return_stderr: bool = False,
1791
+ retry: Retry | None = None,
1792
+ logger: LoggerLike | None = None,
1793
+ ) -> str | None:
1794
+ """Run a command or script."""
1795
+ return run( # pragma: no cover
1796
+ *uv_run_cmd(module, *args),
1797
+ cwd=cwd,
1798
+ env=env,
1799
+ print=print,
1800
+ print_stdout=print_stdout,
1801
+ print_stderr=print_stderr,
1802
+ return_=return_,
1803
+ return_stdout=return_stdout,
1804
+ return_stderr=return_stderr,
1805
+ retry=retry,
1806
+ logger=logger,
1807
+ )
1808
+
1809
+
787
1810
  def uv_run_cmd(module: str, /, *args: str) -> list[str]:
1811
+ """Command to use 'uv' to run a command or script."""
788
1812
  return [
789
1813
  "uv",
790
1814
  "run",
@@ -802,6 +1826,17 @@ def uv_run_cmd(module: str, /, *args: str) -> list[str]:
802
1826
  ##
803
1827
 
804
1828
 
1829
+ @contextmanager
1830
+ def yield_git_repo(url: str, /, *, branch: str | None = None) -> Iterator[Path]:
1831
+ """Yield a temporary git repository."""
1832
+ with TemporaryDirectory() as temp_dir:
1833
+ git_clone(url, temp_dir, branch=branch)
1834
+ yield temp_dir
1835
+
1836
+
1837
+ ##
1838
+
1839
+
805
1840
  @contextmanager
806
1841
  def yield_ssh_temp_dir(
807
1842
  user: str,
@@ -812,6 +1847,7 @@ def yield_ssh_temp_dir(
812
1847
  logger: LoggerLike | None = None,
813
1848
  keep: bool = False,
814
1849
  ) -> Iterator[Path]:
1850
+ """Yield a temporary directory on a remote machine."""
815
1851
  path = Path( # skipif-ci
816
1852
  ssh(user, hostname, *MKTEMP_DIR_CMD, return_=True, retry=retry, logger=logger)
817
1853
  )
@@ -829,36 +1865,83 @@ __all__ = [
829
1865
  "APT_UPDATE",
830
1866
  "BASH_LC",
831
1867
  "BASH_LS",
1868
+ "CHPASSWD",
1869
+ "GIT_BRANCH_SHOW_CURRENT",
832
1870
  "MKTEMP_DIR_CMD",
833
1871
  "RESTART_SSHD",
834
1872
  "UPDATE_CA_CERTIFICATES",
835
1873
  "ChownCmdError",
1874
+ "CpError",
1875
+ "MvFileError",
1876
+ "RsyncCmdError",
1877
+ "RsyncCmdNoSourcesError",
1878
+ "RsyncCmdSourcesNotFoundError",
1879
+ "append_text",
1880
+ "apt_install",
836
1881
  "apt_install_cmd",
1882
+ "apt_remove",
1883
+ "apt_remove_cmd",
1884
+ "apt_update",
1885
+ "cat",
837
1886
  "cd_cmd",
1887
+ "chattr",
1888
+ "chattr_cmd",
1889
+ "chmod",
838
1890
  "chmod_cmd",
1891
+ "chown",
839
1892
  "chown_cmd",
1893
+ "chpasswd",
1894
+ "copy_text",
1895
+ "cp",
840
1896
  "cp_cmd",
1897
+ "curl",
1898
+ "curl_cmd",
841
1899
  "echo_cmd",
1900
+ "env_cmds",
842
1901
  "expand_path",
1902
+ "git_branch_current",
1903
+ "git_checkout",
1904
+ "git_checkout_cmd",
1905
+ "git_clone",
843
1906
  "git_clone_cmd",
844
- "git_hard_reset_cmd",
1907
+ "install",
1908
+ "install_cmd",
845
1909
  "maybe_parent",
846
1910
  "maybe_sudo_cmd",
847
1911
  "mkdir",
848
1912
  "mkdir_cmd",
1913
+ "mv",
849
1914
  "mv_cmd",
1915
+ "replace_text",
1916
+ "ripgrep",
1917
+ "ripgrep_cmd",
1918
+ "rm",
850
1919
  "rm_cmd",
851
1920
  "rsync",
852
1921
  "rsync_cmd",
1922
+ "rsync_many",
853
1923
  "run",
854
1924
  "set_hostname_cmd",
855
1925
  "ssh",
1926
+ "ssh_await",
856
1927
  "ssh_cmd",
1928
+ "ssh_keygen_remove",
1929
+ "ssh_keygen_remove_cmd",
1930
+ "ssh_keyscan",
1931
+ "ssh_keyscan_cmd",
857
1932
  "ssh_opts_cmd",
858
1933
  "sudo_cmd",
859
1934
  "sudo_nopasswd_cmd",
1935
+ "symlink",
860
1936
  "symlink_cmd",
1937
+ "tee_cmd",
1938
+ "touch",
861
1939
  "touch_cmd",
1940
+ "update_ca_certificates",
1941
+ "useradd",
1942
+ "useradd_cmd",
1943
+ "uv_run",
862
1944
  "uv_run_cmd",
1945
+ "yield_git_repo",
863
1946
  "yield_ssh_temp_dir",
864
1947
  ]