dycw-utilities 0.174.5__py3-none-any.whl → 0.174.7__py3-none-any.whl

{dycw_utilities-0.174.5.dist-info → dycw_utilities-0.174.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dycw-utilities
-Version: 0.174.5
+Version: 0.174.7
 Author: Derek Wan
 Author-email: Derek Wan <d.wan@icloud.com>
 Requires-Dist: atomicwrites>=1.4.1,<1.5

{dycw_utilities-0.174.5.dist-info → dycw_utilities-0.174.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-utilities/__init__.py,sha256=HSEHPwrjbITUKM0rHSI676yh7VgQMNCB5ksk8o1Hb8M,60
+utilities/__init__.py,sha256=m6IBJ6HmwjCqt8xNdi4aHkF4CjRw4nrxq8SpNIQAq2Q,60
 utilities/aeventkit.py,sha256=OmDBhYGgbsKrB7cdC5FFpJHUatX9O76eTeKVVTksp2Y,12673
 utilities/altair.py,sha256=rUK99g9x6CYDDfiZrf-aTx5fSRbL1Q8ctgKORowzXHg,9060
 utilities/asyncio.py,sha256=aJySVxBY0gqsIYnoNmH7-1r8djKuf4vSsU69VCD08t8,16772
@@ -80,7 +80,7 @@ utilities/sqlalchemy.py,sha256=HQYpd7LFxdTF5WYVWYtCJeEBI71EJm7ytvCGyAH9B-U,37163
 utilities/sqlalchemy_polars.py,sha256=JCGhB37raSR7fqeWV5dTsciRTMVzIdVT9YSqKT0piT0,13370
 utilities/statsmodels.py,sha256=koyiBHvpMcSiBfh99wFUfSggLNx7cuAw3rwyfAhoKpQ,3410
 utilities/string.py,sha256=shmBK87zZwzGyixuNuXCiUbqzfeZ9xlrFwz6JTaRvDk,582
-utilities/subprocess.py,sha256=r91VteJ7I1upuiWxRT2kvm17xffS5eXMZ3CJXdmZT4s,15686
+utilities/subprocess.py,sha256=1mcFSteDnzIfFZT-UOWPg8lBjDKjRWvgSs5hzHXvsts,22231
 utilities/tempfile.py,sha256=Lx6qa16lL1XVH6WdmD_G9vlN6gLI8nrIurxmsFkPKvg,3022
 utilities/testbook.py,sha256=j1KmaVbrX9VrbeMgtPh5gk55myAsn3dyRUn7jGbPbRk,1294
 utilities/text.py,sha256=7SvwcSR2l_5cOrm1samGnR4C-ZI6qyFLHLzSpO1zeHQ,13958
@@ -97,7 +97,7 @@ utilities/warnings.py,sha256=un1LvHv70PU-LLv8RxPVmugTzDJkkGXRMZTE2-fTQHw,1771
 utilities/whenever.py,sha256=F4ek0-OBWxHYrZdmoZt76N2RnNyKY5KrEHt7rqO4AQE,60183
 utilities/zipfile.py,sha256=24lQc9ATcJxHXBPc_tBDiJk48pWyRrlxO2fIsFxU0A8,699
 utilities/zoneinfo.py,sha256=tdIScrTB2-B-LH0ukb1HUXKooLknOfJNwHk10MuMYvA,3619
-dycw_utilities-0.174.5.dist-info/WHEEL,sha256=ZyFSCYkV2BrxH6-HRVRg3R9Fo7MALzer9KiPYqNxSbo,79
-dycw_utilities-0.174.5.dist-info/entry_points.txt,sha256=ykGI1ArwOPHqm2g5Cqh3ENdMxEej_a_FcOUov5EM5Oc,155
-dycw_utilities-0.174.5.dist-info/METADATA,sha256=qYrj2g20dkkvXEdb1v43eAPpIoIS2vJOu4nM5QEFG54,1709
-dycw_utilities-0.174.5.dist-info/RECORD,,
+dycw_utilities-0.174.7.dist-info/WHEEL,sha256=ZyFSCYkV2BrxH6-HRVRg3R9Fo7MALzer9KiPYqNxSbo,79
+dycw_utilities-0.174.7.dist-info/entry_points.txt,sha256=ykGI1ArwOPHqm2g5Cqh3ENdMxEej_a_FcOUov5EM5Oc,155
+dycw_utilities-0.174.7.dist-info/METADATA,sha256=q0-Hb1ta7Jppeysc4MdeoUaTPr3Nqwwb0uiMzysigw8,1709
+dycw_utilities-0.174.7.dist-info/RECORD,,

utilities/__init__.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
-__version__ = "0.174.5"
+__version__ = "0.174.7"

utilities/subprocess.py

@@ -2,15 +2,18 @@ from __future__ import annotations
 
 import sys
 from contextlib import contextmanager
+from dataclasses import dataclass
 from io import StringIO
 from pathlib import Path
+from shlex import join, quote
 from string import Template
 from subprocess import PIPE, CalledProcessError, Popen
 from threading import Thread
 from time import sleep
-from typing import IO, TYPE_CHECKING, Literal, assert_never, overload
+from typing import IO, TYPE_CHECKING, Literal, assert_never, overload, override
 
 from utilities.errors import ImpossibleCaseError
+from utilities.iterables import always_iterable
 from utilities.logging import to_logger
 from utilities.text import strip_and_dedent
 from utilities.whenever import to_seconds
@@ -18,13 +21,63 @@ from utilities.whenever import to_seconds
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
-    from utilities.types import LoggerLike, PathLike, Retry, StrMapping, StrStrMapping
+    from utilities.types import (
+        LoggerLike,
+        MaybeIterable,
+        PathLike,
+        Retry,
+        StrMapping,
+        StrStrMapping,
+    )
 
 
 _HOST_KEY_ALGORITHMS = ["ssh-ed25519"]
+APT_UPDATE = ["apt", "update", "-y"]
 BASH_LC = ["bash", "-lc"]
 BASH_LS = ["bash", "-ls"]
 MKTEMP_DIR_CMD = ["mktemp", "-d"]
+RESTART_SSHD = ["systemctl", "restart", "sshd"]
+UPDATE_CA_CERTIFICATES: str = "update-ca-certificates"
+
+
+def apt_install_cmd(package: str, /) -> list[str]:
+    return ["apt", "install", "-y", package]
+
+
+def cat_cmd(path: PathLike, /) -> list[str]:
+    return ["cat", str(path)]
+
+
+def cd_cmd(path: PathLike, /) -> list[str]:
+    return ["cd", str(path)]
+
+
+def chmod_cmd(path: PathLike, mode: str, /) -> list[str]:
+    return ["chmod", mode, str(path)]
+
+
+def chown_cmd(
+    path: PathLike, /, *, user: str | None = None, group: str | None = None
+) -> list[str]:
+    match user, group:
+        case None, None:
+            raise ChownCmdError
+        case str(), None:
+            ownership = "user"
+        case None, str():
+            ownership = f":{group}"
+        case str(), str():
+            ownership = f"{user}:{group}"
+        case never:
+            assert_never(never)
+    return ["chown", ownership, str(path)]
+
+
+@dataclass(kw_only=True, slots=True)
+class ChownCmdError(Exception):
+    @override
+    def __str__(self) -> str:
+        return "At least one of 'user' and/or 'group' must be given; got None"
 
 
 def cp_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
@@ -45,6 +98,15 @@ def expand_path(
     return Path(path).expanduser()
 
 
+def git_clone_cmd(url: str, path: PathLike, /) -> list[str]:
+    return ["git", "clone", "--recurse-submodules", url, str(path)]
+
+
+def git_hard_reset_cmd(*, branch: str | None = None) -> list[str]:
+    branch_use = "master" if branch is None else branch
+    return ["git", "hard-reset", branch_use]
+
+
 def maybe_sudo_cmd(cmd: str, /, *args: str, sudo: bool = False) -> list[str]:
     parts: list[str] = [cmd, *args]
     return sudo_cmd(*parts) if sudo else parts
@@ -60,8 +122,13 @@ def mkdir(path: PathLike, /, *, sudo: bool = False, parent: bool = False) -> Non
 
 
 def mkdir_cmd(path: PathLike, /, *, parent: bool = False) -> list[str]:
-    path_use = f"$(dirname {path})" if parent else path
-    return ["mkdir", "-p", str(path_use)]
+    args: list[str] = ["mkdir", "-p"]
+    quoted = quote(str(path))
+    if parent:
+        args.append(f"$(dirname {quoted})")
+    else:
+        args.append(quoted)
+    return args
 
 
 def mv_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
@@ -72,6 +139,114 @@ def rm_cmd(path: PathLike, /) -> list[str]:
     return ["rm", "-rf", str(path)]
 
 
+def rsync(
+    src_or_srcs: MaybeIterable[PathLike],
+    user: str,
+    hostname: str,
+    dest: PathLike,
+    /,
+    *,
+    sudo: bool = False,
+    batch_mode: bool = True,
+    host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
+    strict_host_key_checking: bool = True,
+    print: bool = False,  # noqa: A002
+    retry: Retry | None = None,
+    logger: LoggerLike | None = None,
+    archive: bool = False,
+    chown_user: str | None = None,
+    chown_group: str | None = None,
+    exclude: MaybeIterable[str] | None = None,
+    chmod: str | None = None,
+) -> None:
+    mkdir_args = maybe_sudo_cmd(*mkdir_cmd(dest, parent=True), sudo=sudo)  # skipif-ci
+    ssh(  # skipif-ci
+        user,
+        hostname,
+        *mkdir_args,
+        batch_mode=batch_mode,
+        host_key_algorithms=host_key_algorithms,
+        strict_host_key_checking=strict_host_key_checking,
+        print=print,
+        retry=retry,
+        logger=logger,
+    )
+    rsync_args = rsync_cmd(  # skipif-ci
+        src_or_srcs,
+        user,
+        hostname,
+        dest,
+        archive=archive,
+        chown_user=chown_user,
+        chown_group=chown_group,
+        exclude=exclude,
+        batch_mode=batch_mode,
+        host_key_algorithms=host_key_algorithms,
+        strict_host_key_checking=strict_host_key_checking,
+        sudo=sudo,
+    )
+    run(*rsync_args, print=print, retry=retry, logger=logger)  # skipif-ci
+    if chmod is not None:  # skipif-ci
+        chmod_args = maybe_sudo_cmd(*chmod_cmd(dest, chmod), sudo=sudo)
+        ssh(
+            user,
+            hostname,
+            *chmod_args,
+            batch_mode=batch_mode,
+            host_key_algorithms=host_key_algorithms,
+            strict_host_key_checking=strict_host_key_checking,
+            print=print,
+            retry=retry,
+            logger=logger,
+        )
+
+
+def rsync_cmd(
+    src_or_srcs: MaybeIterable[PathLike],
+    user: str,
+    hostname: str,
+    dest: PathLike,
+    /,
+    *,
+    archive: bool = False,
+    chown_user: str | None = None,
+    chown_group: str | None = None,
+    exclude: MaybeIterable[str] | None = None,
+    batch_mode: bool = True,
+    host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
+    strict_host_key_checking: bool = True,
+    sudo: bool = False,
+) -> list[str]:
+    args: list[str] = ["rsync"]
+    if archive:
+        args.append("--archive")
+    args.append("--checksum")
+    match chown_user, chown_group:
+        case None, None:
+            ...
+        case str(), None:
+            args.extend(["--chown", chown_user])
+        case None, str():
+            args.extend(["--chown", f":{chown_group}"])
+        case str(), str():
+            args.extend(["--chown", f"{chown_user}:{chown_group}"])
+        case never:
+            assert_never(never)
+    args.append("--compress")
+    if exclude is not None:
+        for exclude_i in always_iterable(exclude):
+            args.extend(["--exclude", exclude_i])
+    rsh_args: list[str] = ssh_opts_cmd(
+        batch_mode=batch_mode,
+        host_key_algorithms=host_key_algorithms,
+        strict_host_key_checking=strict_host_key_checking,
+    )
+    args.extend(["--rsh", join(rsh_args)])
+    if sudo:
+        args.extend(["--rsync-path", join(sudo_cmd("rsync"))])
+    return [*args, *map(str, always_iterable(src_or_srcs)), f"{user}@{hostname}:{dest}"]
+
+
 @overload
 def run(
     cmd: str,
@@ -224,8 +399,8 @@ def run(
         if proc.stderr is None:  # pragma: no cover
             raise ImpossibleCaseError(case=[f"{proc.stderr=}"])
         with (
-            _yield_write(proc.stdout, *stdout_outputs),
-            _yield_write(proc.stderr, *stderr_outputs),
+            _run_yield_write(proc.stdout, *stdout_outputs),
+            _run_yield_write(proc.stderr, *stderr_outputs),
         ):
             if input is not None:
                 _ = proc.stdin.write(input)
@@ -307,8 +482,8 @@ def run(
 
 
 @contextmanager
-def _yield_write(input_: IO[str], /, *outputs: IO[str]) -> Iterator[None]:
-    thread = Thread(target=_run_target, args=(input_, *outputs), daemon=True)
+def _run_yield_write(input_: IO[str], /, *outputs: IO[str]) -> Iterator[None]:
+    thread = Thread(target=_run_daemon_target, args=(input_, *outputs), daemon=True)
     thread.start()
     try:
         yield
@@ -316,17 +491,21 @@ def _yield_write(input_: IO[str], /, *outputs: IO[str]) -> Iterator[None]:
         thread.join()
 
 
-def _run_target(input_: IO[str], /, *outputs: IO[str]) -> None:
+def _run_daemon_target(input_: IO[str], /, *outputs: IO[str]) -> None:
     with input_:
         for text in iter(input_.readline, ""):
-            _write_to_streams(text, *outputs)
+            _run_write_to_streams(text, *outputs)
 
 
-def _write_to_streams(text: str, /, *outputs: IO[str]) -> None:
+def _run_write_to_streams(text: str, /, *outputs: IO[str]) -> None:
     for output in outputs:
         _ = output.write(text)
 
 
+def set_hostname_cmd(hostname: str, /) -> list[str]:
+    return ["hostnamectl", "set-hostname", hostname]
+
+
 @overload
 def ssh(
     user: str,
@@ -470,6 +649,20 @@ def ssh_cmd(
     batch_mode: bool = True,
     host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
     strict_host_key_checking: bool = True,
+) -> list[str]:
+    args: list[str] = ssh_opts_cmd(
+        batch_mode=batch_mode,
+        host_key_algorithms=host_key_algorithms,
+        strict_host_key_checking=strict_host_key_checking,
+    )
+    return [*args, f"{user}@{hostname}", *cmd_and_cmds_or_args]
+
+
+def ssh_opts_cmd(
+    *,
+    batch_mode: bool = True,
+    host_key_algorithms: list[str] = _HOST_KEY_ALGORITHMS,
+    strict_host_key_checking: bool = True,
 ) -> list[str]:
     args: list[str] = ["ssh"]
     if batch_mode:
@@ -477,17 +670,44 @@ def ssh_cmd(
     args.extend(["-o", f"HostKeyAlgorithms={','.join(host_key_algorithms)}"])
     if strict_host_key_checking:
         args.extend(["-o", "StrictHostKeyChecking=yes"])
-    return [*args, "-T", f"{user}@{hostname}", *cmd_and_cmds_or_args]
+    return [*args, "-T"]
+
+
+def ssh_keygen_cmd(hostname: str, /) -> list[str]:
+    return ["ssh-keygen", "-f", "~/.ssh/known_hosts", "-R", hostname]
 
 
 def sudo_cmd(cmd: str, /, *args: str) -> list[str]:
     return ["sudo", cmd, *args]
 
 
+def sudo_nopasswd_cmd(user: str, /) -> str:
+    return f"{user} ALL=(ALL) NOPASSWD: ALL"
+
+
+def symlink_cmd(src: PathLike, dest: PathLike, /) -> list[str]:
+    return ["ln", "-s", str(src), str(dest)]
+
+
 def touch_cmd(path: PathLike, /) -> list[str]:
     return ["touch", str(path)]
 
 
+def uv_run_cmd(module: str, /, *args: str) -> list[str]:
+    return [
+        "uv",
+        "run",
+        "--no-dev",
+        "--active",
+        "--prerelease=disallow",
+        "--managed-python",
+        "python",
+        "-m",
+        module,
+        *args,
+    ]
+
+
 @contextmanager
 def yield_ssh_temp_dir(
     user: str,
@@ -498,12 +718,12 @@ def yield_ssh_temp_dir(
     logger: LoggerLike | None = None,
     keep: bool = False,
 ) -> Iterator[Path]:
-    path = Path(
+    path = Path(  # skipif-ci
         ssh(user, hostname, *MKTEMP_DIR_CMD, return_=True, retry=retry, logger=logger)
     )
-    try:
+    try:  # skipif-ci
         yield path
-    finally:
+    finally:  # skipif-ci
         if keep:
             if logger is not None:
                 to_logger(logger).info("Keeping temporary directory '%s'...", path)
@@ -512,21 +732,38 @@ def yield_ssh_temp_dir(
 
 
 __all__ = [
+    "APT_UPDATE",
     "BASH_LC",
     "BASH_LS",
     "MKTEMP_DIR_CMD",
+    "RESTART_SSHD",
+    "UPDATE_CA_CERTIFICATES",
+    "ChownCmdError",
+    "apt_install_cmd",
+    "cd_cmd",
+    "chmod_cmd",
+    "chown_cmd",
     "cp_cmd",
     "echo_cmd",
     "expand_path",
+    "git_clone_cmd",
+    "git_hard_reset_cmd",
     "maybe_sudo_cmd",
     "mkdir",
     "mkdir_cmd",
     "mv_cmd",
     "rm_cmd",
+    "rsync",
+    "rsync_cmd",
     "run",
+    "set_hostname_cmd",
     "ssh",
     "ssh_cmd",
+    "ssh_opts_cmd",
     "sudo_cmd",
+    "sudo_nopasswd_cmd",
+    "symlink_cmd",
     "touch_cmd",
+    "uv_run_cmd",
     "yield_ssh_temp_dir",
 ]