lts-backup-cli 2.0.0__py3-none-any.whl

lts_backup_cli/mariadb.py

@@ -0,0 +1,157 @@
+"""Adapter around the MariaDB client binaries. Runs on the web server (via its Runner) against
+that server's localhost database; credentials via --defaults-extra-file."""
+
+from __future__ import annotations
+
+import tempfile
+from dataclasses import dataclass
+
+from . import validation as v
+from .runner import Runner
+
+# Positional args keep the script constant: nothing user-controlled is interpolated.
+_PROBE = 'for n in "$@"; do command -v "$n" 2>/dev/null; done; true'
+
+
+def probe_binaries(runner: Runner) -> tuple[str, str]:
+    """(client, dumper) available on the runner's host, preferring the mariadb* names."""
+    out = runner.run(
+        ["sh", "-c", _PROBE, "sh", "mariadb", "mysql", "mariadb-dump", "mysqldump"],
+        check=False,
+        mutating=False,
+    ).stdout
+    found = {line.rsplit("/", 1)[-1] for line in out.splitlines()}
+    client = next((n for n in ("mariadb", "mysql") if n in found), "mysql")
+    dumper = next((n for n in ("mariadb-dump", "mysqldump") if n in found), "mysqldump")
+    return client, dumper
+
+
+NATIVE_PLUGIN = "mysql_native_password"
+EXCLUDED_USERS = ("", "root", "mariadb.sys")
+SYSTEM_SCHEMAS = ("information_schema", "mysql", "performance_schema", "sys")
+
+# Database-level privilege columns of mysql.db shared by MariaDB 10.3-10.11 (Grant_priv left out on purpose).
+PRIV_COLUMNS = (
+    ("Select_priv", "SELECT"), ("Insert_priv", "INSERT"), ("Update_priv", "UPDATE"),
+    ("Delete_priv", "DELETE"), ("Create_priv", "CREATE"), ("Drop_priv", "DROP"),
+    ("References_priv", "REFERENCES"), ("Index_priv", "INDEX"), ("Alter_priv", "ALTER"),
+    ("Create_tmp_table_priv", "CREATE TEMPORARY TABLES"), ("Lock_tables_priv", "LOCK TABLES"),
+    ("Create_view_priv", "CREATE VIEW"), ("Show_view_priv", "SHOW VIEW"),
+    ("Create_routine_priv", "CREATE ROUTINE"), ("Alter_routine_priv", "ALTER ROUTINE"),
+    ("Execute_priv", "EXECUTE"), ("Event_priv", "EVENT"), ("Trigger_priv", "TRIGGER"),
+)  # fmt: skip
+
+
+@dataclass
+class DbUser:
+    user: str
+    host: str
+    plugin: str
+    password_hash: str
+    privileges: list[str]
+
+
+def parse_db_users(tsv: str) -> list[DbUser]:
+    """Rows of the db_users() query; a user granted on both `db` and `db\\_` forms is merged."""
+    users: dict[tuple[str, str], DbUser] = {}
+    for line in tsv.splitlines():
+        if not line.strip():
+            continue
+        user, host, plugin, pw_hash, *flags = line.split("\t")
+        if len(flags) != len(PRIV_COLUMNS):
+            raise ValueError(f"unexpected mysql.db row: {line!r}")
+        privs = [name for (_, name), flag in zip(PRIV_COLUMNS, flags) if flag == "Y"]
+        u = users.setdefault(
+            (user, host), DbUser(user, host, plugin or NATIVE_PLUGIN, pw_hash, [])
+        )  # MariaDB 10.3 leaves plugin empty for native passwords
+        u.privileges += [p for p in privs if p not in u.privileges]
+    return list(users.values())
+
+
+class MariaDBClient:
+    def __init__(self, runner: Runner, defaults_file: str, binaries: tuple[str, str]) -> None:
+        """binaries: (client, dumper) as resolved on the runner's host by probe_binaries()."""
+        self.runner = runner
+        self.defaults_file = defaults_file
+        self.client, self.dumper = binaries
+
+    def _defaults(self) -> list[str]:
+        return [f"--defaults-extra-file={self.defaults_file}"]
+
+    def version(self) -> str:
+        return self.runner.run([self.client, "--version"], mutating=False).stdout.strip()
+
+    def ping(self) -> bool:
+        res = self.runner.run(
+            [self.client, *self._defaults(), "-N", "-e", "SELECT 1"],
+            check=False,
+            mutating=False,
+            timeout=10,
+        )
+        return res.returncode == 0
+
+    def list_databases(self) -> list[str]:
+        """User databases, unvalidated: the caller decides what to do with odd names."""
+        res = self.runner.run(
+            [self.client, *self._defaults(), "-N", "-B", "-e", "SHOW DATABASES"], mutating=False
+        )
+        return [d for d in res.stdout.splitlines() if d and d not in SYSTEM_SCHEMAS]
+
+    def dump_argv(self, db: str) -> list[str]:
+        # Without --databases: no CREATE DATABASE/USE, so the dump can be loaded into any name.
+        return [
+            self.dumper,
+            *self._defaults(),
+            "--single-transaction",
+            "--quick",
+            "--routines",
+            "--triggers",
+            v.db_name(db),  # validated: cannot start with '-', so no '--' separator needed
+        ]
+
+    def load_argv(self, db: str) -> list[str]:
+        return [self.client, *self._defaults(), v.db_name(db)]
+
+    def create_database(self, db: str) -> None:
+        db = v.db_name(db)  # validated identifier, safe to interpolate inside backticks
+        self.runner.run(
+            [self.client, *self._defaults(), "-e", f"CREATE DATABASE IF NOT EXISTS `{db}`"]
+        )
+
+    def db_users(self, db: str) -> list[DbUser]:
+        """Accounts with database-level grants on db (read-only)."""
+        db = v.db_name(db)  # validated: safe inside '...'
+        escaped = db.replace("_", "\\\\_")  # grants made on `shop\_db` (e.g. by Virtualmin)
+        cols = ", ".join(f"d.{col}" for col, _ in PRIV_COLUMNS)
+        excluded = ", ".join(f"'{u}'" for u in EXCLUDED_USERS)
+        sql = (
+            f"SELECT d.User, d.Host, u.plugin, "  # noqa: S608 - db is validated by v.db_name
+            f"IF(u.authentication_string <> '', u.authentication_string, u.Password), {cols} "
+            f"FROM mysql.db d JOIN mysql.user u ON u.User = d.User AND u.Host = d.Host "
+            f"WHERE d.Db IN ('{db}', '{escaped}') AND d.User NOT IN ({excluded}) "
+            f"ORDER BY d.User, d.Host"
+        )
+        res = self.runner.run(
+            [self.client, *self._defaults(), "-N", "-B", "-e", sql], mutating=False
+        )
+        return parse_db_users(res.stdout)
+
+    def apply_users(self, db: str, users: list[DbUser]) -> None:
+        """Create/update accounts and grant them on db. SQL is built only from validated fields
+        and sent on stdin, so hashes never reach argv."""
+        db = v.db_name(db)
+        stmts: list[str] = []
+        for u in users:
+            acct = f"'{v.db_user(u.user)}'@'{v.db_host(u.host)}'"
+            pw_hash = v.password_hash(u.password_hash)
+            stmts += [
+                f"CREATE USER IF NOT EXISTS {acct} IDENTIFIED BY PASSWORD '{pw_hash}';",
+                f"ALTER USER {acct} IDENTIFIED BY PASSWORD '{pw_hash}';",
+                f"GRANT {', '.join(v.privileges(u.privileges))} ON `{db}`.* TO {acct};",
+            ]
+        if not stmts:
+            return
+        with tempfile.TemporaryFile() as sql:  # real fd for subprocess; unlinked on creation
+            sql.write(("\n".join(stmts) + "\n").encode())
+            sql.seek(0)
+            self.runner.run([self.client, *self._defaults()], stdin=sql)

lts_backup_cli/restic.py

@@ -0,0 +1,122 @@
+"""Adapter around the restic binary. Only uses flags available in restic >= 0.12 (Ubuntu 22.04)."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from .runner import Runner
+
+GIT_EXCLUDE = ".git"  # restic matches patterns without a leading '/' at any depth
+
+
+@dataclass(frozen=True)
+class Snapshot:
+    id: str
+    short_id: str
+    time: str
+    hostname: str
+    paths: list[str]
+    tags: list[str]
+
+    @classmethod
+    def from_json(cls, d: dict[str, Any]) -> Snapshot:
+        return cls(
+            id=d["id"],
+            short_id=d.get("short_id", d["id"][:8]),
+            time=d["time"],
+            hostname=d.get("hostname", ""),
+            paths=list(d.get("paths") or []),
+            tags=list(d.get("tags") or []),
+        )
+
+
+class ResticClient:
+    def __init__(self, runner: Runner, repository: str, password_file: str) -> None:
+        self.runner = runner
+        self.repository = repository
+        self.password_file = password_file
+
+    def _base(self, *args: str) -> list[str]:
+        return ["restic", "-r", self.repository, "--password-file", self.password_file, *args]
+
+    def version(self) -> str:
+        return self.runner.run(["restic", "version"], mutating=False).stdout.strip()
+
+    def is_initialized(self) -> bool:
+        res = self.runner.run(self._base("cat", "config"), check=False, mutating=False)
+        return res.returncode == 0
+
+    def init(self) -> None:
+        self.runner.run(self._base("init"))
+
+    @staticmethod
+    def _tag_filter(tags: Sequence[str]) -> list[str]:
+        # One comma-joined --tag means "all of these"; repeated --tag would mean "any of these".
+        return ["--tag", ",".join(tags)] if tags else []
+
+    def backup(
+        self,
+        paths: Sequence[str],
+        tags: Sequence[str],
+        excludes: Sequence[str] = (),
+        verbose: bool = False,
+        host: str | None = None,
+    ) -> str:
+        argv = self._base("backup")
+        if host:
+            argv += ["--host", host]
+        for t in tags:
+            argv += ["--tag", t]
+        for e in excludes:
+            argv += ["--exclude", e]
+        if verbose:
+            argv.append("--verbose")
+        argv += ["--", *paths]
+        return self.runner.run(argv).stdout
+
+    def stdin_backup_argv(self, filename: str, tags: Sequence[str], host: str) -> list[str]:
+        """argv for the right side of `dump | restic backup --stdin` (see runner.pipeline)."""
+        argv = self._base("backup", "--stdin", "--stdin-filename", filename, "--host", host)
+        for t in tags:
+            argv += ["--tag", t]
+        return argv
+
+    def snapshots(self, tags: Sequence[str] = ()) -> list[Snapshot]:
+        argv = self._base("snapshots", "--json", *self._tag_filter(tags))
+        out = self.runner.run(argv, mutating=False).stdout
+        return [Snapshot.from_json(d) for d in json.loads(out or "[]")]
+
+    def restore(self, snapshot_id: str, target: str, includes: Sequence[str] = ()) -> None:
+        argv = self._base("restore", snapshot_id, "--target", target)
+        for i in includes:
+            argv += ["--include", i]
+        self.runner.run(argv)
+
+    def dump_argv(self, snapshot_id: str, path: str) -> list[str]:
+        return self._base("dump", snapshot_id, path)
+
+    def forget(
+        self,
+        tags: Sequence[str],
+        *,
+        hourly: int | None = None,
+        daily: int | None = None,
+        weekly: int | None = None,
+        monthly: int | None = None,
+        prune: bool = False,
+    ) -> str:
+        argv = self._base("forget", *self._tag_filter(tags))
+        for flag, val in (
+            ("--keep-hourly", hourly),
+            ("--keep-daily", daily),
+            ("--keep-weekly", weekly),
+            ("--keep-monthly", monthly),
+        ):
+            if val is not None:
+                argv += [flag, str(val)]
+        if prune:
+            argv.append("--prune")
+        return self.runner.run(argv).stdout

lts_backup_cli/runner.py

@@ -0,0 +1,172 @@
+"""The only module allowed to spawn processes. Everything else depends on the Runner protocol."""
+
+from __future__ import annotations
+
+import logging
+import shlex
+import subprocess
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import IO, Protocol
+
+log = logging.getLogger("ltsbu")
+
+
+class CommandError(RuntimeError):
+    def __init__(self, argv: Sequence[str], returncode: int, stderr: str) -> None:
+        self.argv = list(argv)
+        self.returncode = returncode
+        self.stderr = stderr
+        super().__init__(f"{argv[0]} exited {returncode}: {stderr.strip()[-2000:]}")
+
+
+@dataclass
+class Result:
+    returncode: int
+    stdout: str = ""
+    stderr: str = ""
+
+
+class Runner(Protocol):
+    dry_run: bool
+
+    def run(
+        self,
+        argv: Sequence[str],
+        *,
+        env: Mapping[str, str] | None = None,
+        stdin: IO[bytes] | None = None,
+        stdout: IO[bytes] | None = None,
+        check: bool = True,
+        timeout: float | None = None,
+        mutating: bool = True,
+    ) -> Result: ...
+
+
+class SubprocessRunner:
+    dry_run = False
+
+    def run(
+        self,
+        argv: Sequence[str],
+        *,
+        env: Mapping[str, str] | None = None,
+        stdin: IO[bytes] | None = None,
+        stdout: IO[bytes] | None = None,
+        check: bool = True,
+        timeout: float | None = None,
+        mutating: bool = True,
+    ) -> Result:
+        log.debug("exec: %s", shlex.join(argv))
+        import os
+
+        full_env = {**os.environ, **(env or {})}
+        try:
+            proc = subprocess.run(  # noqa: S603 - argv list, never shell
+                list(argv),
+                env=full_env,
+                stdin=stdin,
+                stdout=stdout if stdout is not None else subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=timeout,
+                check=False,
+            )
+        except FileNotFoundError:
+            raise CommandError(argv, 127, f"command not found: {argv[0]}") from None
+        except subprocess.TimeoutExpired:
+            raise CommandError(argv, 124, f"timed out after {timeout}s") from None
+        out = proc.stdout.decode(errors="replace") if isinstance(proc.stdout, bytes) else ""
+        err = proc.stderr.decode(errors="replace") if proc.stderr else ""
+        if check and proc.returncode != 0:
+            raise CommandError(argv, proc.returncode, err)
+        return Result(proc.returncode, out, err)
+
+
+class DryRunRunner(SubprocessRunner):
+    """Executes read-only commands (mutating=False); only prints mutating ones."""
+
+    dry_run = True
+
+    def __init__(self, echo: bool = True) -> None:
+        self.echo = echo
+        self.planned: list[list[str]] = []
+
+    def run(
+        self,
+        argv: Sequence[str],
+        *,
+        env: Mapping[str, str] | None = None,
+        stdin: IO[bytes] | None = None,
+        stdout: IO[bytes] | None = None,
+        check: bool = True,
+        timeout: float | None = None,
+        mutating: bool = True,
+    ) -> Result:
+        if not mutating:
+            return super().run(
+                argv, env=env, stdin=stdin, stdout=stdout, check=check, timeout=timeout
+            )
+        self.planned.append(list(argv))
+        if self.echo:
+            import click
+
+            click.echo(f"[dry-run] {shlex.join(argv)}")
+        return Result(0)
+
+
+def pipeline(left: Sequence[str], right: Sequence[str]) -> list[str]:
+    """`left | right` without shell=True: each side is quoted; pipefail surfaces left failures."""
+    return ["bash", "-o", "pipefail", "-c", f"{shlex.join(left)} | {shlex.join(right)}"]
+
+
+class SshRunner:
+    """Runs argv on a remote host. sshd hands the command line to the remote shell, so every
+    argument is shlex-quoted; a drop-in Runner for adapters."""
+
+    def __init__(self, inner: Runner, alias: str, connect_timeout: int = 10) -> None:
+        if not alias or alias.startswith("-") or not all(c.isalnum() or c in "._-" for c in alias):
+            raise ValueError(f"unsafe ssh host alias {alias!r}")
+        self.inner = inner
+        self.alias = alias
+        self.connect_timeout = connect_timeout
+
+    @property
+    def dry_run(self) -> bool:
+        return self.inner.dry_run
+
+    @dry_run.setter
+    def dry_run(self, value: bool) -> None:
+        self.inner.dry_run = value
+
+    def wrap(self, argv: Sequence[str]) -> list[str]:
+        return [
+            "ssh",
+            "-o",
+            "BatchMode=yes",
+            "-o",
+            f"ConnectTimeout={self.connect_timeout}",
+            self.alias,
+            "--",
+            shlex.join(argv),
+        ]
+
+    def run(
+        self,
+        argv: Sequence[str],
+        *,
+        env: Mapping[str, str] | None = None,
+        stdin: IO[bytes] | None = None,
+        stdout: IO[bytes] | None = None,
+        check: bool = True,
+        timeout: float | None = None,
+        mutating: bool = True,
+    ) -> Result:
+        # env is not forwarded: sshd drops it unless AcceptEnv, and secrets never travel that way
+        return self.inner.run(
+            self.wrap(argv),
+            stdin=stdin,
+            stdout=stdout,
+            check=check,
+            timeout=timeout,
+            mutating=mutating,
+        )

lts_backup_cli/schedule.py

@@ -0,0 +1,111 @@
+"""Periodic backups as systemd timers. Rendering is pure; install/uninstall go through the Runner."""
+
+from __future__ import annotations
+
+import os
+import zlib
+from pathlib import Path
+
+from . import config as c
+from . import validation as v
+from .runner import Runner
+
+# Night-time weekly slots (01:00-04:30, every 30 min) spread over the week, so heavy
+# weekly runs (prune) of different repos land on different nights.
+WEEKLY_TIMES = tuple(f"{h:02d}:{m:02d}" for h in range(1, 5) for m in (0, 30))
+_SLOTS = tuple((d, t) for d in v.WEEKDAYS for t in WEEKLY_TIMES)
+
+
+def default_schedule(
+    server: str, repo: str, taken: set[tuple[str, str]] | frozenset[tuple[str, str]] = frozenset()
+) -> c.Schedule:
+    """Every 2h at a per-repo minute, plus a weekly run in a free night slot. Deterministic."""
+    h = zlib.crc32(f"{server}/{repo}".encode())
+    start = h % len(_SLOTS)
+    order = _SLOTS[start:] + _SLOTS[:start]
+    day, time = next((s for s in order if s not in taken), order[0])
+    return c.Schedule(every_hours=2, minute=h % 60, weekly_day=day, weekly_time=time)
+
+
+def interval_calendar(s: c.Schedule) -> str:
+    # shortcut: N/h restarts at midnight, so a non-divisor of 24 gives one short gap a day
+    return f"*-*-* 00/{s.every_hours}:{s.minute:02d}:00"
+
+
+def weekly_calendar(s: c.Schedule) -> str:
+    return f"{s.weekly_day} *-*-* {s.weekly_time}:00"
+
+
+def unit_base(server: str, repo: str) -> str:
+    return f"ltsbu-{server}-{repo}"
+
+
+def _no_space(path: str) -> str:
+    if any(ch.isspace() for ch in path):
+        raise v.ValidationError(f"path with whitespace cannot be used in a systemd unit: {path!r}")
+    return path
+
+
+def render_units(
+    server: str, repo: str, sch: c.Schedule, python: str, config_path: str
+) -> dict[str, str]:
+    """{filename: text} for the interval and weekly service/timer pairs."""
+    base = unit_base(v.label(server), v.label(repo))
+    cmd = (
+        f"{_no_space(python)} -m lts_backup_cli --config {_no_space(config_path)} backup "
+        f"--server {server} --repo {repo}"
+    )
+    units: dict[str, str] = {}
+    for name, calendar, extra, what in (
+        (base, interval_calendar(sch), "", f"every {sch.every_hours}h"),
+        (f"{base}-weekly", weekly_calendar(sch), " --weekly", "weekly, with prune"),
+    ):
+        units[f"{name}.service"] = (
+            "[Unit]\n"
+            f"Description=ltsbu backup {server}/{repo} ({what})\n"
+            "Wants=network-online.target\nAfter=network-online.target\n\n"
+            "[Service]\nType=oneshot\nNice=10\nIOSchedulingClass=idle\n"
+            f"ExecStart={cmd}{extra}\n"
+        )
+        units[f"{name}.timer"] = (
+            "[Unit]\n"
+            f"Description=ltsbu backup {server}/{repo} ({what})\n\n"
+            "[Timer]\n"
+            f"OnCalendar={calendar}\n"
+            "Persistent=true\n"
+            "RandomizedDelaySec=5min\n\n"
+            "[Install]\nWantedBy=timers.target\n"
+        )
+    return units
+
+
+def unit_dir(root: bool) -> Path:
+    if root:
+        return Path("/etc/systemd/system")
+    base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
+    return Path(base) / "systemd" / "user"
+
+
+def _systemctl(user: bool, *args: str) -> list[str]:
+    return ["systemctl", *(["--user"] if user else []), *args]
+
+
+def install(runner: Runner, units: dict[str, str], directory: Path, user: bool) -> None:
+    if not runner.dry_run:
+        directory.mkdir(parents=True, exist_ok=True)
+        for name, text in units.items():
+            (directory / name).write_text(text)
+    runner.run(_systemctl(user, "daemon-reload"))
+    runner.run(
+        _systemctl(user, "enable", "--now", *sorted(n for n in units if n.endswith(".timer")))
+    )
+
+
+def uninstall(runner: Runner, server: str, repo: str, directory: Path, user: bool) -> None:
+    base = unit_base(server, repo)
+    names = [f"{base}{sfx}" for sfx in (".timer", "-weekly.timer", ".service", "-weekly.service")]
+    runner.run(_systemctl(user, "disable", "--now", *names[:2]), check=False)
+    if not runner.dry_run:
+        for n in names:
+            (directory / n).unlink(missing_ok=True)
+    runner.run(_systemctl(user, "daemon-reload"))