lts-backup-cli 2.0.0__py3-none-any.whl

lts_backup_cli/__init__.py

@@ -0,0 +1,3 @@
+"""lts-backup-cli: restic + MariaDB backup utility."""
+
+__version__ = "2.0.0"

lts_backup_cli/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

lts_backup_cli/apache.py

@@ -0,0 +1,45 @@
+"""Apache/httpd vhost discovery. Pure: config text in, vhosts out."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from . import validation as v
+
+# Ubuntu (sites-enabled), Rocky (conf.d) and Virtualmin on Rocky (vhosts appended to httpd.conf).
+CONF_GLOBS = (
+    "/etc/apache2/sites-enabled/*",
+    "/etc/httpd/conf/httpd.conf",
+    "/etc/httpd/conf.d/*.conf",
+)
+
+_BLOCK_RE = re.compile(r"<VirtualHost\b[^>]*>(.*?)</VirtualHost>", re.IGNORECASE | re.DOTALL)
+_DIRECTIVE_RE = re.compile(r"^\s*(ServerName|DocumentRoot)\s+(\S+)", re.IGNORECASE | re.MULTILINE)
+
+
+@dataclass(frozen=True)
+class VHost:
+    server_name: str
+    document_root: str
+
+
+def parse_vhosts(text: str) -> list[VHost]:
+    """Named vhosts with a valid absolute DocumentRoot, deduplicated (:80/:443 pairs), sorted.
+    Nameless (default) vhosts are skipped: add their docroot as a folder repo instead."""
+    # shortcut: no Include/Macro expansion, upgrade if a server uses mod_macro
+    text = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))
+    found: set[VHost] = set()
+    for block in _BLOCK_RE.findall(text):
+        values = {k.lower(): val.strip("\"'") for k, val in _DIRECTIVE_RE.findall(block)}
+        name, root = values.get("servername", ""), values.get("documentroot", "")
+        try:
+            found.add(VHost(v.hostname(name.split(":")[0]), v.remote_abs_path(root)))
+        except v.ValidationError:
+            continue
+    return sorted(found, key=lambda h: (h.server_name, h.document_root))
+
+
+def repo_name_for(server_name: str) -> str:
+    """vhost name -> repo label (dots are not allowed in labels)."""
+    return re.sub(r"[^a-z0-9_-]", "-", server_name.lower())[:63]

lts_backup_cli/cli.py

@@ -0,0 +1,559 @@
+"""Composition root: parses options, wires adapters/services, maps errors to exit codes."""
+
+from __future__ import annotations
+
+import functools
+import json
+import logging
+import os
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Callable, TypeVar
+
+import click
+
+from . import __version__, doctor, schedule, services, sshconfig
+from . import config as c
+from . import validation as v
+from . import wizard as wz
+from .runner import CommandError, DryRunRunner, Runner, SubprocessRunner
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+EXIT_DOCTOR_FAILED = 3
+
+
+@dataclass
+class State:
+    config_path: Path | None = None
+    server: str | None = None
+    repo: str | None = None
+    dry_run: bool = False
+    verbose: bool = False
+    # Injection points for tests:
+    runner: Runner | None = None
+    asker: wz.Asker | None = None
+    ssh_config_path: Path | None = None
+    units_dir: Path | None = None
+
+    def get_runner(self) -> Runner:
+        logging.basicConfig(
+            level=logging.DEBUG if self.verbose else logging.WARNING,
+            format="%(message)s",
+            stream=sys.stderr,
+        )
+        if self.runner is not None:
+            self.runner.dry_run = self.dry_run
+            return self.runner
+        return DryRunRunner() if self.dry_run else SubprocessRunner()
+
+    @property
+    def cfg_path(self) -> Path:
+        return (self.config_path or c.default_config_path()).absolute()
+
+    @property
+    def ssh_path(self) -> Path:
+        return self.ssh_config_path or sshconfig.default_path()
+
+    @property
+    def user_units(self) -> bool:
+        return os.geteuid() != 0
+
+    @property
+    def units_path(self) -> Path:
+        return self.units_dir or schedule.unit_dir(root=not self.user_units)
+
+    def load(self) -> c.Config:
+        return c.load(self.cfg_path)
+
+    def servers(self, cfg: c.Config) -> list[c.ServerConfig]:
+        if self.server:
+            return [cfg.server(v.label(self.server))]
+        return sorted(cfg.servers.values(), key=lambda s: s.label)
+
+    def pick(self, cfg: c.Config) -> tuple[c.ServerConfig, c.RepoConfig]:
+        """--server/--repo, or the only candidate when there is exactly one."""
+        if not cfg.servers:
+            raise c.ConfigError(NO_SERVERS)
+        repo = v.label(self.repo) if self.repo else None
+        candidates = [
+            (s, r)
+            for s in self.servers(cfg)
+            for r in sorted(s.repos.values(), key=lambda r: r.name)
+            if repo is None or r.name == repo
+        ]
+        if len(candidates) == 1:
+            return candidates[0]
+        if not candidates:
+            if repo and self.server:
+                cfg.server(self.server).repo(repo)  # raises with the known names
+            raise c.ConfigError(f"unknown repo {repo!r}" if repo else "no repos configured")
+        if repo:
+            servers = ", ".join(s.label for s, _ in candidates)
+            raise c.ConfigError(
+                f"repo {repo!r} exists on several servers ({servers}); use --server"
+            )
+        names = ", ".join(f"{s.label}/{r.name}" for s, r in candidates)
+        raise c.ConfigError(f"several repos configured ({names}); use --repo (and --server)")
+
+
+NO_SERVERS = "no servers configured; run 'ltsbu discover <host>'"
+
+
+class Group(click.Group):
+    """Turns domain errors into clean 'Error: ...' messages with exit code 1."""
+
+    def invoke(self, ctx: click.Context) -> Any:
+        try:
+            return super().invoke(ctx)
+        except (
+            v.ValidationError,
+            c.ConfigError,
+            services.ServiceError,
+            wz.WizardError,
+            CommandError,
+        ) as e:
+            raise click.ClickException(str(e)) from None
+        except KeyboardInterrupt:
+            raise click.Abort() from None
+
+
+def common(f: F) -> F:
+    """--server, --repo, --dry-run, --verbose also accepted after the subcommand name."""
+
+    @click.option("--server", "-s", metavar="LABEL", help="Server label.")
+    @click.option("--repo", "-r", metavar="NAME", help="Repo name on the server.")
+    @click.option(
+        "--dry-run", is_flag=True, help="Print mutating commands instead of running them."
+    )
+    @click.option("--verbose", "-v", is_flag=True, help="Show executed commands.")
+    @functools.wraps(f)
+    def wrapper(
+        *args: Any, server: str | None, repo: str | None, dry_run: bool, verbose: bool, **kw: Any
+    ) -> Any:
+        st = click.get_current_context().ensure_object(State)
+        st.server = server or st.server
+        st.repo = repo or st.repo
+        st.dry_run = dry_run or st.dry_run
+        st.verbose = verbose or st.verbose
+        return f(*args, **kw)
+
+    return wrapper  # type: ignore[return-value]
+
+
+def _confirm(st: State, yes: bool, message: str) -> None:
+    if st.dry_run or yes:
+        return
+    if not sys.stdin.isatty():
+        raise click.ClickException(f"{message} -- refusing without --yes (no terminal)")
+    click.confirm(message, abort=True)
+
+
+@click.group(cls=Group, context_settings={"help_option_names": ["-h", "--help"]})
+@click.option(
+    "--config",
+    "config_path",
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Config file (default ~/.config/ltsbu/config.toml).",
+)
+@click.option("--server", "-s", metavar="LABEL", help="Server label.")
+@click.option("--repo", "-r", metavar="NAME", help="Repo name on the server.")
+@click.option("--dry-run", is_flag=True, help="Print mutating commands instead of running them.")
+@click.option("--verbose", "-v", is_flag=True, help="Show executed commands.")
+@click.version_option(__version__, prog_name="ltsbu")
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    config_path: Path | None,
+    server: str | None,
+    repo: str | None,
+    dry_run: bool,
+    verbose: bool,
+) -> None:
+    """ltsbu - per-server restic backups of Apache vhosts, MariaDB databases and folders."""
+    st = ctx.obj if isinstance(ctx.obj, State) else State()
+    st.config_path = config_path or st.config_path
+    st.server, st.repo, st.dry_run, st.verbose = server, repo, dry_run, verbose
+    ctx.obj = st
+
+
+# --- doctor -----------------------------------------------------------------------------
+
+
+@cli.command("doctor")
+@common
+@click.pass_obj
+def doctor_cmd(st: State) -> None:
+    """Check SSH, server binaries, databases, storage, repositories and timers."""
+    _run_doctor(st)
+
+
+_ICON = {
+    doctor.OK: click.style("OK  ", fg="green"),
+    doctor.WARN: click.style("WARN", fg="yellow"),
+    doctor.FAIL: click.style("FAIL", fg="red"),
+}
+
+
+def _run_doctor(st: State) -> None:
+    cfg = st.load() if st.cfg_path.exists() else c.Config()
+    ctx = doctor.Context(
+        st.get_runner(),
+        cfg,
+        st.cfg_path,
+        st.ssh_path,
+        st.servers(cfg),
+        st.units_path,
+        st.user_units,
+    )
+    results = doctor.run(ctx)
+    for r in results:
+        click.echo(f"[{_ICON[r.status]}] {r.name}" + (f"  - {r.detail}" if r.detail else ""))
+    failed = sum(r.status == doctor.FAIL for r in results)
+    click.echo(f"\n{len(results)} checks, {failed} failed")
+    if failed:
+        raise click.exceptions.Exit(EXIT_DOCTOR_FAILED)
+
+
+# --- discover ---------------------------------------------------------------------------
+
+
+@cli.command("discover")
+@common
+@click.argument("host")
+@click.option("--ssh-user", help="SSH user on the server (skipped for a ~/.ssh/config alias).")
+@click.option("--ssh-port", type=int, help="SSH port [22].")
+@click.option("--identity", help="Local SSH private key for the server.")
+@click.option("--storage-host", help="SFTP storage host, as reached from the server.")
+@click.option("--storage-user", help="SSH user on the storage host.")
+@click.option("--storage-port", type=int, help="SSH port of the storage host [22].")
+@click.option("--storage-path", help="Base path on the storage host [/srv/restic].")
+@click.option("--storage-identity", help="Private key path ON THE SERVER for the storage host.")
+@click.option("--db-user", help="MariaDB admin user (password from $LTSBU_DB_PASSWORD or prompt).")
+@click.option("--no-db-login", is_flag=True, help="Do not set up a MariaDB login now.")
+@click.option("--vhosts", help="'all', 'none' or comma-separated repo names.")
+@click.option("--databases", help="'all', 'none' or comma-separated database names.")
+@click.option("--folders", help="Extra folders on the server, space separated.")
+@click.option("--every-hours", type=int, help="Backup interval in hours [2].")
+@click.option("--weekly-day", help="Day of the weekly (pruning) run [staggered].")
+@click.option("--weekly-time", help="Time of the weekly run, HH:MM [staggered, at night].")
+@click.option("--keep-hourly", type=int)
+@click.option("--keep-daily", type=int)
+@click.option("--keep-weekly", type=int)
+@click.option("--keep-monthly", type=int)
+@click.option("--remove", is_flag=True, help="Remove repos that are no longer selected.")
+@click.option("--no-restic-init", is_flag=True, help="Do not initialise missing repositories.")
+@click.option("--no-timers", is_flag=True, help="Do not install systemd timers.")
+@click.option(
+    "--non-interactive", is_flag=True, help="Never prompt; use options, env and defaults."
+)
+@click.pass_obj
+def discover_cmd(st: State, /, host: str, non_interactive: bool, **opts: Any) -> None:
+    """Find HOST's vhosts and databases and create or edit its backups (wizard)."""
+    cfg = st.load()
+    presets = _discover_presets(non_interactive, opts)
+    asker = st.asker or (
+        wz.ScriptedAsker(presets) if non_interactive else wz.QuestionaryAsker(presets)
+    )
+    runner = st.get_runner()
+    svc = services.DiscoverService(
+        runner, cfg, st.cfg_path, st.ssh_path, st.units_path, st.user_units, click.echo
+    )
+    label = v.label(st.server) if st.server else ""
+    req = wz.run(asker, svc, cfg, host, label, click.echo)
+    svc.apply(req)
+    if not st.dry_run:
+        click.echo("run 'ltsbu doctor' to verify everything end to end")
+
+
+def _pick_all(value: str | None) -> Any:
+    if value is None:
+        return None
+    if value == "all":
+        return wz.ALL
+    return [] if value == "none" else [x.strip() for x in value.split(",") if x.strip()]
+
+
+def _discover_presets(non_interactive: bool, o: dict[str, Any]) -> dict[str, Any]:
+    p: dict[str, Any] = {}
+    for key in (
+        "ssh_user",
+        "storage_host",
+        "storage_user",
+        "storage_path",
+        "storage_identity",
+        "folders",
+    ):
+        if o[key]:
+            p[key] = o[key]
+    for key in ("ssh_port", "storage_port"):
+        if o[key] is not None:
+            p[key] = str(o[key])
+    if o["identity"]:
+        p["identity"] = os.path.expanduser(o["identity"])
+    for key in ("vhosts", "databases"):
+        if (picked := _pick_all(o[key])) is not None:
+            p[key] = picked
+    tuned = {
+        "every_hours": o["every_hours"], "weekly_day": o["weekly_day"], "weekly_time": o["weekly_time"],
+        **{f"keep_{k}": o[f"keep_{k}"] for k in ("hourly", "daily", "weekly", "monthly")},
+    }  # fmt: skip
+    if any(x is not None for x in tuned.values()):
+        p["customize"] = True
+        p.update({f"*.{k}": str(x) for k, x in tuned.items() if x is not None})
+    if o["no_db_login"]:
+        p["db_login"] = False
+    elif o["db_user"]:
+        p["db_login"], p["db_user"] = True, o["db_user"]
+        if os.environ.get("LTSBU_DB_PASSWORD"):
+            p["db_password"] = os.environ["LTSBU_DB_PASSWORD"]
+    if non_interactive:
+        p.setdefault("identity", wz.NONE)
+        p.setdefault("db_login", "db_user" in p)
+        p.setdefault("storage_identity", "-")
+        p.setdefault("folders", "-")
+        p.setdefault("customize", False)
+        p["confirm_remove"] = o["remove"]
+        p["apply"] = True
+        p["actions"] = [
+            *([] if o["no_restic_init"] else ["restic_init"]),
+            *([] if o["no_timers"] else ["timers"]),
+        ]
+    elif o["remove"]:
+        p["confirm_remove"] = True
+    return p
+
+
+# --- list -------------------------------------------------------------------------------
+
+
+@cli.command("list")
+@common
+@click.pass_obj
+def list_cmd(st: State) -> None:
+    """List configured servers and their repos."""
+    cfg = st.load()
+    if not cfg.servers:
+        click.echo(NO_SERVERS)
+        return
+    rows = [("SERVER", "REPO", "KIND", "WHAT", "SCHEDULE", "REPOSITORY")] + [
+        (
+            s.label,
+            r.name,
+            r.kind,
+            ", ".join(r.databases or r.paths),
+            f"every {r.schedule.every_hours}h, weekly {r.schedule.weekly_day} {r.schedule.weekly_time}",
+            s.repository(r),
+        )
+        for s in st.servers(cfg)
+        for r in sorted(s.repos.values(), key=lambda r: r.name)
+    ]
+    widths = [max(len(row[i]) for row in rows) for i in range(len(rows[0]))]
+    for row in rows:
+        click.echo("  ".join(cell.ljust(w) for cell, w in zip(row, widths)).rstrip())
+
+
+# --- backup -----------------------------------------------------------------------------
+
+
+@cli.command("backup")
+@common
+@click.option(
+    "--all", "all_repos", is_flag=True, help="Every repo (of --server, or of all servers)."
+)
+@click.option("--weekly", is_flag=True, help="Weekly run: tag 'weekly' and prune after forget.")
+@click.option("--include-git", is_flag=True, help="Do not exclude .git directories this run.")
+@click.pass_obj
+def backup_cmd(st: State, all_repos: bool, weekly: bool, include_git: bool) -> None:
+    """Back up repos on their servers, then rotate them with the keep policy."""
+    cfg = st.load()
+    if all_repos:
+        targets = [
+            (s, r) for s in st.servers(cfg) for r in sorted(s.repos.values(), key=lambda r: r.name)
+        ]
+        if not targets:
+            raise c.ConfigError(NO_SERVERS)
+    else:
+        targets = [st.pick(cfg)]
+    base = st.get_runner()
+    failed: list[str] = []
+    with services.backup_lock():
+        for server, repo in targets:
+            svc = services.BackupService(services.runner_for(base, server), server, click.echo)
+            where = f"{server.label}/{repo.name}"
+            try:
+                failed += [f"{where}:{db}" for db in svc.run(repo, weekly, include_git, st.verbose)]
+            except (v.ValidationError, services.ServiceError, CommandError) as e:
+                if not all_repos:
+                    raise
+                click.echo(f"Error: [{where}] {e}", err=True)
+                failed.append(where)
+    if failed:
+        raise click.ClickException(f"backup failed for: {', '.join(failed)}")
+
+
+# --- snapshot ---------------------------------------------------------------------------
+
+
+@cli.command("snapshot")
+@common
+@click.option("--json", "as_json", is_flag=True, help="JSON output.")
+@click.pass_obj
+def snapshot_cmd(st: State, as_json: bool) -> None:
+    """List a repo's snapshots."""
+    server, repo = st.pick(st.load())
+    runner = services.runner_for(st.get_runner(), server)
+    snaps = services.restic_for(runner, server, repo).snapshots(services.repo_tags(server, repo))
+    snaps.sort(key=lambda s: s.time)
+    if as_json:
+        click.echo(json.dumps([asdict(s) for s in snaps], indent=2))
+        return
+    if not snaps:
+        click.echo(f"no snapshots for {server.label}/{repo.name}")
+        return
+    click.echo(f"{'ID':<10}{'TIME':<21}{'DATABASE':<20}{'WEEKLY':<8}PATHS")
+    for s in snaps:
+        db = services.snapshot_db(s) or "-"
+        weekly = "yes" if "weekly" in s.tags else ""
+        click.echo(
+            f"{s.short_id:<10}{s.time[:19].replace('T', ' '):<21}{db:<20}{weekly:<8}{', '.join(s.paths)}"
+        )
+
+
+# --- restore ----------------------------------------------------------------------------
+
+
+@cli.command("restore")
+@common
+@click.argument("snapshot", default="latest")
+@click.option(
+    "--target", help="Files repos: restore under this directory on the server ('/' = in place)."
+)
+@click.option("--database", help="Database repos: which database to restore.")
+@click.option("--into", help="Database repos: load into this database instead [same name].")
+@click.option("--yes", "-y", is_flag=True, help="Do not ask for confirmation.")
+@click.pass_obj
+def restore_cmd(
+    st: State, snapshot: str, target: str | None, database: str | None, into: str | None, yes: bool
+) -> None:
+    """Restore files or a database from SNAPSHOT (default: latest), on the server."""
+    server, repo = st.pick(st.load())
+    svc = services.RestoreService(services.runner_for(st.get_runner(), server), server, click.echo)
+    plan = svc.plan(repo, snapshot, target, database, into)
+    click.echo(f"snapshot {plan.snapshot.short_id} ({plan.snapshot.time[:19]})")
+    if plan.target == "/":
+        _confirm(st, yes, f"Overwrite {', '.join(plan.snapshot.paths)} IN PLACE on {server.label}?")
+    if plan.into:
+        _confirm(
+            st,
+            yes,
+            f"Load {plan.database} into database '{plan.into}' on {server.label} (overwrites tables)?",
+        )
+    svc.execute(repo, plan)
+    click.echo("restore complete")
+
+
+# --- forget -----------------------------------------------------------------------------
+
+
+@cli.command("forget")
+@common
+@click.option("--keep-hourly", type=int, help="Hourly snapshots to keep.")
+@click.option("--keep-daily", type=int, help="Daily snapshots to keep.")
+@click.option("--keep-weekly", type=int, help="Weekly snapshots to keep.")
+@click.option("--keep-monthly", type=int, help="Monthly snapshots to keep.")
+@click.option("--prune", is_flag=True, help="Also remove unreferenced data.")
+@click.option("--yes", "-y", is_flag=True, help="Do not ask for confirmation.")
+@click.pass_obj
+def forget_cmd(
+    st: State,
+    keep_hourly: int | None,
+    keep_daily: int | None,
+    keep_weekly: int | None,
+    keep_monthly: int | None,
+    prune: bool,
+    yes: bool,
+) -> None:
+    """Remove snapshots outside the retention policy (backups already do this)."""
+    server, repo = st.pick(st.load())
+    _confirm(
+        st, yes, f"Forget snapshots of '{server.label}/{repo.name}' outside the retention policy?"
+    )
+    out = services.ForgetService(services.runner_for(st.get_runner(), server), server).run(
+        repo, keep_hourly, keep_daily, keep_weekly, keep_monthly, prune
+    )
+    if out:
+        click.echo(out.rstrip())
+
+
+# --- credentials ----------------------------------------------------------------------
+
+
+@cli.group("credentials")
+def credentials() -> None:
+    """Move servers, their secrets and site database users between admin boxes."""
+
+
+@credentials.command("export")
+@common
+@click.argument("file", required=False, type=click.Path(dir_okay=False, path_type=Path))
+@click.option("--no-db-users", is_flag=True, help="Do not include the sites' database users.")
+@click.pass_obj
+def credentials_export(st: State, file: Path | None, no_db_users: bool) -> None:
+    """Write config, secrets and site DB users to FILE (default: stdout)."""
+    cfg = st.load()
+    labels = [s.label for s in st.servers(cfg)]
+    if not labels:
+        raise c.ConfigError(NO_SERVERS)
+    text, warnings = services.credentials_bundle(
+        st.get_runner(), cfg, labels, db_users=not no_db_users
+    )
+    for w in warnings:
+        click.echo(f"warning: {w}", err=True)
+    if file is None or str(file) == "-":
+        click.echo(text, nl=False)
+    else:
+        c.write_secret(file, text)
+        click.echo(f"exported {', '.join(labels)} to {file} (mode 0600)", err=True)
+    click.echo(
+        "warning: this bundle can decrypt and delete your backups; "
+        "pipe it over ssh or shred every copy",
+        err=True,
+    )
+
+
+@credentials.command("import")
+@common
+@click.argument("file", type=click.File("r"))
+@click.option(
+    "--force", is_flag=True, help="Replace servers that already exist (keeps their connection)."
+)
+@click.option("--no-ssh-config", is_flag=True, help="Do not edit ~/.ssh/config.")
+@click.option("--no-db-users", is_flag=True, help="Do not create the sites' database users.")
+@click.pass_obj
+def credentials_import(
+    st: State, file: Any, force: bool, no_ssh_config: bool, no_db_users: bool
+) -> None:
+    """Import a bundle from 'ltsbu credentials export' (FILE or - for stdin)."""
+    cfg = st.load()
+    svc = services.CredentialsImportService(
+        st.get_runner(), cfg, st.cfg_path, st.ssh_path, click.echo
+    )
+    warnings = svc.apply(
+        file.read(), file.name, force=force, write_ssh=not no_ssh_config, db_users=not no_db_users
+    )
+    for w in warnings:
+        click.echo(f"warning: {w}", err=True)
+    if not st.dry_run:
+        click.echo(
+            "run 'ltsbu discover <host>' to refresh server-side secrets and timers, then 'ltsbu doctor'"
+        )
+
+
+def main() -> None:
+    cli(prog_name="ltsbu")
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()