lts-backup-cli 3.1.0__tar.gz → 3.2.1__tar.gz

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lts-backup-cli
-Version: 3.1.0
+Version: 3.2.1
 Summary: LTS backup utility: a central backup server pulls Apache vhosts, MariaDB databases, folders and Docker hosts over SSH into local restic repositories, with discovery and scheduled rotation.
 Author-email: LTS <joaomartins@lts.pt>
 License: BSD-3-Clause
@@ -209,7 +209,7 @@ ltsbu discover lab.example.com --non-interactive \
 | `--repo-root`, `--mirror-root` | `/srv/restic`, `/srv/ltsbu/mirror` (on this backup server) |
 | `--db-user` / `--no-db-login` | password from `$LTSBU_DB_PASSWORD` / skip databases |
 | `--vhosts`, `--databases` | `all`, `none` or a comma-separated list; new server: all; edit: unchanged |
-| `--folders "/a /b"` | none |
+| `--folders "/a /b"` | `~/public_html` of the SSH user on a new server, if it exists and no vhost already covers it; otherwise none (`-` = none) |
 | `--docker all\|none`, `--docker-containers` | non-interactive: `none`; `all` = one repo per compose project plus `docker`; containers: `all`, `none` or a list (default: every running container with a `/var/lib/mysql` mount) |
 | `--every-hours N`, `--weekly-day DAY`, `--weekly-time HH:MM` | `2`, staggered night slot |
 | `--keep-hourly/-daily/-weekly/-monthly N` | `12 / 7 / 4 / 6` |
@@ -233,6 +233,7 @@ that ltsbu can't log in to.
 | containers | each container to dump is running and has `mariadb-dump`/`mysqldump` inside |
 | restic repository | every local repo opens with its password |
 | timers | both timers of every repo are installed and enabled; user timers warn if linger is off |
+| last run | the last interval and weekly run of every repo succeeded (else: which `journalctl -u` to read) |
 
 ### `ltsbu list`
 
@@ -315,7 +316,7 @@ A server that already exists here needs `--force`, and then **keeps its local co
 | Option | Meaning |
 |---|---|
 | `--server LABEL`, `-s` | server to act on; optional when only one matches |
-| `--repo NAME`, `-r` | repo on that server; optional when only one matches |
+| `--repo NAME`, `-r`, `--label` | repo on that server; optional when only one matches |
 | `--dry-run` | print the commands that would change something instead of running them; read-only commands still run (a new server's `discover` logs in through a temporary copy of `~/.ssh/config` with the block it would add) |
 | `--verbose`, `-v` | log every external command (secrets are never on a command line) |
 | `--config FILE` | alternative config file (default `~/.config/ltsbu/config.toml`); put it before the command |
@@ -349,6 +350,8 @@ Exit codes: `0` success, `1` error, `2` invalid usage, `3` doctor found failures
   `files` on a docker repo's file snapshot, and `weekly` on weekly runs.
 - **One at a time:** a lock (`~/.cache/ltsbu/backup.lock`) makes overlapping timer runs queue
   instead of loading the servers in parallel.
+- **Stale locks:** each backup first runs `restic unlock`, which only removes locks of restic
+  processes that died (a lock left behind would otherwise make the rotation fail).
 - Every argument sent over SSH is shell-quoted. The only secret ever written to a server (the MariaDB
   login) travels **on stdin** and is written `0600` with `umask 077`.
 

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/README.md

@@ -174,7 +174,7 @@ ltsbu discover lab.example.com --non-interactive \
 | `--repo-root`, `--mirror-root` | `/srv/restic`, `/srv/ltsbu/mirror` (on this backup server) |
 | `--db-user` / `--no-db-login` | password from `$LTSBU_DB_PASSWORD` / skip databases |
 | `--vhosts`, `--databases` | `all`, `none` or a comma-separated list; new server: all; edit: unchanged |
-| `--folders "/a /b"` | none |
+| `--folders "/a /b"` | `~/public_html` of the SSH user on a new server, if it exists and no vhost already covers it; otherwise none (`-` = none) |
 | `--docker all\|none`, `--docker-containers` | non-interactive: `none`; `all` = one repo per compose project plus `docker`; containers: `all`, `none` or a list (default: every running container with a `/var/lib/mysql` mount) |
 | `--every-hours N`, `--weekly-day DAY`, `--weekly-time HH:MM` | `2`, staggered night slot |
 | `--keep-hourly/-daily/-weekly/-monthly N` | `12 / 7 / 4 / 6` |
@@ -198,6 +198,7 @@ that ltsbu can't log in to.
 | containers | each container to dump is running and has `mariadb-dump`/`mysqldump` inside |
 | restic repository | every local repo opens with its password |
 | timers | both timers of every repo are installed and enabled; user timers warn if linger is off |
+| last run | the last interval and weekly run of every repo succeeded (else: which `journalctl -u` to read) |
 
 ### `ltsbu list`
 
@@ -280,7 +281,7 @@ A server that already exists here needs `--force`, and then **keeps its local co
 | Option | Meaning |
 |---|---|
 | `--server LABEL`, `-s` | server to act on; optional when only one matches |
-| `--repo NAME`, `-r` | repo on that server; optional when only one matches |
+| `--repo NAME`, `-r`, `--label` | repo on that server; optional when only one matches |
 | `--dry-run` | print the commands that would change something instead of running them; read-only commands still run (a new server's `discover` logs in through a temporary copy of `~/.ssh/config` with the block it would add) |
 | `--verbose`, `-v` | log every external command (secrets are never on a command line) |
 | `--config FILE` | alternative config file (default `~/.config/ltsbu/config.toml`); put it before the command |
@@ -314,6 +315,8 @@ Exit codes: `0` success, `1` error, `2` invalid usage, `3` doctor found failures
   `files` on a docker repo's file snapshot, and `weekly` on weekly runs.
 - **One at a time:** a lock (`~/.cache/ltsbu/backup.lock`) makes overlapping timer runs queue
   instead of loading the servers in parallel.
+- **Stale locks:** each backup first runs `restic unlock`, which only removes locks of restic
+  processes that died (a lock left behind would otherwise make the rotation fail).
 - Every argument sent over SSH is shell-quoted. The only secret ever written to a server (the MariaDB
   login) travels **on stdin** and is written `0600` with `umask 077`.
 

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/features/backup.feature

@@ -100,3 +100,16 @@ Feature: Back up a server's repos and rotate them
     When I run "ltsbu backup --server lab --repo docker --weekly"
     Then the exit code should be 0
     And the image "lts/electro-site:php7.2" should have been saved into restic
+
+  Scenario: --label is an alias of --repo
+    Given a configured server "lab" with repos
+      | repo   | kind   | what                |
+      | docker | docker | /srv/docker/volumes |
+      | proxy  | docker | /home/lts/proxy     |
+    When I run "ltsbu backup --server lab --label proxy"
+    Then the exit code should be 0
+    And rsync should have been called with "ltsbu-lab:/home/lts/proxy"
+    And rsync should not have been called with "ltsbu-lab:/srv/docker/volumes"
+    When I run "ltsbu --label docker backup --server lab"
+    Then the exit code should be 0
+    And rsync should have been called with "ltsbu-lab:/srv/docker/volumes"

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/features/discover.feature

@@ -117,3 +117,10 @@ Feature: Discover a server and configure its backups
     When I run "ltsbu discover lab.example.com --non-interactive --ssh-user root --no-db-login --docker none"
     Then the exit code should be 0
     And server "lab" should have repos "blog-example-com, shop-example-com"
+
+  Scenario: A new server's public_html is proposed as an extra folder
+    Given the server's home is "/home/backup"
+    And the server has the paths "/home/backup/public_html"
+    When I run "ltsbu discover web1.example.com --non-interactive --ssh-user backup --no-db-login --vhosts none"
+    Then the exit code should be 0
+    And server "web1" should have repos "home-backup-public_html"

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/__init__.py

@@ -1,3 +1,3 @@
 """lts-backup-cli: restic + MariaDB backup utility."""
 
-__version__ = "3.1.0"
+__version__ = "3.2.1"

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/cli.py

@@ -123,7 +123,7 @@ def common(f: F) -> F:
     """--server, --repo, --dry-run, --verbose also accepted after the subcommand name."""
 
     @click.option("--server", "-s", metavar="LABEL", help="Server label.")
-    @click.option("--repo", "-r", metavar="NAME", help="Repo name on the server.")
+    @click.option("--repo", "-r", "--label", metavar="NAME", help="Repo name on the server.")
     @click.option(
         "--dry-run", is_flag=True, help="Print mutating commands instead of running them."
     )
@@ -158,7 +158,7 @@ def _confirm(st: State, yes: bool, message: str) -> None:
     help="Config file (default ~/.config/ltsbu/config.toml).",
 )
 @click.option("--server", "-s", metavar="LABEL", help="Server label.")
-@click.option("--repo", "-r", metavar="NAME", help="Repo name on the server.")
+@click.option("--repo", "-r", "--label", metavar="NAME", help="Repo name on the server.")
 @click.option("--dry-run", is_flag=True, help="Print mutating commands instead of running them.")
 @click.option("--verbose", "-v", is_flag=True, help="Show executed commands.")
 @click.version_option(__version__, prog_name="ltsbu")
@@ -231,7 +231,10 @@ def _run_doctor(st: State) -> None:
 @click.option("--no-db-login", is_flag=True, help="Do not set up a MariaDB login now.")
 @click.option("--vhosts", help="'all', 'none' or comma-separated repo names.")
 @click.option("--databases", help="'all', 'none' or comma-separated database names.")
-@click.option("--folders", help="Extra folders on the server, space separated.")
+@click.option(
+    "--folders",
+    help="Extra folders on the server, space separated; '-' for none [new server: ~/public_html if it exists].",
+)
 @click.option(
     "--docker", type=click.Choice(["all", "none"]), help="Back up the server's Docker host."
 )
@@ -311,7 +314,6 @@ def _discover_presets(non_interactive: bool, o: dict[str, Any]) -> dict[str, Any
     if non_interactive:
         p.setdefault("db_login", "db_user" in p)
         p.setdefault("docker", False)
-        p.setdefault("folders", "-")
         p.setdefault("customize", False)
         p["confirm_remove"] = o["remove"]
         p["apply"] = True

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/doctor.py

@@ -257,6 +257,19 @@ def check_timers(ctx: Context) -> Iterable[CheckResult]:
                 OK if enabled else FAIL,
                 res.stdout.strip().replace("\n", ", "),
             )
+            services_ = [f"{base}.service", f"{base}-weekly.service"]
+            res = ctx.runner.run(
+                ["systemctl", *(["--user"] if ctx.user_units else []), "show", "-p", "Result",
+                 "--value", *services_],
+                check=False,
+                mutating=False,
+            )  # fmt: skip
+            bad = [u for u, r in zip(services_, res.stdout.split("\n")) if r and r != "success"]
+            yield CheckResult(
+                f"[{s.label}/{repo.name}] last run",
+                FAIL if bad else OK,
+                "; ".join(f"failed, see journalctl -u {u}" for u in bad),
+            )
     if ctx.user_units and any(s.repos for s in ctx.servers):
         res = ctx.runner.run(
             ["loginctl", "show-user", getpass.getuser(), "-p", "Linger"],

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/restic.py

@@ -49,6 +49,10 @@ class ResticClient:
         res = self.runner.run(self._base("cat", "config"), check=False, mutating=False)
         return res.returncode == 0
 
+    def unlock(self) -> None:
+        """Remove stale locks only (dead process on this host, or older than 30 minutes)."""
+        self.runner.run(self._base("unlock"))
+
     def init(self) -> None:
         self.runner.run(self._base("init"))
 

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/schedule.py

@@ -65,6 +65,7 @@ def render_units(
             f"Description=ltsbu backup {server}/{repo} ({what})\n"
             "Wants=network-online.target\nAfter=network-online.target\n\n"
             "[Service]\nType=oneshot\nNice=10\nIOSchedulingClass=idle\n"
+            "Environment=HOME=%h\n"  # system units have no $HOME: restic's cache needs one
             f"ExecStart={cmd}{extra}\n"
         )
         units[f"{name}.timer"] = (

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/services.py

@@ -126,8 +126,13 @@ class BackupService:
         restic = restic_for(self.runner, self.server, repo)
         tags = repo_tags(self.server, repo) + (["weekly"] if weekly else [])
         failed: list[str] = []
-        if repo.kind == "db":
-            db = mariadb_for(runner_for(self.runner, self.server), self.server)
+        db = (
+            mariadb_for(runner_for(self.runner, self.server), self.server)
+            if repo.kind == "db"
+            else None
+        )
+        restic.unlock()  # a killed restic run leaves a lock that makes forget fail
+        if db:
             for name in repo.databases:
                 if not self._dump(restic, repo, name, db.dump_argv(name), [*tags, f"db:{name}"]):
                     failed.append(name)
@@ -444,11 +449,14 @@ class DiscoverService:
         """docker.propose, keeping only paths that exist on the server (Portainer's compose
         dirs are container paths, already inside its data volume)."""
         split = docker.propose(found.containers, str(found.docker_root))
-        wanted = sorted({p for paths, _, _ in split.values() for p in paths})
-        out = runner.run(["sh", "-c", _EXISTING, "sh", *wanted], mutating=False).stdout
-        present = set(out.splitlines())
+        present = self.existing(runner, sorted({p for ps, _, _ in split.values() for p in ps}))
         return {n: ([p for p in ps if p in present], ex, dbs) for n, (ps, ex, dbs) in split.items()}
 
+    def existing(self, runner: Runner, paths: list[str]) -> set[str]:
+        """The `paths` that exist on the server (one constant script, paths as argv)."""
+        out = runner.run(["sh", "-c", _EXISTING, "sh", *paths], mutating=False).stdout
+        return set(out.splitlines())
+
     def databases(
         self, runner: Runner, server: c.ServerConfig
     ) -> tuple[list[str] | None, list[str]]:

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/wizard.py

@@ -251,6 +251,10 @@ def _db_login(
     return None
 
 
+def _overlaps(a: str, b: str) -> bool:
+    return a == b or a.startswith(b.rstrip("/") + "/") or b.startswith(a.rstrip("/") + "/")
+
+
 def _folder_name(path: str) -> str:
     return apache.repo_name_for(path.strip("/").replace("/", "-") or "root")
 
@@ -403,10 +407,14 @@ def run(
 
     # --- extra folders (one repo each) ---
     folder_paths = [p for r in existing.values() if r.kind == "folder" for p in r.paths]
+    default = " ".join(folder_paths) or "-"
+    if not old and found.home.startswith("/"):
+        pub = f"{found.home.rstrip('/')}/public_html"  # v1's default files path
+        covered = any(_overlaps(pub, h.document_root) for h in found.vhosts)
+        if not covered and pub in svc.existing(runner, [pub]):
+            default = pub
     raw = ask.text(
-        "folders",
-        "Extra folders on the server (space separated, empty for none):",
-        " ".join(folder_paths) or "-",
+        "folders", "Extra folders on the server (space separated, '-' for none):", default
     )
     for path in (p for p in raw.split() if p != "-"):
         path = v.remote_abs_path(path)

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/tests/smoke/smoke.sh

@@ -20,6 +20,8 @@ done
 if ltsbu configure --help >/dev/null 2>&1; then fail "v1 'configure' command still present"; fi
 ltsbu restore --help | grep -q -- "--container" || fail "restore --container not wired"
 ltsbu restore --help | grep -q -- "--image" || fail "restore --image not wired"
+ltsbu backup --help | grep -q -- "--label" || fail "backup --label not wired"
+ltsbu --help | grep -q -- "--label" || fail "global --label not wired"
 ltsbu discover --help | grep -q -- "--docker" || fail "discover --docker not wired"
 if ltsbu discover --help | grep -q -- "--storage-host"; then fail "v2 --storage-host still present"; fi
 

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/tests/unit/test_doctor.py

@@ -244,3 +244,29 @@ def test_docker_images_to_save_must_exist(
     runner.on("image", "inspect", "lts/gone:1", returncode=1, stderr="No such image")
     res = {r.name: r.status for r in d.check_server_docker(ctx(runner, cfg, tmp_path))}
     assert res["[web1] image lts/a:1"] == d.OK and res["[web1] image lts/gone:1"] == d.FAIL
+
+
+def test_failed_last_run_is_reported_with_where_to_look(
+    runner: FakeRunner, cfg: c.Config, tmp_path: Path
+) -> None:
+    cx = ctx(healthy(runner), cfg, tmp_path)
+    schedule.install(
+        FakeRunner(), schedule.render_units("web1", "shop", c.Schedule(), "/py", "/c"),
+        cx.units_dir, True,
+    )  # fmt: skip
+    runner.on("Result", "ltsbu-web1-shop.service", stdout="exit-code\nsuccess\n")
+    res = {r.name: r for r in d.check_timers(cx)}
+    last = res["[web1/shop] last run"]
+    assert last.status == d.FAIL and "journalctl -u ltsbu-web1-shop.service" in last.detail
+
+
+def test_last_run_ok_when_systemd_reports_success_or_nothing(
+    runner: FakeRunner, cfg: c.Config, tmp_path: Path
+) -> None:
+    cx = ctx(healthy(runner), cfg, tmp_path)
+    schedule.install(
+        FakeRunner(), schedule.render_units("web1", "shop", c.Schedule(), "/py", "/c"),
+        cx.units_dir, True,
+    )  # fmt: skip
+    runner.on("Result", "ltsbu-web1-shop.service", stdout="success\nsuccess\n")
+    assert {r.name: r.status for r in d.check_timers(cx)}["[web1/shop] last run"] == d.OK

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/tests/unit/test_schedule.py

@@ -61,6 +61,8 @@ def test_render_units() -> None:
         "--server web1 --repo shop\n" in svc
     )
     assert "--weekly" in units["ltsbu-web1-shop-weekly.service"]
+    # system units get no $HOME: restic would run without its cache
+    assert "Environment=HOME=%h\n" in svc
     timer = units["ltsbu-web1-shop.timer"]
     assert "OnCalendar=*-*-* 00/2:07:00" in timer and "Persistent=true" in timer
     assert "RandomizedDelaySec=" in timer and "WantedBy=timers.target" in timer

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/tests/unit/test_services.py

@@ -59,7 +59,8 @@ def test_vhost_backup_pulls_then_backs_up_the_local_mirror_then_rotates(
     runner: FakeRunner, cfg: c.Config
 ) -> None:
     s.BackupService(runner, web1(cfg), quiet).run(web1(cfg).repos["shop"])
-    mkdir, rsync, backup, forget = runner.calls
+    unlock, mkdir, rsync, backup, forget = runner.calls
+    assert unlock.argv[-1] == "unlock"
     mirror = f"{web1(cfg).mirror_root}/web1/shop"
     assert mkdir.argv == ["mkdir", "-p", "-m", "700", mirror]
     assert rsync.argv[0] == "rsync" and rsync.argv[-2:] == [
@@ -115,6 +116,14 @@ def test_vanished_files_are_reported_not_fatal(runner: FakeRunner, cfg: c.Config
     assert any("vanished" in line for line in lines) and runner.find("backup")
 
 
+def test_backup_first_clears_stale_restic_locks(runner: FakeRunner, cfg: c.Config) -> None:
+    # a killed restic (e.g. a manual `restic dump | head`) leaves a lock that blocks forget
+    s.BackupService(runner, web1(cfg), quiet).run(web1(cfg).repos["shop"])
+    unlock = next(i for i, call in enumerate(runner.calls) if call.has("unlock"))
+    assert runner.calls[unlock].mutating and "--remove-all" not in runner.calls[unlock].argv
+    assert unlock < next(i for i, call in enumerate(runner.calls) if call.has("rsync"))
+
+
 def test_backup_prunes_mirror_entries_of_dropped_paths_before_the_snapshot(
     runner: FakeRunner, cfg: c.Config
 ) -> None:

{lts_backup_cli-3.1.0 → lts_backup_cli-3.2.1}/tests/unit/test_wizard.py

@@ -110,6 +110,38 @@ def test_select_subset_and_folders(server_runner: FakeRunner, tmp_path: Path) ->
     assert req.server.repos["srv-data"].kind == "folder"
 
 
+@pytest.mark.parametrize(
+    ("present", "vhost_root", "expected"),
+    [
+        ("/home/backup/public_html\n", "/var/www/shop", {"home-backup-public_html"}),
+        ("", "/var/www/shop", set()),  # no public_html on the server
+        ("/home/backup/public_html\n", "/home/backup/public_html", set()),  # already a vhost
+        ("/home/backup/public_html\n", "/home/backup/public_html/shop", set()),  # vhost inside
+    ],
+)
+def test_new_server_proposes_public_html_as_the_default_folder(
+    server_runner: FakeRunner, tmp_path: Path, present: str, vhost_root: str, expected: set[str]
+) -> None:
+    server_runner.rules = [r for r in server_runner.rules if r.stdout != APACHE]
+    server_runner.on(s._READ_APACHE, stdout=APACHE.replace("/var/www/shop", vhost_root))
+    server_runner.on(s._EXISTING, "/home/backup/public_html", stdout=present)
+    req = run(server_runner, c.Config(), tmp_path, fresh(vhosts=["shop-example-com"]))
+    folders = {n for n, r in req.server.repos.items() if r.kind == "folder"}
+    assert folders == expected
+    if expected:
+        assert req.server.repos["home-backup-public_html"].paths == ["/home/backup/public_html"]
+
+
+def test_editing_never_adds_public_html_by_itself(
+    server_runner: FakeRunner, tmp_path: Path
+) -> None:
+    server_runner.on(s._EXISTING, "/home/backup/public_html", stdout="/home/backup/public_html\n")
+    cfg = c.Config()
+    cfg.servers["web1"] = run(server_runner, cfg, tmp_path, fresh(folders="-")).server
+    again = run(server_runner, cfg, tmp_path, fresh()).server
+    assert not [r for r in again.repos.values() if r.kind == "folder"]
+
+
 def test_invalid_folder_is_rejected(server_runner: FakeRunner, tmp_path: Path) -> None:
     with pytest.raises(Exception, match="invalid remote path"):
         run(server_runner, c.Config(), tmp_path, fresh(folders="relative/dir"))