lts-backup-cli 3.2.0__tar.gz → 3.2.1__tar.gz

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lts-backup-cli
-Version: 3.2.0
+Version: 3.2.1
 Summary: LTS backup utility: a central backup server pulls Apache vhosts, MariaDB databases, folders and Docker hosts over SSH into local restic repositories, with discovery and scheduled rotation.
 Author-email: LTS <joaomartins@lts.pt>
 License: BSD-3-Clause
@@ -233,6 +233,7 @@ that ltsbu can't log in to.
 | containers | each container to dump is running and has `mariadb-dump`/`mysqldump` inside |
 | restic repository | every local repo opens with its password |
 | timers | both timers of every repo are installed and enabled; user timers warn if linger is off |
+| last run | the last interval and weekly run of every repo succeeded (else: which `journalctl -u` to read) |
 
 ### `ltsbu list`
 
@@ -349,6 +350,8 @@ Exit codes: `0` success, `1` error, `2` invalid usage, `3` doctor found failures
   `files` on a docker repo's file snapshot, and `weekly` on weekly runs.
 - **One at a time:** a lock (`~/.cache/ltsbu/backup.lock`) makes overlapping timer runs queue
   instead of loading the servers in parallel.
+- **Stale locks:** each backup first runs `restic unlock`, which only removes locks of restic
+  processes that died (a lock left behind would otherwise make the rotation fail).
 - Every argument sent over SSH is shell-quoted. The only secret ever written to a server (the MariaDB
   login) travels **on stdin** and is written `0600` with `umask 077`.
 

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/README.md

@@ -198,6 +198,7 @@ that ltsbu can't log in to.
 | containers | each container to dump is running and has `mariadb-dump`/`mysqldump` inside |
 | restic repository | every local repo opens with its password |
 | timers | both timers of every repo are installed and enabled; user timers warn if linger is off |
+| last run | the last interval and weekly run of every repo succeeded (else: which `journalctl -u` to read) |
 
 ### `ltsbu list`
 
@@ -314,6 +315,8 @@ Exit codes: `0` success, `1` error, `2` invalid usage, `3` doctor found failures
   `files` on a docker repo's file snapshot, and `weekly` on weekly runs.
 - **One at a time:** a lock (`~/.cache/ltsbu/backup.lock`) makes overlapping timer runs queue
   instead of loading the servers in parallel.
+- **Stale locks:** each backup first runs `restic unlock`, which only removes locks of restic
+  processes that died (a lock left behind would otherwise make the rotation fail).
 - Every argument sent over SSH is shell-quoted. The only secret ever written to a server (the MariaDB
   login) travels **on stdin** and is written `0600` with `umask 077`.
 

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/__init__.py

@@ -1,3 +1,3 @@
 """lts-backup-cli: restic + MariaDB backup utility."""
 
-__version__ = "3.2.0"
+__version__ = "3.2.1"

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/doctor.py

@@ -257,6 +257,19 @@ def check_timers(ctx: Context) -> Iterable[CheckResult]:
                 OK if enabled else FAIL,
                 res.stdout.strip().replace("\n", ", "),
             )
+            services_ = [f"{base}.service", f"{base}-weekly.service"]
+            res = ctx.runner.run(
+                ["systemctl", *(["--user"] if ctx.user_units else []), "show", "-p", "Result",
+                 "--value", *services_],
+                check=False,
+                mutating=False,
+            )  # fmt: skip
+            bad = [u for u, r in zip(services_, res.stdout.split("\n")) if r and r != "success"]
+            yield CheckResult(
+                f"[{s.label}/{repo.name}] last run",
+                FAIL if bad else OK,
+                "; ".join(f"failed, see journalctl -u {u}" for u in bad),
+            )
     if ctx.user_units and any(s.repos for s in ctx.servers):
         res = ctx.runner.run(
             ["loginctl", "show-user", getpass.getuser(), "-p", "Linger"],

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/restic.py

@@ -49,6 +49,10 @@ class ResticClient:
         res = self.runner.run(self._base("cat", "config"), check=False, mutating=False)
         return res.returncode == 0
 
+    def unlock(self) -> None:
+        """Remove stale locks only (dead process on this host, or older than 30 minutes)."""
+        self.runner.run(self._base("unlock"))
+
     def init(self) -> None:
         self.runner.run(self._base("init"))
 

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/schedule.py

@@ -65,6 +65,7 @@ def render_units(
             f"Description=ltsbu backup {server}/{repo} ({what})\n"
             "Wants=network-online.target\nAfter=network-online.target\n\n"
             "[Service]\nType=oneshot\nNice=10\nIOSchedulingClass=idle\n"
+            "Environment=HOME=%h\n"  # system units have no $HOME: restic's cache needs one
             f"ExecStart={cmd}{extra}\n"
         )
         units[f"{name}.timer"] = (

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/src/lts_backup_cli/services.py

@@ -126,8 +126,13 @@ class BackupService:
         restic = restic_for(self.runner, self.server, repo)
         tags = repo_tags(self.server, repo) + (["weekly"] if weekly else [])
         failed: list[str] = []
-        if repo.kind == "db":
-            db = mariadb_for(runner_for(self.runner, self.server), self.server)
+        db = (
+            mariadb_for(runner_for(self.runner, self.server), self.server)
+            if repo.kind == "db"
+            else None
+        )
+        restic.unlock()  # a killed restic run leaves a lock that makes forget fail
+        if db:
             for name in repo.databases:
                 if not self._dump(restic, repo, name, db.dump_argv(name), [*tags, f"db:{name}"]):
                     failed.append(name)

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/tests/unit/test_doctor.py

@@ -244,3 +244,29 @@ def test_docker_images_to_save_must_exist(
     runner.on("image", "inspect", "lts/gone:1", returncode=1, stderr="No such image")
     res = {r.name: r.status for r in d.check_server_docker(ctx(runner, cfg, tmp_path))}
     assert res["[web1] image lts/a:1"] == d.OK and res["[web1] image lts/gone:1"] == d.FAIL
+
+
+def test_failed_last_run_is_reported_with_where_to_look(
+    runner: FakeRunner, cfg: c.Config, tmp_path: Path
+) -> None:
+    cx = ctx(healthy(runner), cfg, tmp_path)
+    schedule.install(
+        FakeRunner(), schedule.render_units("web1", "shop", c.Schedule(), "/py", "/c"),
+        cx.units_dir, True,
+    )  # fmt: skip
+    runner.on("Result", "ltsbu-web1-shop.service", stdout="exit-code\nsuccess\n")
+    res = {r.name: r for r in d.check_timers(cx)}
+    last = res["[web1/shop] last run"]
+    assert last.status == d.FAIL and "journalctl -u ltsbu-web1-shop.service" in last.detail
+
+
+def test_last_run_ok_when_systemd_reports_success_or_nothing(
+    runner: FakeRunner, cfg: c.Config, tmp_path: Path
+) -> None:
+    cx = ctx(healthy(runner), cfg, tmp_path)
+    schedule.install(
+        FakeRunner(), schedule.render_units("web1", "shop", c.Schedule(), "/py", "/c"),
+        cx.units_dir, True,
+    )  # fmt: skip
+    runner.on("Result", "ltsbu-web1-shop.service", stdout="success\nsuccess\n")
+    assert {r.name: r.status for r in d.check_timers(cx)}["[web1/shop] last run"] == d.OK

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/tests/unit/test_schedule.py

@@ -61,6 +61,8 @@ def test_render_units() -> None:
         "--server web1 --repo shop\n" in svc
     )
     assert "--weekly" in units["ltsbu-web1-shop-weekly.service"]
+    # system units get no $HOME: restic would run without its cache
+    assert "Environment=HOME=%h\n" in svc
     timer = units["ltsbu-web1-shop.timer"]
     assert "OnCalendar=*-*-* 00/2:07:00" in timer and "Persistent=true" in timer
     assert "RandomizedDelaySec=" in timer and "WantedBy=timers.target" in timer

{lts_backup_cli-3.2.0 → lts_backup_cli-3.2.1}/tests/unit/test_services.py

@@ -59,7 +59,8 @@ def test_vhost_backup_pulls_then_backs_up_the_local_mirror_then_rotates(
     runner: FakeRunner, cfg: c.Config
 ) -> None:
     s.BackupService(runner, web1(cfg), quiet).run(web1(cfg).repos["shop"])
-    mkdir, rsync, backup, forget = runner.calls
+    unlock, mkdir, rsync, backup, forget = runner.calls
+    assert unlock.argv[-1] == "unlock"
     mirror = f"{web1(cfg).mirror_root}/web1/shop"
     assert mkdir.argv == ["mkdir", "-p", "-m", "700", mirror]
     assert rsync.argv[0] == "rsync" and rsync.argv[-2:] == [
@@ -115,6 +116,14 @@ def test_vanished_files_are_reported_not_fatal(runner: FakeRunner, cfg: c.Config
     assert any("vanished" in line for line in lines) and runner.find("backup")
 
 
+def test_backup_first_clears_stale_restic_locks(runner: FakeRunner, cfg: c.Config) -> None:
+    # a killed restic (e.g. a manual `restic dump | head`) leaves a lock that blocks forget
+    s.BackupService(runner, web1(cfg), quiet).run(web1(cfg).repos["shop"])
+    unlock = next(i for i, call in enumerate(runner.calls) if call.has("unlock"))
+    assert runner.calls[unlock].mutating and "--remove-all" not in runner.calls[unlock].argv
+    assert unlock < next(i for i, call in enumerate(runner.calls) if call.has("rsync"))
+
+
 def test_backup_prunes_mirror_entries_of_dropped_paths_before_the_snapshot(
     runner: FakeRunner, cfg: c.Config
 ) -> None: