backup-docker-to-local 1.1.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

{backup_docker_to_local-1.1.1.dist-info → backup_docker_to_local-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: backup-docker-to-local
-Version: 1.1.1
+Version: 1.3.0
 Summary: Backup Docker volumes to local with rsync and optional DB dumps.
 Author: Kevin Veen-Birkenbach
 License: AGPL-3.0-or-later
@@ -147,7 +147,7 @@ baudolo \
 | Flag            | Description                                 |
 | --------------- | ------------------------------------------- |
 | `--everything`  | Always stop containers and re-run rsync     |
-| `--dump-only`   | Only create SQL dumps, skip file backups    |
+| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump.    |
 | `--shutdown`    | Do not restart containers after backup      |
 | `--backups-dir` | Backup root directory (default: `/Backups`) |
 | `--repo-name`   | Backup namespace under machine hash         |

{backup_docker_to_local-1.1.1.dist-info → backup_docker_to_local-1.3.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
-backup_docker_to_local-1.1.1.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+backup_docker_to_local-1.3.0.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
 baudolo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 baudolo/backup/__init__.py,sha256=5BfF8JBXB2j6sAptcmswtbjlajNWxOho6_CjwIamO7k,30
 baudolo/backup/__main__.py,sha256=118gZ0wij9_PAtR-jlG7LizrhrxwhHlAcAPW1eFeJtU,140
-baudolo/backup/app.py,sha256=UE6i4U5nJJsLFjKsU7qYk52vVIdbNtyrOM04PpZaQA4,6197
-baudolo/backup/cli.py,sha256=4SJVYK-HwNrsHtO9crmKXE0XVcjwEu_sIwVXnh8aM8E,2238
+baudolo/backup/app.py,sha256=Msx4DLi9URo9KWiT73riRD9csAQKvqZC6R2uRuEFqhk,6551
+baudolo/backup/cli.py,sha256=CIckfVsU03ajZTI7zj3TB-AuvaOeJMbTB6rNCOKymXE,2473
 baudolo/backup/compose.py,sha256=dJiZnHlBZdeXTVqLR5zIbAtwR97YWxbULVn6VL16Hqg,1136
-baudolo/backup/db.py,sha256=vHaBrXlxFGGd-NLaunKUZtE7MxVhLj4pYAFAzxs5xKI,2777
+baudolo/backup/db.py,sha256=_7_Vy8nMql1vaaL1g0kbLtRHhGx65T560DsckZFRJ-k,4462
 baudolo/backup/docker.py,sha256=6Sj9fpf1bm-CoqoeerQaq059VyDnF1Pj2ketzIt1Nkk,1364
 baudolo/backup/shell.py,sha256=guMHWcRb0Qlrz79gqFqmJLXVQK8cJEvNkhkMe5Tpgwc,738
 baudolo/backup/volume.py,sha256=DWBp_dZUo-a0MfnbwEd8jRAAHvSyN6vGdRSye0iQJbE,1485
@@ -17,9 +17,9 @@ baudolo/restore/run.py,sha256=rai5F27D6F8RRnFMyjcEGiHBAlVXtMOJoazs6zkLSC0,2302
 baudolo/restore/db/__init__.py,sha256=C4K_YAB0F9p8QhZRZIRyV2naMb54Qf_1O2cDxwwVQAI,59
 baudolo/restore/db/mariadb.py,sha256=_JEQFuF4GhTl8lKmgk2gsHYcXHcpCKrf32tigMSeTKI,2782
 baudolo/restore/db/postgres.py,sha256=_NIlcxdpcQIUZ8SNBWfWIaH1oGIkmZy9emFMgjGznhk,1446
-baudolo/seed/__main__.py,sha256=Fxto3wfzrA80gTH5p13wH0aqy-1PtN6bwklla9aGDo4,2046
-backup_docker_to_local-1.1.1.dist-info/METADATA,sha256=JChpUJ4hPotP_Un3EeTogZfyYmQHXZAPkrBBRonmwGE,7116
-backup_docker_to_local-1.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-backup_docker_to_local-1.1.1.dist-info/entry_points.txt,sha256=92f5jPSjW__9-u5gzwmWkdiHKt4ysEFCVmwMtorQCv4,147
-backup_docker_to_local-1.1.1.dist-info/top_level.txt,sha256=y_5iNpF7EdLzqWWXIDfcTJpqijyy2hvrYgNiJXrN4r4,8
-backup_docker_to_local-1.1.1.dist-info/RECORD,,
+baudolo/seed/__main__.py,sha256=BnN0Ckdjr30r6yxORq-ctrdmkT0yfeRIuXnyhKSe8Us,3430
+backup_docker_to_local-1.3.0.dist-info/METADATA,sha256=-LXJPQPpZmk8IMnlfJcNzEWzMOjfr7S0yXkkPC82n8Q,7200
+backup_docker_to_local-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+backup_docker_to_local-1.3.0.dist-info/entry_points.txt,sha256=92f5jPSjW__9-u5gzwmWkdiHKt4ysEFCVmwMtorQCv4,147
+backup_docker_to_local-1.3.0.dist-info/top_level.txt,sha256=y_5iNpF7EdLzqWWXIDfcTJpqijyy2hvrYgNiJXrN4r4,8
+backup_docker_to_local-1.3.0.dist-info/RECORD,,

baudolo/backup/app.py

@@ -72,6 +72,7 @@ def requires_stop(containers: list[str], images_no_stop_required: list[str]) ->
             return True
     return False
 
+
 def backup_mariadb_or_postgres(
     *,
     container: str,
@@ -132,7 +133,12 @@ def main() -> int:
     versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
     version_dir = create_version_directory(versions_dir, backup_time)
 
-    databases_df = pandas.read_csv(args.databases_csv, sep=";")
+    # IMPORTANT:
+    # - keep_default_na=False prevents empty fields from turning into NaN
+    # - dtype=str keeps all columns stable for comparisons/validation
+    databases_df = pandas.read_csv(
+        args.databases_csv, sep=";", keep_default_na=False, dtype=str
+    )
 
     print("💾 Start volume backups...", flush=True)
 
@@ -140,8 +146,16 @@ def main() -> int:
         print(f"Start backup routine for volume: {volume_name}", flush=True)
         containers = containers_using_volume(volume_name)
 
+        # EARLY SKIP: if all linked containers are ignored, do not create any dirs
+        if volume_is_fully_ignored(containers, args.images_no_backup_required):
+            print(
+                f"Skipping volume '{volume_name}' entirely (all linked containers are ignored).",
+                flush=True,
+            )
+            continue
+
         vol_dir = create_volume_directory(version_dir, volume_name)
-            
+
         found_db, dumped_any = _backup_dumps_for_volume(
             containers=containers,
             vol_dir=vol_dir,
@@ -149,26 +163,19 @@ def main() -> int:
             database_containers=args.database_containers,
         )
 
-        # dump-only logic:
-        if args.dump_only:
-            if found_db and not dumped_any:
-                print(
-                    f"WARNING: dump-only requested but no DB dump was produced for DB volume '{volume_name}'. Falling back to file backup.",
-                    flush=True,
-                )
-                # continue to file backup below
-            else:
-                # keep old behavior: skip file backups
-                continue
-
-
-        # skip file backup if all linked containers are ignored
-        if volume_is_fully_ignored(containers, args.images_no_backup_required):
-            print(
-                f"Skipping file backup for volume '{volume_name}' (all linked containers are ignored).",
-                flush=True,
-            )
-            continue
+        # dump-only-sql logic:
+        if args.dump_only_sql:
+            if found_db:
+                if not dumped_any:
+                    print(
+                        f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. Falling back to file backup.",
+                        flush=True,
+                    )
+                    # fall through to file backup below
+                else:
+                    # DB volume successfully dumped -> skip file backup
+                    continue
+            # Non-DB volume -> always do file backup (fall through)
 
         if args.everything:
             # "everything": always do pre-rsync, then stop + rsync again

baudolo/backup/cli.py

@@ -68,10 +68,15 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Do not restart containers after backup",
     )
+
     p.add_argument(
-        "--dump-only",
+        "--dump-only-sql",
         action="store_true",
-        help="Only create DB dumps (skip ALL file rsync backups)",
+        help=(
+            "Create database dumps only for DB volumes. "
+            "File backups are skipped for DB volumes if a dump succeeds, "
+            "but non-DB volumes are still backed up. "
+            "If a DB dump cannot be produced, baudolo falls back to a file backup."
+        ),
     )
-
     return p.parse_args()

baudolo/backup/db.py

@@ -4,6 +4,8 @@ import os
 import pathlib
 import re
 import logging
+from typing import Optional
+
 import pandas
 
 from .shell import BackupException, execute_shell_command
@@ -12,17 +14,55 @@ log = logging.getLogger(__name__)
 
 
 def get_instance(container: str, database_containers: list[str]) -> str:
+    """
+    Derive a stable instance name from the container name.
+    """
     if container in database_containers:
         return container
     return re.split(r"(_|-)(database|db|postgres)", container)[0]
 
 
-def fallback_pg_dumpall(container: str, username: str, password: str, out_file: str) -> None:
+def _validate_database_value(value: Optional[str], *, instance: str) -> str:
+    """
+    Enforce explicit database semantics:
+
+    - "*"       => dump ALL databases (cluster dump for Postgres)
+    - "<name>"  => dump exactly this database
+    - ""        => invalid configuration (would previously result in NaN / nan.backup.sql)
+    """
+    v = (value or "").strip()
+    if v == "":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            "column 'database' must be '*' or a concrete database name (not empty)."
+        )
+    return v
+
+
+def _atomic_write_cmd(cmd: str, out_file: str) -> None:
+    """
+    Write dump output atomically:
+    - write to <file>.tmp
+    - rename to <file> only on success
+
+    This prevents empty or partial dump files from being treated as valid backups.
+    """
+    tmp = f"{out_file}.tmp"
+    execute_shell_command(f"{cmd} > {tmp}")
+    execute_shell_command(f"mv {tmp} {out_file}")
+
+
+def fallback_pg_dumpall(
+    container: str, username: str, password: str, out_file: str
+) -> None:
+    """
+    Perform a full Postgres cluster dump using pg_dumpall.
+    """
     cmd = (
         f"PGPASSWORD={password} docker exec -i {container} "
-        f"pg_dumpall -U {username} -h localhost > {out_file}"
+        f"pg_dumpall -U {username} -h localhost"
     )
-    execute_shell_command(cmd)
+    _atomic_write_cmd(cmd, out_file)
 
 
 def backup_database(
@@ -34,12 +74,15 @@ def backup_database(
     database_containers: list[str],
 ) -> bool:
     """
-    Returns True if at least one dump file was produced, else False.
+    Backup databases for a given DB container.
+
+    Returns True if at least one dump was produced.
     """
     instance_name = get_instance(container, database_containers)
-    entries = databases_df.loc[databases_df["instance"] == instance_name]
+
+    entries = databases_df[databases_df["instance"] == instance_name]
     if entries.empty:
-        log.warning("No entry found for instance '%s' (skipping DB dump)", instance_name)
+        log.debug("No database entries for instance '%s'", instance_name)
         return False
 
     out_dir = os.path.join(volume_dir, "sql")
@@ -48,43 +91,54 @@ def backup_database(
     produced = False
 
     for row in entries.itertuples(index=False):
-        db_name = row.database
-        user = row.username
-        password = row.password
+        raw_db = getattr(row, "database", "")
+        user = (getattr(row, "username", "") or "").strip()
+        password = (getattr(row, "password", "") or "").strip()
+
+        db_value = _validate_database_value(raw_db, instance=instance_name)
+
+        # Explicit: dump ALL databases
+        if db_value == "*":
+            if db_type != "postgres":
+                raise ValueError(
+                    f"databases.csv entry for instance '{instance_name}': "
+                    "'*' is currently only supported for Postgres."
+                )
 
+            cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
+            fallback_pg_dumpall(container, user, password, cluster_file)
+            produced = True
+            continue
+
+        # Concrete database dump
+        db_name = db_value
         dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
 
         if db_type == "mariadb":
             cmd = (
                 f"docker exec {container} /usr/bin/mariadb-dump "
-                f"-u {user} -p{password} {db_name} > {dump_file}"
+                f"-u {user} -p{password} {db_name}"
             )
-            execute_shell_command(cmd)
+            _atomic_write_cmd(cmd, dump_file)
             produced = True
             continue
 
         if db_type == "postgres":
-            cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
-
-            if not db_name:
-                fallback_pg_dumpall(container, user, password, cluster_file)
-                return True
-
             try:
                 cmd = (
                     f"PGPASSWORD={password} docker exec -i {container} "
-                    f"pg_dump -U {user} -d {db_name} -h localhost > {dump_file}"
+                    f"pg_dump -U {user} -d {db_name} -h localhost"
                 )
-                execute_shell_command(cmd)
+                _atomic_write_cmd(cmd, dump_file)
                 produced = True
             except BackupException as e:
-                print(f"pg_dump failed: {e}", flush=True)
-                print(
-                    f"Falling back to pg_dumpall for instance '{instance_name}'",
-                    flush=True,
+                # Explicit DB dump failed -> hard error
+                raise BackupException(
+                    f"Postgres dump failed for instance '{instance_name}', "
+                    f"database '{db_name}'. This database was explicitly configured "
+                    "and therefore must succeed.\n"
+                    f"{e}"
                 )
-                fallback_pg_dumpall(container, user, password, cluster_file)
-                produced = True
             continue
 
     return produced

baudolo/seed/__main__.py

@@ -1,67 +1,116 @@
-import pandas as pd
+#!/usr/bin/env python3
+from __future__ import annotations
+
 import argparse
 import os
+import re
+import sys
+import pandas as pd
+from typing import Optional
+from pandas.errors import EmptyDataError
 
+DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
 
-def check_and_add_entry(file_path, instance, database, username, password):
-    # Check if the file exists and is not empty
-    if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
-        # Read the existing CSV file with header
-        df = pd.read_csv(file_path, sep=";")
-    else:
-        # Create a new DataFrame with columns if file does not exist
-        df = pd.DataFrame(columns=["instance", "database", "username", "password"])
-
-    # Check if the entry exists and remove it
-    mask = (
-        (df["instance"] == instance)
-        & (
-            (df["database"] == database)
-            | (((df["database"].isna()) | (df["database"] == "")) & (database == ""))
+
+def _validate_database_value(value: Optional[str], *, instance: str) -> str:
+    v = (value or "").strip()
+    if v == "":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            "column 'database' must be '*' or a concrete database name (not empty)."
         )
-        & (df["username"] == username)
-    )
+    if v == "*":
+        return "*"
+    if v.lower() == "nan":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': database must not be 'nan'."
+        )
+    if not DB_NAME_RE.match(v):
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            f"invalid database name '{v}'. Allowed: letters, numbers, '_' and '-'."
+        )
+    return v
 
-    if not df[mask].empty:
-        print("Replacing existing entry.")
-        df = df[~mask]
-    else:
-        print("Adding new entry.")
 
-    # Create a new DataFrame for the new entry
-    new_entry = pd.DataFrame(
-        [
-            {
-                "instance": instance,
-                "database": database,
-                "username": username,
-                "password": password,
-            }
-        ]
-    )
+def _empty_df() -> pd.DataFrame:
+    return pd.DataFrame(columns=["instance", "database", "username", "password"])
 
-    # Add (or replace) the entry using concat
-    df = pd.concat([df, new_entry], ignore_index=True)
 
-    # Save the updated CSV file
+def check_and_add_entry(
+    file_path: str,
+    instance: str,
+    database: Optional[str],
+    username: str,
+    password: str,
+) -> None:
+    """
+    Add or update an entry in databases.csv.
+
+    The function enforces strict validation:
+    - database MUST be set
+    - database MUST be '*' or a valid database name
+    """
+    database = _validate_database_value(database, instance=instance)
+
+    if os.path.exists(file_path):
+        try:
+            df = pd.read_csv(
+                file_path,
+                sep=";",
+                dtype=str,
+                keep_default_na=False,
+            )
+        except EmptyDataError:
+            print(
+                f"WARNING: databases.csv exists but is empty: {file_path}. Creating header columns.",
+                file=sys.stderr,
+            )
+            df = _empty_df()
+    else:
+        df = _empty_df()
+    mask = (df["instance"] == instance) & (df["database"] == database)
+
+    if mask.any():
+        print("Updating existing entry.")
+        df.loc[mask, ["username", "password"]] = [username, password]
+    else:
+        print("Adding new entry.")
+        new_entry = pd.DataFrame(
+            [[instance, database, username, password]],
+            columns=["instance", "database", "username", "password"],
+        )
+        df = pd.concat([df, new_entry], ignore_index=True)
+
     df.to_csv(file_path, sep=";", index=False)
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser(
-        description="Check and replace (or add) a database entry in a CSV file."
+        description="Seed or update databases.csv for backup configuration."
+    )
+    parser.add_argument("file", help="Path to databases.csv")
+    parser.add_argument("instance", help="Instance name (e.g. bigbluebutton)")
+    parser.add_argument(
+        "database",
+        help="Database name or '*' to dump all databases",
     )
-    parser.add_argument("file_path", help="Path to the CSV file")
-    parser.add_argument("instance", help="Database instance")
-    parser.add_argument("database", help="Database name")
-    parser.add_argument("username", help="Username")
-    parser.add_argument("password", nargs="?", default="", help="Password (optional)")
+    parser.add_argument("username", help="Database username")
+    parser.add_argument("password", help="Database password")
 
     args = parser.parse_args()
 
-    check_and_add_entry(
-        args.file_path, args.instance, args.database, args.username, args.password
-    )
+    try:
+        check_and_add_entry(
+            file_path=args.file,
+            instance=args.instance,
+            database=args.database,
+            username=args.username,
+            password=args.password,
+        )
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)
 
 
 if __name__ == "__main__":