backup-docker-to-local 1.1.0__tar.gz → 1.2.0__tar.gz

{backup_docker_to_local-1.1.0/src/backup_docker_to_local.egg-info → backup_docker_to_local-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: backup-docker-to-local
-Version: 1.1.0
+Version: 1.2.0
 Summary: Backup Docker volumes to local with rsync and optional DB dumps.
 Author: Kevin Veen-Birkenbach
 License: AGPL-3.0-or-later
@@ -147,7 +147,7 @@ baudolo \
 | Flag            | Description                                 |
 | --------------- | ------------------------------------------- |
 | `--everything`  | Always stop containers and re-run rsync     |
-| `--dump-only`   | Only create SQL dumps, skip file backups    |
+| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump.    |
 | `--shutdown`    | Do not restart containers after backup      |
 | `--backups-dir` | Backup root directory (default: `/Backups`) |
 | `--repo-name`   | Backup namespace under machine hash         |

{backup_docker_to_local-1.1.0 → backup_docker_to_local-1.2.0}/README.md

@@ -134,7 +134,7 @@ baudolo \
 | Flag            | Description                                 |
 | --------------- | ------------------------------------------- |
 | `--everything`  | Always stop containers and re-run rsync     |
-| `--dump-only`   | Only create SQL dumps, skip file backups    |
+| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump.    |
 | `--shutdown`    | Do not restart containers after backup      |
 | `--backups-dir` | Backup root directory (default: `/Backups`) |
 | `--repo-name`   | Backup namespace under machine hash         |

{backup_docker_to_local-1.1.0 → backup_docker_to_local-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "backup-docker-to-local"
-version = "1.1.0"
+version = "1.2.0"
 description = "Backup Docker volumes to local with rsync and optional DB dumps."
 readme = "README.md"
 requires-python = ">=3.9"

{backup_docker_to_local-1.1.0 → backup_docker_to_local-1.2.0/src/backup_docker_to_local.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: backup-docker-to-local
-Version: 1.1.0
+Version: 1.2.0
 Summary: Backup Docker volumes to local with rsync and optional DB dumps.
 Author: Kevin Veen-Birkenbach
 License: AGPL-3.0-or-later
@@ -147,7 +147,7 @@ baudolo \
 | Flag            | Description                                 |
 | --------------- | ------------------------------------------- |
 | `--everything`  | Always stop containers and re-run rsync     |
-| `--dump-only`   | Only create SQL dumps, skip file backups    |
+| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump.    |
 | `--shutdown`    | Do not restart containers after backup      |
 | `--backups-dir` | Backup root directory (default: `/Backups`) |
 | `--repo-name`   | Backup namespace under machine hash         |

{backup_docker_to_local-1.1.0 → backup_docker_to_local-1.2.0}/src/baudolo/backup/app.py

@@ -72,28 +72,27 @@ def requires_stop(containers: list[str], images_no_stop_required: list[str]) ->
             return True
     return False
 
-
 def backup_mariadb_or_postgres(
     *,
     container: str,
     volume_dir: str,
     databases_df: "pandas.DataFrame",
     database_containers: list[str],
-) -> bool:
+) -> tuple[bool, bool]:
     """
-    Returns True if the container is a DB container we handled.
+    Returns (is_db_container, dumped_any)
     """
     for img in ["mariadb", "postgres"]:
         if has_image(container, img):
-            backup_database(
+            dumped = backup_database(
                 container=container,
                 volume_dir=volume_dir,
                 db_type=img,
                 databases_df=databases_df,
                 database_containers=database_containers,
             )
-            return True
-    return False
+            return True, dumped
+    return False, False
 
 
 def _backup_dumps_for_volume(
@@ -102,21 +101,26 @@ def _backup_dumps_for_volume(
     vol_dir: str,
     databases_df: "pandas.DataFrame",
     database_containers: list[str],
-) -> bool:
+) -> tuple[bool, bool]:
     """
-    Create DB dumps for any mariadb/postgres containers attached to this volume.
-    Returns True if at least one dump was produced.
+    Returns (found_db_container, dumped_any)
     """
+    found_db = False
     dumped_any = False
+
     for c in containers:
-        if backup_mariadb_or_postgres(
+        is_db, dumped = backup_mariadb_or_postgres(
             container=c,
             volume_dir=vol_dir,
             databases_df=databases_df,
             database_containers=database_containers,
-        ):
+        )
+        if is_db:
+            found_db = True
+        if dumped:
             dumped_any = True
-    return dumped_any
+
+    return found_db, dumped_any
 
 
 def main() -> int:
@@ -128,7 +132,12 @@ def main() -> int:
     versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
     version_dir = create_version_directory(versions_dir, backup_time)
 
-    databases_df = pandas.read_csv(args.databases_csv, sep=";")
+    # IMPORTANT:
+    # - keep_default_na=False prevents empty fields from turning into NaN
+    # - dtype=str keeps all columns stable for comparisons/validation
+    databases_df = pandas.read_csv(
+        args.databases_csv, sep=";", keep_default_na=False, dtype=str
+    )
 
     print("💾 Start volume backups...", flush=True)
 
@@ -136,27 +145,36 @@ def main() -> int:
         print(f"Start backup routine for volume: {volume_name}", flush=True)
         containers = containers_using_volume(volume_name)
 
+        # EARLY SKIP: if all linked containers are ignored, do not create any dirs
+        if volume_is_fully_ignored(containers, args.images_no_backup_required):
+            print(
+                f"Skipping volume '{volume_name}' entirely (all linked containers are ignored).",
+                flush=True,
+            )
+            continue
+
         vol_dir = create_volume_directory(version_dir, volume_name)
 
-        # Old behavior: DB dumps are additional to file backups.
-        _backup_dumps_for_volume(
+        found_db, dumped_any = _backup_dumps_for_volume(
             containers=containers,
             vol_dir=vol_dir,
             databases_df=databases_df,
             database_containers=args.database_containers,
         )
 
-        # dump-only: skip ALL file rsync backups
-        if args.dump_only:
-            continue
-
-        # skip file backup if all linked containers are ignored
-        if volume_is_fully_ignored(containers, args.images_no_backup_required):
-            print(
-                f"Skipping file backup for volume '{volume_name}' (all linked containers are ignored).",
-                flush=True,
-            )
-            continue
+        # dump-only-sql logic:
+        if args.dump_only_sql:
+            if found_db:
+                if not dumped_any:
+                    print(
+                        f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. Falling back to file backup.",
+                        flush=True,
+                    )
+                    # fall through to file backup below
+                else:
+                    # DB volume successfully dumped -> skip file backup
+                    continue
+            # Non-DB volume -> always do file backup (fall through)
 
         if args.everything:
             # "everything": always do pre-rsync, then stop + rsync again

{backup_docker_to_local-1.1.0 → backup_docker_to_local-1.2.0}/src/baudolo/backup/cli.py

@@ -68,10 +68,15 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Do not restart containers after backup",
     )
+    
     p.add_argument(
-        "--dump-only",
+        "--dump-only-sql",
         action="store_true",
-        help="Only create DB dumps (skip ALL file rsync backups)",
+        help=(
+            "Create database dumps only for DB volumes. "
+            "File backups are skipped for DB volumes if a dump succeeds, "
+            "but non-DB volumes are still backed up. "
+            "If a DB dump cannot be produced, baudolo falls back to a file backup."
+        ),
     )
-
     return p.parse_args()

backup_docker_to_local-1.2.0/src/baudolo/backup/db.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import re
+import logging
+from typing import Optional
+
+import pandas
+
+from .shell import BackupException, execute_shell_command
+
+log = logging.getLogger(__name__)
+
+
+def get_instance(container: str, database_containers: list[str]) -> str:
+    """
+    Derive a stable instance name from the container name.
+    """
+    if container in database_containers:
+        return container
+    return re.split(r"(_|-)(database|db|postgres)", container)[0]
+
+
+def _validate_database_value(value: Optional[str], *, instance: str) -> str:
+    """
+    Enforce explicit database semantics:
+
+    - "*"       => dump ALL databases (cluster dump for Postgres)
+    - "<name>"  => dump exactly this database
+    - ""        => invalid configuration (would previously result in NaN / nan.backup.sql)
+    """
+    v = (value or "").strip()
+    if v == "":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            "column 'database' must be '*' or a concrete database name (not empty)."
+        )
+    return v
+
+
+def _atomic_write_cmd(cmd: str, out_file: str) -> None:
+    """
+    Write dump output atomically:
+    - write to <file>.tmp
+    - rename to <file> only on success
+
+    This prevents empty or partial dump files from being treated as valid backups.
+    """
+    tmp = f"{out_file}.tmp"
+    execute_shell_command(f"{cmd} > {tmp}")
+    execute_shell_command(f"mv {tmp} {out_file}")
+
+
+def fallback_pg_dumpall(container: str, username: str, password: str, out_file: str) -> None:
+    """
+    Perform a full Postgres cluster dump using pg_dumpall.
+    """
+    cmd = (
+        f"PGPASSWORD={password} docker exec -i {container} "
+        f"pg_dumpall -U {username} -h localhost"
+    )
+    _atomic_write_cmd(cmd, out_file)
+
+
+def backup_database(
+    *,
+    container: str,
+    volume_dir: str,
+    db_type: str,
+    databases_df: "pandas.DataFrame",
+    database_containers: list[str],
+) -> bool:
+    """
+    Backup databases for a given DB container.
+
+    Returns True if at least one dump was produced.
+    """
+    instance_name = get_instance(container, database_containers)
+
+    entries = databases_df[databases_df["instance"] == instance_name]
+    if entries.empty:
+        log.debug("No database entries for instance '%s'", instance_name)
+        return False
+
+    out_dir = os.path.join(volume_dir, "sql")
+    pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
+
+    produced = False
+
+    for row in entries.itertuples(index=False):
+        raw_db = getattr(row, "database", "")
+        user = (getattr(row, "username", "") or "").strip()
+        password = (getattr(row, "password", "") or "").strip()
+
+        db_value = _validate_database_value(raw_db, instance=instance_name)
+
+        # Explicit: dump ALL databases
+        if db_value == "*":
+            if db_type != "postgres":
+                raise ValueError(
+                    f"databases.csv entry for instance '{instance_name}': "
+                    "'*' is currently only supported for Postgres."
+                )
+
+            cluster_file = os.path.join(
+                out_dir, f"{instance_name}.cluster.backup.sql"
+            )
+            fallback_pg_dumpall(container, user, password, cluster_file)
+            produced = True
+            continue
+
+        # Concrete database dump
+        db_name = db_value
+        dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
+
+        if db_type == "mariadb":
+            cmd = (
+                f"docker exec {container} /usr/bin/mariadb-dump "
+                f"-u {user} -p{password} {db_name}"
+            )
+            _atomic_write_cmd(cmd, dump_file)
+            produced = True
+            continue
+
+        if db_type == "postgres":
+            try:
+                cmd = (
+                    f"PGPASSWORD={password} docker exec -i {container} "
+                    f"pg_dump -U {user} -d {db_name} -h localhost"
+                )
+                _atomic_write_cmd(cmd, dump_file)
+                produced = True
+            except BackupException as e:
+                # Explicit DB dump failed -> hard error
+                raise BackupException(
+                    f"Postgres dump failed for instance '{instance_name}', "
+                    f"database '{db_name}'. This database was explicitly configured "
+                    "and therefore must succeed.\n"
+                    f"{e}"
+                )
+            continue
+
+    return produced

backup_docker_to_local-1.2.0/src/baudolo/seed/__main__.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+import pandas as pd
+from typing import Optional
+
+
+DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
+
+def _validate_database_value(value: Optional[str], *, instance: str) -> str:
+    v = (value or "").strip()
+    if v == "":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            "column 'database' must be '*' or a concrete database name (not empty)."
+        )
+    if v == "*":
+        return "*"
+    if v.lower() == "nan":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': database must not be 'nan'."
+        )
+    if not DB_NAME_RE.match(v):
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            f"invalid database name '{v}'. Allowed: letters, numbers, '_' and '-'."
+        )
+    return v
+
+def check_and_add_entry(
+    file_path: str,
+    instance: str,
+    database: Optional[str],
+    username: str,
+    password: str,
+) -> None:
+    """
+    Add or update an entry in databases.csv.
+
+    The function enforces strict validation:
+    - database MUST be set
+    - database MUST be '*' or a valid database name
+    """
+    database = _validate_database_value(database, instance=instance)
+
+    if os.path.exists(file_path):
+        df = pd.read_csv(
+            file_path,
+            sep=";",
+            dtype=str,
+            keep_default_na=False,
+        )
+    else:
+        df = pd.DataFrame(
+            columns=["instance", "database", "username", "password"]
+        )
+
+    mask = (df["instance"] == instance) & (df["database"] == database)
+
+    if mask.any():
+        print("Updating existing entry.")
+        df.loc[mask, ["username", "password"]] = [username, password]
+    else:
+        print("Adding new entry.")
+        new_entry = pd.DataFrame(
+            [[instance, database, username, password]],
+            columns=["instance", "database", "username", "password"],
+        )
+        df = pd.concat([df, new_entry], ignore_index=True)
+
+    df.to_csv(file_path, sep=";", index=False)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Seed or update databases.csv for backup configuration."
+    )
+    parser.add_argument("file", help="Path to databases.csv")
+    parser.add_argument("instance", help="Instance name (e.g. bigbluebutton)")
+    parser.add_argument(
+        "database",
+        help="Database name or '*' to dump all databases",
+    )
+    parser.add_argument("username", help="Database username")
+    parser.add_argument("password", help="Database password")
+
+    args = parser.parse_args()
+
+    try:
+        check_and_add_entry(
+            file_path=args.file,
+            instance=args.instance,
+            database=args.database,
+            username=args.username,
+            password=args.password,
+        )
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

backup_docker_to_local-1.1.0/src/baudolo/backup/db.py

@@ -1,82 +0,0 @@
-from __future__ import annotations
-
-import os
-import pathlib
-import re
-
-import pandas
-import logging
-
-from .shell import BackupException, execute_shell_command
-
-log = logging.getLogger(__name__)
-
-
-def get_instance(container: str, database_containers: list[str]) -> str:
-    if container in database_containers:
-        return container
-    return re.split(r"(_|-)(database|db|postgres)", container)[0]
-
-
-def fallback_pg_dumpall(
-    container: str, username: str, password: str, out_file: str
-) -> None:
-    cmd = (
-        f"PGPASSWORD={password} docker exec -i {container} "
-        f"pg_dumpall -U {username} -h localhost > {out_file}"
-    )
-    execute_shell_command(cmd)
-
-
-def backup_database(
-    *,
-    container: str,
-    volume_dir: str,
-    db_type: str,
-    databases_df: "pandas.DataFrame",
-    database_containers: list[str],
-) -> None:
-    instance_name = get_instance(container, database_containers)
-    entries = databases_df.loc[databases_df["instance"] == instance_name]
-    if entries.empty:
-        log.warning("No entry found for instance '%s'", instance_name)
-        return
-
-    out_dir = os.path.join(volume_dir, "sql")
-    pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
-
-    for row in entries.iloc:
-        db_name = row["database"]
-        user = row["username"]
-        password = row["password"]
-
-        dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
-
-        if db_type == "mariadb":
-            cmd = (
-                f"docker exec {container} /usr/bin/mariadb-dump "
-                f"-u {user} -p{password} {db_name} > {dump_file}"
-            )
-            execute_shell_command(cmd)
-            continue
-
-        if db_type == "postgres":
-            cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
-            if not db_name:
-                fallback_pg_dumpall(container, user, password, cluster_file)
-                return
-
-            try:
-                cmd = (
-                    f"PGPASSWORD={password} docker exec -i {container} "
-                    f"pg_dump -U {user} -d {db_name} -h localhost > {dump_file}"
-                )
-                execute_shell_command(cmd)
-            except BackupException as e:
-                print(f"pg_dump failed: {e}", flush=True)
-                print(
-                    f"Falling back to pg_dumpall for instance '{instance_name}'",
-                    flush=True,
-                )
-                fallback_pg_dumpall(container, user, password, cluster_file)
-            continue

backup_docker_to_local-1.1.0/src/baudolo/seed/__main__.py

@@ -1,68 +0,0 @@
-import pandas as pd
-import argparse
-import os
-
-
-def check_and_add_entry(file_path, instance, database, username, password):
-    # Check if the file exists and is not empty
-    if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
-        # Read the existing CSV file with header
-        df = pd.read_csv(file_path, sep=";")
-    else:
-        # Create a new DataFrame with columns if file does not exist
-        df = pd.DataFrame(columns=["instance", "database", "username", "password"])
-
-    # Check if the entry exists and remove it
-    mask = (
-        (df["instance"] == instance)
-        & (
-            (df["database"] == database)
-            | (((df["database"].isna()) | (df["database"] == "")) & (database == ""))
-        )
-        & (df["username"] == username)
-    )
-
-    if not df[mask].empty:
-        print("Replacing existing entry.")
-        df = df[~mask]
-    else:
-        print("Adding new entry.")
-
-    # Create a new DataFrame for the new entry
-    new_entry = pd.DataFrame(
-        [
-            {
-                "instance": instance,
-                "database": database,
-                "username": username,
-                "password": password,
-            }
-        ]
-    )
-
-    # Add (or replace) the entry using concat
-    df = pd.concat([df, new_entry], ignore_index=True)
-
-    # Save the updated CSV file
-    df.to_csv(file_path, sep=";", index=False)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Check and replace (or add) a database entry in a CSV file."
-    )
-    parser.add_argument("file_path", help="Path to the CSV file")
-    parser.add_argument("instance", help="Database instance")
-    parser.add_argument("database", help="Database name")
-    parser.add_argument("username", help="Username")
-    parser.add_argument("password", nargs="?", default="", help="Password (optional)")
-
-    args = parser.parse_args()
-
-    check_and_add_entry(
-        args.file_path, args.instance, args.database, args.username, args.password
-    )
-
-
-if __name__ == "__main__":
-    main()