dar-backup 0.6.18__py3-none-any.whl → 0.6.20__py3-none-any.whl

dar_backup/manager.py

@@ -20,11 +20,14 @@
    This script creates and maintains `dar` databases with catalogs.
 """
 
-
+import argcomplete
 import argparse
 import os
 import re
 import sys
+import subprocess
+
+from inputimeout import inputimeout, TimeoutOccurred
 
 
 from . import __about__ as about
@@ -32,13 +35,17 @@ from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import setup_logging
 from dar_backup.util import CommandResult
 from dar_backup.util import get_logger
+from dar_backup.util import get_binary_info
+from dar_backup.util import show_version
+from dar_backup.util import print_aligned_settings
 
 from dar_backup.command_runner import CommandRunner   
 from dar_backup.command_runner import CommandResult
+from dar_backup.util import backup_definition_completer, list_archive_completer, archive_content_completer, add_specific_archive_completer
 
 from datetime import datetime
 from time import time
-from typing import Dict, List, NamedTuple
+from typing import Dict, List, NamedTuple, Tuple
 
 # Constants
 SCRIPTNAME = os.path.basename(__file__)
@@ -49,6 +56,15 @@ DB_SUFFIX = ".db"
 logger = None
 runner = None
 
+
+def get_db_dir(config_settings: ConfigSettings) -> str:
+    """
+    Return the correct directory for storing catalog databases.
+    Uses manager_db_dir if set, otherwise falls back to backup_dir.
+    """
+    return getattr(config_settings, "manager_db_dir", None) or config_settings.backup_dir
+
+
 def show_more_help():
     help_text = f"""
 NAME
@@ -57,68 +73,95 @@ NAME
     print(help_text)
 
 
-def create_db(backup_def: str, config_settings: ConfigSettings):
+def create_db(backup_def: str, config_settings: ConfigSettings, logger, runner) -> int:
+    db_dir = get_db_dir(config_settings)
+
+    if not os.path.exists(db_dir):
+        logger.error(f"DB dir does not exist: {db_dir}")
+        return 1
+    if not os.path.isdir(db_dir):
+        logger.error(f"DB path exists but is not a directory: {db_dir}")
+        return 1
+    if not os.access(db_dir, os.W_OK):
+        logger.error(f"DB dir is not writable: {db_dir}")
+        return 1
+
     database = f"{backup_def}{DB_SUFFIX}"
-    
-    database_path = os.path.join(config_settings.backup_dir, database)
-    
-    logger.debug(f"BACKUP_DIR: {config_settings.backup_dir}")
+    database_path = os.path.join(db_dir, database)
+
+    logger.debug(f"DB directory: {db_dir}")
 
     if os.path.exists(database_path):
-        logger.warning(f'"{database_path}" already exists, skipping creation')
+        logger.info(f'"{database_path}" already exists, skipping creation')
         return 0
     else:
         logger.info(f'Create catalog database: "{database_path}"')
-        command = ['dar_manager', '--create' , database_path]
+        command = ['dar_manager', '--create', database_path]
         process = runner.run(command)
         logger.debug(f"return code from 'db created': {process.returncode}")
         if process.returncode == 0:
             logger.info(f'Database created: "{database_path}"')
         else:
             logger.error(f'Something went wrong creating the database: "{database_path}"')
-            stdout, stderr = process.stdout, process.stderr 
+            stdout, stderr = process.stdout, process.stderr
             logger.error(f"stderr: {stderr}")
             logger.error(f"stdout: {stdout}")
 
     return process.returncode
 
 
-def list_catalogs(backup_def: str, config_settings: ConfigSettings) -> NamedTuple:
+def list_catalogs(backup_def: str, config_settings: ConfigSettings, suppress_output=False) -> CommandResult:
     """
+    List catalogs from the database for the given backup definition.
+
     Returns:
-       a typing.NamedTuple of class dar-backup.util.CommandResult with the following properties:
-        - process: of type subprocess.CompletedProcess: The result of the command execution.
-        - stdout: of type str: The standard output of the command.
-        - stderr: of type str: The standard error of the command.
-        - returncode: of type int: The return code of the command.
-        - timeout: of type int: The timeout value in seconds used to run the command.
-        - command: of type list[str): The command executed.
+        A CommandResult containing the raw stdout/stderr and return code.
     """
     database = f"{backup_def}{DB_SUFFIX}"
-    database_path = os.path.join(config_settings.backup_dir, database)
+    database_path = os.path.join(get_db_dir(config_settings), database)
+
     if not os.path.exists(database_path):
         error_msg = f'Database not found: "{database_path}"'
         logger.error(error_msg)
         return CommandResult(1, '', error_msg)
 
-        # commandResult = CommandResult(
-        # process=None,
-        # stdout='',
-        # stderr=error_msg,
-        # returncode=1,
-        # timeout=1,
-        # command=[])
-
-        # return commandResult
     command = ['dar_manager', '--base', database_path, '--list']
     process = runner.run(command)
-    stdout, stderr = process.stdout, process.stderr 
+    stdout, stderr = process.stdout, process.stderr
+
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
-        logger.error(f"stderr: {stderr}")  
+        logger.error(f"stderr: {stderr}")
         logger.error(f"stdout: {stdout}")
-    else:
-        print(stdout)
+        return process
+
+    # Extract only archive basenames from stdout
+    archive_names = []
+    for line in stdout.splitlines():
+        line = line.strip()
+        if not line or "archive #" in line or "dar path" in line or "compression" in line:
+            continue
+        parts = line.split("\t")
+        if len(parts) >= 3:
+            archive_names.append(parts[2].strip())
+
+    # Sort by prefix and date
+    def extract_date(arch_name):
+        match = re.search(r"(\d{4}-\d{2}-\d{2})", arch_name)
+        if match:
+            return datetime.strptime(match.group(1), "%Y-%m-%d")
+        return datetime.min
+
+    def sort_key(name):
+        prefix = name.split("_", 1)[0]
+        return (prefix, extract_date(name))
+
+    archive_names = sorted(archive_names, key=sort_key)
+
+    if not suppress_output:
+        for name in archive_names:
+            print(name)
+
     return process
 
 
@@ -130,47 +173,65 @@ def cat_no_for_name(archive: str, config_settings: ConfigSettings) -> int:
       - the found number, if the archive catalog is present in the database
       - "-1" if the archive is not found
     """
+    
     backup_def = backup_def_from_archive(archive)
-    process = list_catalogs(backup_def, config_settings)
+    process = list_catalogs(backup_def, config_settings, suppress_output=True)
     if process.returncode != 0:
         logger.error(f"Error listing catalogs for backup def: '{backup_def}'")
         return -1
     line_no = 1
     for line in process.stdout.splitlines():
-        #print(f"{line_no}: '{line}'")
         line_no += 1
         search = re.search(rf".*?(\d+)\s+.*?({archive}).*", line)
         if search:
-            #print(f"FOUND: archive: {search.group(2)}, catalog #: '{search.group(1)}'")
             logger.info(f"Found archive: '{archive}', catalog #: '{search.group(1)}'")
             return int(search.group(1))
     return -1
 
 
-
-def list_archive_contents(archive: str, config_settings: ConfigSettings) -> int :
+def list_archive_contents(archive: str, config_settings: ConfigSettings) -> int:
     """
-    List the contents of a specific archive, given the archive name
+    List the contents of a specific archive, given the archive name.
+    Prints only actual file entries (lines beginning with '[ Saved ]').
+    If none are found, a notice is printed instead.
     """
     backup_def = backup_def_from_archive(archive)
     database = f"{backup_def}{DB_SUFFIX}"
-    database_path = os.path.join(config_settings.backup_dir, database)
+    database_path = os.path.join(get_db_dir(config_settings), database)
+
     if not os.path.exists(database_path):
         logger.error(f'Database not found: "{database_path}"')
         return 1
+
     cat_no = cat_no_for_name(archive, config_settings)
     if cat_no < 0:
         logger.error(f"archive: '{archive}' not found in database: '{database_path}'")
         return 1
+
+
     command = ['dar_manager', '--base', database_path, '-u', f"{cat_no}"]
-    process = runner.run(command)
-    stdout, stderr = process.stdout, process.stderr 
+    process = runner.run(command, timeout = 10)
+
+
+    stdout = process.stdout or ""
+    stderr = process.stderr or ""
+
+
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
-        logger.error(f"stderr: {stderr}")  
+        logger.error(f"stderr: {stderr}")
         logger.error(f"stdout: {stdout}")
+
+
+    combined_lines = (stdout + "\n" + stderr).splitlines()
+    file_lines = [line for line in combined_lines if line.strip().startswith("[ Saved ]")]
+
+    if file_lines:
+        for line in file_lines:
+            print(line)
     else:
-        print(stdout)
+        print(f"[info] Archive '{archive}' is empty.")
+
     return process.returncode
 
 
@@ -179,8 +240,9 @@ def list_catalog_contents(catalog_number: int, backup_def: str, config_settings:
     """
     List the contents of catalog # in catalog database for given backup definition
     """
+    logger = get_logger()
     database = f"{backup_def}{DB_SUFFIX}"
-    database_path = os.path.join(config_settings.backup_dir, database)
+    database_path = os.path.join(get_db_dir(config_settings), database)
     if not os.path.exists(database_path):
         logger.error(f'Catalog database not found: "{database_path}"')
         return 1
@@ -201,7 +263,7 @@ def find_file(file, backup_def, config_settings):
     Find a specific file
     """
     database = f"{backup_def}{DB_SUFFIX}"
-    database_path = os.path.join(config_settings.backup_dir, database)
+    database_path = os.path.join(get_db_dir(config_settings), database)
     if not os.path.exists(database_path):
         logger.error(f'Database not found: "{database_path}"')
         return 1
@@ -217,42 +279,82 @@ def find_file(file, backup_def, config_settings):
     return process.returncode
 
 
-def add_specific_archive(archive: str, config_settings: ConfigSettings, directory: str =None) -> int:    
-    # sanity check - does dar backup exist?
+def add_specific_archive(archive: str, config_settings: ConfigSettings, directory: str = None) -> int:
+    """
+    Adds the specified archive to its catalog database. Prompts for confirmation if it's older than existing entries.
+
+    Returns:
+        0 on success
+        1 on failure
+    """
+    # Determine archive path
     if not directory:
         directory = config_settings.backup_dir
-    archive = os.path.basename(archive)  # remove path if it was given
-    archive_path = os.path.join(directory, f'{archive}')
+    archive = os.path.basename(archive)  # strip path if present
+    archive_path = os.path.join(directory, archive)
+    archive_test_path = os.path.join(directory, f'{archive}.1.dar')
 
-    archive_test_path =  os.path.join(directory, f'{archive}.1.dar')
     if not os.path.exists(archive_test_path):
         logger.error(f'dar backup: "{archive_test_path}" not found, exiting')
         return 1
-        
-    # sanity check - does backup definition exist?
+
+    # Validate backup definition
     backup_definition = archive.split('_')[0]
     backup_def_path = os.path.join(config_settings.backup_d_dir, backup_definition)
     if not os.path.exists(backup_def_path):
         logger.error(f'backup definition "{backup_definition}" not found (--add-specific-archive option probably not correct), exiting')
         return 1
-    
+
+    # Determine catalog DB path
     database = f"{backup_definition}{DB_SUFFIX}"
-    database_path = os.path.realpath(os.path.join(config_settings.backup_dir, database))
+    database_path = os.path.realpath(os.path.join(get_db_dir(config_settings), database))
+
+    # Safety check: is archive older than latest in catalog?
+    try:
+        result = subprocess.run(
+            ["dar_manager", "--base", database_path, "--list"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            text=True,
+            check=True
+        )
+        all_lines = result.stdout.splitlines()
+        date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+        catalog_dates = [
+            datetime.strptime(date_match.group(), "%Y-%m-%d")
+            for line in all_lines
+            if (date_match := date_pattern.search(line))
+        ]
+
+        if catalog_dates:
+            latest_date = max(catalog_dates)
+            archive_date_match = date_pattern.search(archive)
+            if archive_date_match:
+                archive_date = datetime.strptime(archive_date_match.group(), "%Y-%m-%d")
+                if archive_date < latest_date:
+                    if not confirm_add_old_archive(archive, latest_date.strftime("%Y-%m-%d")):
+                        logger.info(f"Archive {archive} skipped due to user declining to add older archive.")
+                        return 1
+
+    except subprocess.CalledProcessError:
+        logger.warning("Could not determine latest catalog date for chronological check.")
+
     logger.info(f'Add "{archive_path}" to catalog: "{database}"')
-    
-    command = ['dar_manager', '--base', database_path, "--add", archive_path, "-Q"]
+
+    command = ['dar_manager', '--base', database_path, "--add", archive_path, "-Q", "--alter=ignore-order"]
     process = runner.run(command)
     stdout, stderr = process.stdout, process.stderr
 
     if process.returncode == 0:
-        logger.info(f'"{archive_path}" added to it\'s catalog')
+        logger.info(f'"{archive_path}" added to its catalog')
     elif process.returncode == 5:
-        logger.warning(f'Something did not go completely right adding "{archive_path}" to it\'s catalog, dar_manager error: "{process.returncode}"')
-    else: 
-        logger.error(f'something went wrong adding "{archive_path}" to it\'s catalog, dar_manager error: "{process.returncode}"')
+        logger.warning(f'Something did not go completely right adding "{archive_path}" to its catalog, dar_manager error: "{process.returncode}"')
+    else:
+        logger.error(f'something went wrong adding "{archive_path}" to its catalog, dar_manager error: "{process.returncode}"')
         logger.error(f"stderr: {stderr}")
         logger.error(f"stdout: {stdout}")
-     
+
     return process.returncode
 
 
@@ -330,6 +432,31 @@ def backup_def_from_archive(archive: str) -> str:
     return None
 
 
+def confirm_add_old_archive(archive_name: str, latest_known_date: str, timeout_secs: int = 20) -> bool:
+    """
+    Confirm with the user if they want to proceed with adding an archive older than the most recent in the catalog.
+    Returns True if the user confirms with "yes", False otherwise.
+    """
+    try:
+        prompt = (
+            f"⚠️ Archive '{archive_name}' is older than the latest in the catalog ({latest_known_date}).\n"
+            f"Adding older archives may lead to inconsistent restore chains.\n"
+            f"Are you sure you want to continue? (yes/no): "
+        )
+        confirmation = inputimeout(prompt=prompt, timeout=timeout_secs)
+
+        if confirmation is None:
+            logger.info(f"No confirmation received for old archive: {archive_name}. Skipping.")
+            return False
+        return confirmation.strip().lower() == "yes"
+
+    except TimeoutOccurred:
+        logger.info(f"Timeout waiting for confirmation for old archive: {archive_name}. Skipping.")
+        return False
+    except KeyboardInterrupt:
+        logger.info(f"User interrupted confirmation for old archive: {archive_name}. Skipping.")
+        return False
+
 
 def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> int:
     """
@@ -341,7 +468,7 @@ def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> in
 
     """
     backup_def = backup_def_from_archive(archive)
-    database_path = os.path.join(config_settings.backup_dir, f"{backup_def}{DB_SUFFIX}")
+    database_path = os.path.join(get_db_dir(config_settings), f"{backup_def}{DB_SUFFIX}")
     cat_no:int = cat_no_for_name(archive, config_settings)
     if cat_no >= 0:
         command = ['dar_manager', '--base', database_path, "--delete", str(cat_no)]
@@ -366,12 +493,11 @@ def build_arg_parser():
     parser.add_argument('--create-db', action='store_true', help='Create missing databases for all backup definitions')
     parser.add_argument('--alternate-archive-dir', type=str, help='Use this directory instead of BACKUP_DIR in config file')
     parser.add_argument('--add-dir', type=str, help='Add all archive catalogs in this directory to databases')
-    parser.add_argument('-d', '--backup-def', type=str, help='Restrict to work only on this backup definition')
-    parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog database')
-    parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog database')
+    parser.add_argument('-d', '--backup-def', type=str, help='Restrict to work only on this backup definition').completer = backup_definition_completer
+    parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog database').completer = add_specific_archive_completer
+    parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog database').completer = archive_content_completer
     parser.add_argument('-l', '--list-catalogs', action='store_true', help='List catalogs in databases for all backup definitions')
-    parser.add_argument('--list-catalog-contents', type=int, help="List contents of a catalog. Argument is the 'archive #', '-d <definition>' argument is also required")
-    parser.add_argument('--list-archive-contents', type=str, help="List contents of the archive's catalog.")
+    parser.add_argument('--list-archive-contents', type=str, help="List contents of the archive's catalog. Argument is the archive name.").completer = archive_content_completer
     parser.add_argument('--find-file', type=str, help="List catalogs containing <path>/file. '-d <definition>' argument is also required")
     parser.add_argument('--verbose', action='store_true', help='Be more verbose')
     parser.add_argument('--log-level', type=str, help="`debug` or `trace`, default is `info`", default="info")
@@ -382,8 +508,6 @@ def build_arg_parser():
     return parser
 
 
-
-
 def main():
     global logger, runner 
 
@@ -394,9 +518,10 @@ def main():
         return
 
     parser = argparse.ArgumentParser(description="Creates/maintains `dar` database catalogs")
-    # [parser.add_argument(...) as before...]
-
     parser = build_arg_parser()
+
+    argcomplete.autocomplete(parser)
+
     args = parser.parse_args()
 
     if args.more_help:
@@ -405,16 +530,13 @@ def main():
         return
 
     if args.version:
-        print(f"{SCRIPTNAME} {about.__version__}")
-        print(f"Source code is here: https://github.com/per2jensen/dar-backup")
-        print('''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
-THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-See section 15 and section 16 in the supplied "LICENSE" file.''')
+        show_version()
         sys.exit(0)
-        return
 
     args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
     config_settings = ConfigSettings(args.config_file)
+    print(f"Config settings: {config_settings}")
+
     if not os.path.dirname(config_settings.logfile_location):
         print(f"Directory for log file '{config_settings.logfile_location}' does not exist, exiting")
         sys.exit(1)
@@ -425,12 +547,23 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
     command_logger = get_logger(command_output_logger=True)
     runner = CommandRunner(logger=logger, command_logger=command_logger)
 
+    start_msgs: List[Tuple[str, str]] = []
+
     start_time = int(time())
-    logger.info(f"=====================================")
-    logger.info(f"{SCRIPTNAME} started, version: {about.__version__}")
+    start_msgs.append((f"{SCRIPTNAME}:", about.__version__))
     logger.info(f"START TIME: {start_time}")
     logger.debug(f"`args`:\n{args}")
     logger.debug(f"`config_settings`:\n{config_settings}")
+    start_msgs.append(("Config file:", args.config_file))
+    args.verbose and start_msgs.append(("Backup dir:", config_settings.backup_dir))
+    start_msgs.append(("Logfile:", config_settings.logfile_location))
+    args.verbose and start_msgs.append(("--alternate-archive-dir:", args.alternate_archive_dir))
+    args.verbose and start_msgs.append(("--cleanup-specific-archives:", args.cleanup_specific_archives)) 
+    dar_manager_properties = get_binary_info(command='dar_manager')
+    start_msgs.append(("dar_manager:", dar_manager_properties['path']))
+    start_msgs.append(("dar_manager v.:", dar_manager_properties['version']))
+    
+    print_aligned_settings(start_msgs)
 
     # --- Sanity checks ---
     if args.add_dir and not args.add_dir.strip():
@@ -475,11 +608,6 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
         sys.exit(1)
         return
 
-    if args.list_catalog_contents and not args.backup_def:
-        logger.error(f"--list-catalog-contents requires the --backup-def, exiting")
-        sys.exit(1)
-        return
-
     if args.find_file and not args.backup_def:
         logger.error(f"--find-file requires the --backup-def, exiting")
         sys.exit(1)
@@ -496,14 +624,14 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
     # --- Functional logic ---
     if args.create_db:
         if args.backup_def:
-            sys.exit(create_db(args.backup_def, config_settings))
+            sys.exit(create_db(args.backup_def, config_settings, logger, runner))
             return
         else:
             for root, dirs, files in os.walk(config_settings.backup_d_dir):
                 for file in files:
                     current_backupdef = os.path.basename(file)
                     logger.debug(f"Create catalog db for backup definition: '{current_backupdef}'")
-                    result = create_db(current_backupdef, config_settings)
+                    result = create_db(current_backupdef, config_settings, logger, runner)
                     if result != 0:
                         sys.exit(result)
                         return
@@ -538,10 +666,6 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
         sys.exit(result)
         return
 
-    if args.list_catalog_contents:
-        result = list_catalog_contents(args.list_catalog_contents, args.backup_def, config_settings)
-        sys.exit(result)
-        return
 
     if args.find_file:
         result = find_file(args.find_file, args.backup_def, config_settings)