dar-backup 0.6.1__py3-none-any.whl → 0.6.3__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.1"
+__version__ = "0.6.3"

dar_backup/dar_backup.py

@@ -17,56 +17,49 @@ from argparse import ArgumentParser
 from datetime import datetime
 from pathlib import Path
 from time import time
+from typing import List
 
 from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import list_backups
 from dar_backup.util import run_command
 from dar_backup.util import setup_logging
-from dar_backup.util import extract_error_lines
 from dar_backup.util import BackupError
-from dar_backup.util import DifferentialBackupError
-from dar_backup.util import IncrementalBackupError
 from dar_backup.util import RestoreError
 
 
 logger = None
 
-
-def backup(backup_file: str, backup_definition: str, darrc: str,  config_settings: ConfigSettings):
+def generic_backup(type: str, command: List[str], backup_file: str, backup_definition: str, darrc: str,  config_settings: ConfigSettings):
     """
-    Performs a full backup using the 'dar' command.
+    Performs a backup using the 'dar' command.
 
     This function initiates a full backup operation by constructing and executing a command
     with the 'dar' utility. It checks if the backup file already exists to avoid overwriting
     previous backups. If the backup file does not exist, it proceeds with the backup operation.
 
     Args:
+        type (str): The type of backup (FULL, DIFF, INCR).
+        command (List[str]): The command to execute for the backup operation.
         backup_file (str): The base name of the backup file. The actual backup will be saved
                            as '{backup_file}.1.dar'.
         backup_definition (str): The path to the backup definition file. This file contains
                                  specific instructions for the 'dar' utility, such as which
                                  directories to include or exclude.
+        darrc (str): The path to the '.darrc' configuration file.
+        config_settings (ConfigSettings): An instance of the ConfigSettings class.
+
 
-    Note:
-        This function logs an error and returns early if the backup file already exists.
-        It logs the command being executed and reports upon successful completion of the backup.
 
     Raises:
         BackupError: If an error occurs during the backup process.
     """
-    if os.path.exists(backup_file + '.1.dar'):
-        logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
-        return
-
-    logger.info(f"===> Starting FULL backup for {backup_definition}")
-    command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition, '-Q', "compress-exclusion",  "verbose"]
+    logger.info(f"===> Starting {type} backup for {backup_definition}")
     logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
     try:
         process = run_command(command, config_settings.command_timeout_secs)
-        logger.info("Back from run_command")
         if process.returncode == 0:
-            logger.info("FULL backup completed successfully.")
+            logger.info(f"{type} backup completed successfully.")
         elif process.returncode == 5:
             logger.warning("Backup completed with some files not backed up, this can happen if files are changed/deleted during the backup.")
         else:
@@ -78,106 +71,6 @@ def backup(backup_file: str, backup_definition: str, darrc: str,  config_setting
         logger.exception(f"Unexpected error during backup")
         raise BackupError(f"Unexpected error during backup: {e}") from e
  
- 
-
-
-
-def differential_backup(backup_file: str, backup_definition: str, base_backup_file: str, darrc: str, config_settings: ConfigSettings):
-    """
-    Creates a differential backup based on a specified base backup.
-
-    This function performs a differential backup by comparing the current state of the data
-    against a specified base backup file. It captures only the changes made since that base
-    backup, resulting in a smaller and faster backup process compared to a full backup.
-
-    Args:
-        backup_file (str): The base name for the differential backup file. The actual backup
-                           will be saved as '{backup_file}.1.dar'.
-        backup_definition (str): The path to the backup definition file. This file contains
-                                 specific instructions for the 'dar' utility, such as which
-                                 directories to include or exclude.
-        base_backup_file (str): The base name of the full backup file that serves as the
-                                reference point for the differential backup.
-
-    Note:
-        This function logs an error and returns early if the differential backup file already exists.
-        It logs the command being executed and reports upon successful completion of the differential backup.
-
-    Raises:
-        DifferentialBackupError: If the differential backup command fails or encounters an unexpected error.
-    """
-    if os.path.exists(backup_file + '.1.dar'):
-        logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
-        return
-
-    logger.info(f"===> Starting DIFF backup for {backup_definition}")
-    command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition, '-A', base_backup_file, '-Q', "compress-exclusion",  "verbose"]
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
-    try:
-        process = run_command(command)
-        if process.returncode == 0:
-            logger.info("DIFF backup completed successfully.")
-        elif process.returncode == 5:
-            logger.warning("Backup completed with some files not backed up, this can happen if files are changed/deleted during the backup.")
-        else:
-            raise Exception(str(process))
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Differential backup command failed: {e}")
-        raise DifferentialBackupError(f"Differential backup command failed: {e}") from e
-    except Exception as e:
-        logger.exception(f"Unexpected error during differential backup")
-        logger.error("Exception details:", exc_info=True)
-        raise DifferentialBackupError(f"Unexpected error during differential backup: {e}") from e
-
-
-def incremental_backup(backup_file: str, backup_definition: str, last_backup_file: str, darrc: str, config_settings: ConfigSettings):
-    """
-    Creates an incremental backup based on the last backup file.
-
-    This function performs an incremental backup by comparing the current state of the data
-    against the last backup file, whether it's a full backup or the most recent incremental backup.
-    It captures only the changes made since that last backup, making it efficient for frequent
-    backups with minimal data changes.
-
-    Args:
-        backup_file (str): The base name for the incremental backup file. The actual backup
-                           will be saved with a unique identifier to distinguish it from other backups.
-        backup_definition (str): The path to the backup definition file. This file contains
-                                 specific instructions for the 'dar' utility, such as which
-                                 directories to include or exclude.
-        last_backup_file (str): The base name of the last backup file (full or incremental) that
-                                serves as the reference point for the incremental backup.
-
-    Note:
-        This function checks if the incremental backup file already exists to prevent overwriting
-        previous backups. It logs the command being executed and reports upon successful completion
-        of the incremental backup.
-    
-    Raises:
-        IncrementalBackupError: If the incremental backup command fails or an unexpected error occurs.
-    """
-    if os.path.exists(backup_file + '.1.dar'):
-        logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
-        return
-
-    logger.info(f"===> Starting INCR backup for {backup_definition}")
-    command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition, '-A', last_backup_file, '-Q', "compress-exclusion",  "verbose"]
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
-    try:
-        process = run_command(command, config_settings.command_timeout_secs)
-        if process.returncode == 0:
-            logger.info("INCR backup completed successfully.")
-        elif process.returncode == 5:
-            logger.warning("Backup completed with some files not backed up, this can happen if files are changed/deleted during the backup.")
-        else:
-            raise Exception(str(process))
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Incremental backup command failed: {e}")
-        raise IncrementalBackupError(f"Incremental backup command failed: {e}") from e
-    except Exception as e:
-        logger.exception(f"Unexpected error during incremental backup")
-        raise IncrementalBackupError(f"Unexpected error during incremental backup: {e}") from e
-
 
 # Function to recursively find <File> tags and build their full paths
 def find_files_with_paths(element: ET, current_path=""):
@@ -331,7 +224,7 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
     Restores a backup file to a specified directory.
 
     Args:
-        backup_name (str): The name of the backup file.
+        backup_name (str): The base name of the backup file, without the "slice number.dar"
         backup_dir (str): The directory where the backup file is located.
         restore_dir (str): The directory where the backup should be restored to.
         selection (str, optional): A selection criteria to restore specific files or directories. Defaults to None.
@@ -342,16 +235,19 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
         if not os.path.exists(restore_dir):
             os.makedirs(restore_dir)
         command.extend(['-R', restore_dir])
+    else:
+        raise RestoreError("Restore directory ('-R <dir>') not specified")
     if selection:
         selection_criteria = shlex.split(selection)
         command.extend(selection_criteria)
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
+    logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
     try:
         process = run_command(command, config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"Restore completed successfully to: '{restore_dir}'")
         else:
-            raise Exception(str(process))
+            logger.error(f"Restore command failed: \n ==> stdout: {process.stdout}, \n ==> stderr: {process.stderr}")
+            raise RestoreError(str(process))
     except subprocess.CalledProcessError as e:
         raise RestoreError(f"Restore command failed: {e}") from e
     except Exception as e:
@@ -405,6 +301,7 @@ def list_contents(backup_name, backup_dir, selection=None):
         None
     """
     backup_path = os.path.join(backup_dir, backup_name)
+
     try:
         command = ['dar', '-l', backup_path, '-am', '-as', '-Q']
         if selection:
@@ -427,6 +324,32 @@ def list_contents(backup_name, backup_dir, selection=None):
 
 
 
+
+def create_backup_command(backup_type: str, backup_file: str, darrc: str, backup_definition_path: str, latest_base_backup: str = None) -> List[str]:
+    """
+    Generate the backup command for the specified backup type.
+
+    Args:
+        backup_type (str): The type of backup (FULL, DIFF, INCR).
+        backup_file (str): The backup file path. Example: /path/to/example_2021-01-01_FULL
+        darrc (str): Path to the .darrc configuration file.
+        backup_definition_path (str): Path to the backup definition file.
+        latest_base_backup (str, optional): Path to the latest base backup for DIFF or INCR types.
+
+    Returns:
+        List[str]: The constructed backup command.
+    """
+    base_command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition_path, '-Q', "compress-exclusion", "verbose"]
+    
+    if backup_type in ['DIFF', 'INCR']:
+        if not latest_base_backup:
+            raise ValueError(f"Base backup is required for {backup_type} backups.")
+        base_command.extend(['-A', latest_base_backup])
+    
+    return base_command
+
+
+
 def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, backup_type: str):
     """
     Perform backup operation.
@@ -434,32 +357,12 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
     Args:
         args: Command-line arguments.
         config_settings: An instance of the ConfigSettings class.
-        backup_d: Directory containing backup definitions.
-        backup_dir: Directory to store backup files.
-        test_restore_dir: Directory to test restore backup files.
         backup_type: Type of backup (FULL, DIFF, INCR).
-        min_size_verification_mb: Minimum size for verification in MB.
-        max_size_verification_mb: Maximum size for verification in MB.
-        no_files_verification: Flag indicating whether to skip file verification.
-
-    Returns:
-        None
-
-    Raises:
-        FileNotFoundError: If `backup_d` does not exist or a specified backup definition file does not exist.
-        PermissionError: If there is insufficient permission to access directories or files specified.
-        OSError: For various system-related errors, such as exhaustion of file descriptors.
-        ValueError: If there is an issue with the format string in `datetime.now().strftime`.
-        subprocess.CalledProcessError: If a subprocess invoked during the backup process exits with a non-zero status.
-        Exception: Catches any unexpected exceptions that may occur during the backup process.
-
-    Note: 
-      This function assumes that any exceptions raised by the `backup` function or related subprocesses are handled
-      within those functions or propagated up to be handled by the caller of `perform_backup`.
     """
     logger.debug(f"perform_backup({backup_type}) started")
     backup_definitions = []
 
+    # Gather backup definitions
     if args.backup_definition:
         if '_' in args.backup_definition:
             logger.error(f"Skipping backup definition: '{args.backup_definition}' due to '_' in name")
@@ -482,16 +385,15 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                 logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
                 continue
 
-            if backup_type == 'FULL':
-                backup(backup_file, backup_definition_path, args.darrc, config_settings)
-            else:
+            latest_base_backup = None
+            if backup_type in ['DIFF', 'INCR']:
                 base_backup_type = 'FULL' if backup_type == 'DIFF' else 'DIFF'
-                
+
                 if args.alternate_reference_archive:
-                    latest_base_backup = os.path.join(config_settings.backup_dir, args.alternate_reference_archive)  # expects alternerate reference archive to be without slice number
+                    latest_base_backup = os.path.join(config_settings.backup_dir, args.alternate_reference_archive)
                     logger.info(f"Using alternate reference archive: {latest_base_backup}")
                     if not os.path.exists(latest_base_backup + '.1.dar'):
-                        logger.error(f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exciting.")
+                        logger.error(f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exiting.")
                         sys.exit(1)
                 else:
                     base_backups = sorted(
@@ -503,33 +405,36 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                         continue
                     latest_base_backup = os.path.join(config_settings.backup_dir, base_backups[-1].rsplit('.', 2)[0])
 
-                if backup_type == 'DIFF':
-                    differential_backup(backup_file, backup_definition_path, latest_base_backup, args.darrc, config_settings)
-                elif backup_type == 'INCR':
-                    incremental_backup(backup_file, backup_definition_path, latest_base_backup, args.darrc, config_settings) 
+            # Generate the backup command
+            command = create_backup_command(backup_type, backup_file, args.darrc, backup_definition_path, latest_base_backup)
+
+            # Perform backup
+            generic_backup(backup_type, command, backup_file, backup_definition_path, args.darrc, config_settings)
 
             logger.info("Starting verification...")
             result = verify(args, backup_file, backup_definition_path, config_settings)
             if result:
                 logger.info("Verification completed successfully.")
             else:
-                logger.error("Verification failed.") 
+                logger.error("Verification failed.")
+
             if config_settings.par2_enabled:
-                logger.info("Generate par2 redundancy files") 
-                generate_par2_files(backup_file, config_settings)  # do this even if verification failed, because verification could fail on an open file.
+                logger.info("Generate par2 redundancy files.")
+                generate_par2_files(backup_file, config_settings, args)
                 logger.info("par2 files completed successfully.")
-        # we want to continue with other backup definitions, thus only logging an error
         except Exception as e:
-            logger.exception(f"Error during {backup_type} backup process, continuing on next backup definition")
-            logger.error("Exception details:", exc_info=True)
+            logger.exception(f"Error during {backup_type} backup process, continuing to next backup definition.")
+
+
 
-def generate_par2_files(backup_file: str, config_settings: ConfigSettings):
+def generate_par2_files(backup_file: str, config_settings: ConfigSettings, args):
     """
     Generate PAR2 files for a given backup file in the specified backup directory.
 
     Args:
         backup_file (str): The name of the backup file.
-        backup_dir (str): The path to the backup directory.
+        config_settings: The configuration settings object.
+        args: The command-line arguments object.
 
     Raises:
         subprocess.CalledProcessError: If the par2 command fails to execute.
@@ -537,19 +442,39 @@ def generate_par2_files(backup_file: str, config_settings: ConfigSettings):
     Returns:
         None
     """
+    # Regular expression to match DAR slice files
+    dar_slice_pattern = re.compile(rf"{re.escape(os.path.basename(backup_file))}\.([0-9]+)\.dar")
+
+    # List of DAR slice files to be processed
+    dar_slices: List[str] = []
+
     for filename in os.listdir(config_settings.backup_dir):
-        if os.path.basename(backup_file) in filename:
-            # Construct the full path to the file
-            file_path = os.path.join(config_settings.backup_dir, filename)
-            # Run the par2 command to generate redundancy files with error correction
-            command = ['par2', 'create', f'-r{config_settings.error_correction_percent}', '-q', '-q', file_path]
-            process = run_command(command, config_settings.command_timeout_secs)
-            if process.returncode != 0:
-                logger.error(f"Error generating par2 files for {file_path}")
-                raise subprocess.CalledProcessError(process.returncode, command)
-            logger.debug(f"par2 files generated for {file_path}")
+        match = dar_slice_pattern.match(filename)
+        if match:
+            dar_slices.append(filename)
 
+    # Sort the DAR slices based on the slice number
+    dar_slices.sort(key=lambda x: int(dar_slice_pattern.match(x).group(1)))
+    number_of_slices = len(dar_slices)
+    counter = 1
 
+    for slice_file in dar_slices:
+        file_path = os.path.join(config_settings.backup_dir, slice_file)
+    
+        if args.verbose or args.log_level == "debug" or args.log_level == "trace":
+            logger.info(f"{counter}/{number_of_slices}: Now generating par2 files for {file_path}")
+
+        # Run the par2 command to generate redundancy files with error correction
+        command = ['par2', 'create', f'-r{config_settings.error_correction_percent}', '-q', '-q', file_path]
+        process = run_command(command, config_settings.command_timeout_secs)
+
+        if process.returncode == 0:
+            if args.verbose or args.log_level == "debug" or args.log_level == "trace":
+                logger.info(f"{counter}/{number_of_slices}: Done")
+        else:
+            logger.error(f"Error generating par2 files for {file_path}")
+            raise subprocess.CalledProcessError(process.returncode, command)
+        counter += 1
 
 
 
@@ -658,8 +583,9 @@ def main():
     parser.add_argument('-l', '--list', action='store_true', help="List available archives.")
     parser.add_argument('--list-contents', help="List the contents of the specified archive.")
     parser.add_argument('--selection', help="dar file selection for listing/restoring specific files/directories.")
-    parser.add_argument('-r', '--restore', nargs=1, type=str, help="Restore specified archive.")
-    parser.add_argument('--restore-dir', nargs=1, type=str, help="Directory to restore files to.")
+#    parser.add_argument('-r', '--restore', nargs=1, type=str, help="Restore specified archive.")
+    parser.add_argument('-r', '--restore', type=str, help="Restore specified archive.")
+    parser.add_argument('--restore-dir',   type=str, help="Directory to restore files to.")
     parser.add_argument('--verbose', action='store_true', help="Print various status messages to screen")
     parser.add_argument('--log-level', type=str, help="`debug` or `trace`", default="info")
     parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
@@ -708,7 +634,10 @@ def main():
             args.verbose and (print(f"Alternate ref archive: {args.alternate_reference_archive}"))
         args.verbose and (print(f"Backup.d dir:      {config_settings.backup_d_dir}"))
         args.verbose and (print(f"Backup dir:        {config_settings.backup_dir}"))
-        args.verbose and (print(f"Test restore dir:  {config_settings.test_restore_dir}"))
+
+        restore_dir = args.restore_dir if args.restore_dir else config_settings.test_restore_dir
+        args.verbose and (print(f"Restore dir:       {restore_dir}"))
+
         args.verbose and (print(f"Logfile location:  {config_settings.logfile_location}"))
         args.verbose and (print(f".darrc location:   {args.darrc}"))
         args.verbose and (print(f"PAR2 enabled:      {config_settings.par2_enabled}"))
@@ -735,10 +664,9 @@ def main():
         elif args.incremental_backup  and not args.full_backup and not args.differential_backup:
             perform_backup(args, config_settings, "INCR")
         elif args.list_contents:
-            print(f"Listing contents of {args.list_contents}")
             list_contents(args.list_contents, config_settings.backup_dir, args.selection)
         elif args.restore:
-            restore_dir = args.restore_dir if args.restore_dir else config_settings.test_restore_dir
+            logger.debug(f"Restoring {args.restore} to {restore_dir}")
             restore_backup(args.restore, config_settings, restore_dir, args.selection)
         else:
             parser.print_help()

dar_backup/manager.py

@@ -21,8 +21,9 @@
 """
 
 
-import os
 import argparse
+import os
+import re
 import sys
 
 
@@ -30,7 +31,9 @@ from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import run_command
 from dar_backup.util import setup_logging
+from datetime import datetime
 from time import time
+from typing import Dict, List, NamedTuple
 
 # Constants
 SCRIPTNAME = os.path.basename(__file__)
@@ -70,9 +73,20 @@ def create_db(backup_def: str, config_settings: ConfigSettings):
             logger.error(f"stderr: {stderr}")
             logger.error(f"stdout: {stdout}")
 
+    return process.returncode
 
 
-def list_db(backup_def: str, config_settings: ConfigSettings):
+def list_catalogs(backup_def: str, config_settings: ConfigSettings) -> NamedTuple:
+    """
+    Returns:
+       a typing.NamedTuple of class dar-backup.util.CommandResult with the following properties:
+        - process: of type subprocess.CompletedProcess: The result of the command execution.
+        - stdout: of type str: The standard output of the command.
+        - stderr: of type str: The standard error of the command.
+        - returncode: of type int: The return code of the command.
+        - timeout: of type int: The timeout value in seconds used to run the command.
+        - command: of type list[str): The command executed.
+    """
     database = f"{backup_def}{DB_SUFFIX}"
     database_path = os.path.join(config_settings.backup_dir, database)
     if not os.path.exists(database_path):
@@ -87,27 +101,101 @@ def list_db(backup_def: str, config_settings: ConfigSettings):
         logger.error(f"stdout: {stdout}")
     else:
         print(stdout)
-    sys.exit(process.returncode)
+    return process
+
+
+def cat_no_for_name(archive: str, config_settings: ConfigSettings) -> int:
+    """
+    Find the catalog number for the given archive name
+
+    Returns:
+      - the found number, if the archive catalog is present in the database
+      - "-1" if the archive is not found
+    """
+    backup_def = backup_def_from_archive(archive)
+    process = list_catalogs(backup_def, config_settings)
+    if process.returncode != 0:
+        logger.error(f"Error listing catalogs for backup def: '{backup_def}'")
+        return -1
+    line_no = 1
+    for line in process.stdout.splitlines():
+        #print(f"{line_no}: {line}")
+        line_no += 1
+        search = re.search(f"\s+(\d+)\s+.*?({archive}).*", line)
+        if search:
+            #print("FOUND")
+            logger.info(f"Found archive: '{archive}', catalog #: '{search.group(1)}'")
+            return int(search.group(1))
+    return -1
+
+
+
+
+
+def list_catalog_contents(catalog_number: int, backup_def: str, config_settings: ConfigSettings):
+    """
+    List the contents of catalog # in catalog database for given backup definition
+    """
+    database = f"{backup_def}{DB_SUFFIX}"
+    database_path = os.path.join(config_settings.backup_dir, database)
+    if not os.path.exists(database_path):
+        logger.error(f'Database not found: "{database_path}"')
+        return 1
+    command = ['dar_manager', '--base', database_path, '-u', f"{catalog_number}"]
+    process = run_command(command)
+    stdout, stderr = process.stdout, process.stderr 
+    if process.returncode != 0:
+        logger.error(f'Error listing catalogs for: "{database_path}"')
+        logger.error(f"stderr: {stderr}")  
+        logger.error(f"stdout: {stdout}")
+    else:
+        print(stdout)
+    return process.returncode
+
+
+def find_file(file, backup_def, config_settings):
+    """
+    Find a specific file
+    """
+    database = f"{backup_def}{DB_SUFFIX}"
+    database_path = os.path.join(config_settings.backup_dir, database)
+    if not os.path.exists(database_path):
+        logger.error(f'Database not found: "{database_path}"')
+        return 1
+    command = ['dar_manager', '--base', database_path, '-f', f"{file}"]
+    process = run_command(command)
+    stdout, stderr = process.stdout, process.stderr 
+    if process.returncode != 0:
+        logger.error(f'Error finding file: {file} in: "{database_path}"')
+        logger.error(f"stderr: {stderr}")  
+        logger.error(f"stdout: {stdout}")
+    else:
+        print(stdout)
+    return process.returncode
 
 
-def add_specific_archive(archive: str, config_settings: ConfigSettings):    
+def add_specific_archive(archive: str, config_settings: ConfigSettings, directory: str =None) -> int:    
     # sanity check - does dar backup exist?
+    if not directory:
+        directory = config_settings.backup_dir
     archive = os.path.basename(archive)  # remove path if it was given
-    archive_path = os.path.join(config_settings.backup_dir, f'{archive}.1.dar')
-    if not os.path.exists(archive_path):
-        logger.error(f'dar backup: "{archive_path}" not found, exiting')
-        sys.exit(1)
+    archive_path = os.path.join(directory, f'{archive}')
+
+    archive_test_path =  os.path.join(directory, f'{archive}.1.dar')
+    if not os.path.exists(archive_test_path):
+        logger.error(f'dar backup: "{archive_test_path}" not found, exiting')
+        return 1
         
     # sanity check - does backup definition exist?
     backup_definition = archive.split('_')[0]
     backup_def_path = os.path.join(config_settings.backup_d_dir, backup_definition)
     if not os.path.exists(backup_def_path):
         logger.error(f'backup definition "{backup_definition}" not found (--add-specific-archive option probably not correct), exiting')
-        sys.exit(1)
+        return 1
     
     database = f"{backup_definition}{DB_SUFFIX}"
     database_path = os.path.realpath(os.path.join(config_settings.backup_dir, database))
-    logger.info(f'Add "{archive_path}" to catalog "{database}"')
+    logger.info(f'Add "{archive_path}" to catalog: "{database}"')
     
     command = ['dar_manager', '--base', database_path, "--add", archive_path, "-ai", "-Q"]
     process = run_command(command)
@@ -122,7 +210,99 @@ def add_specific_archive(archive: str, config_settings: ConfigSettings):
         logger.error(f"stderr: {stderr}")
         logger.error(f"stdout: {stdout}")
      
-    sys.exit(process.returncode)
+    return process.returncode
+
+
+
+def add_directory(args: argparse.ArgumentParser, config_settings: ConfigSettings) -> None:
+    """
+    Loop over the DAR archives in the given directory args.add_dir in increasing order by date and add them to their catalog database.
+
+    Args:
+        args (argparse.ArgumentParser): The command-line arguments object containing the add_dir attribute.
+        config_settings (ConfigSettings): The configuration settings object.
+
+    This function performs the following steps:
+    1. Checks if the specified directory exists. If not, raises a RuntimeError.
+    2. Uses a regular expression to match DAR archive files with base names in the format <string>_{FULL, DIFF, INCR}_YYYY-MM-DD.
+    3. Lists the DAR archives in the specified directory and extracts their base names and dates.
+    4. Sorts the DAR archives by date.
+    5. Loops over the sorted DAR archives and adds each archive to its catalog database using the add_specific_archive function.
+
+    Example:
+        args = argparse.ArgumentParser()
+        args.add_dir = '/path/to/dar/archives'
+        config_settings = ConfigSettings()
+        add_directory(args, config_settings)
+    """
+    if not os.path.exists(args.add_dir):
+        raise RuntimeError(f"Directory {args.add_dir} does not exist")
+
+    # Regular expression to match DAR archive files with base name and date in the format <string>_{FULL, DIFF, INCR}_YYYY-MM-DD
+    #dar_pattern = re.compile(r'^(.*?_(FULL|DIFF|INCR)_(\d{4}-\d{2}-\d{2}))\.\d+\.dar$')
+    dar_pattern = re.compile(r'^(.*?_(FULL|DIFF|INCR)_(\d{4}-\d{2}-\d{2}))\.1.dar$') # just read slice #1 of an archive
+    # List of DAR archives with their dates and base names
+    dar_archives = []
+
+    for filename in os.listdir(args.add_dir):
+        logger.debug(f"check if '{filename}' is a dar archive slice #1?")
+        match = dar_pattern.match(filename)
+        if match:
+            base_name = match.group(1)
+            date_str = match.group(3)
+            date_obj = datetime.strptime(date_str, '%Y-%m-%d')
+            dar_archives.append((date_obj, base_name))
+            logger.debug(f" -> yes: base name: {base_name}, date: {date_str}")
+
+    if not dar_archives or len(dar_archives) == 0:
+        logger.info(f"No 'dar' archives found in directory {args.add_dir}")
+        return
+
+    # Sort the DAR archives by date
+    dar_archives.sort()
+
+    # Loop over the sorted DAR archives and process them
+    result: List[Dict] = []
+    for date_obj, base_name in dar_archives:
+        logger.info(f"Adding dar archive: '{base_name}' to it's catalog database")
+        result_archive = add_specific_archive(base_name, config_settings, args.add_dir)
+        result.append({ f"{base_name}" : result_archive})
+        if result_archive != 0:
+            logger.error(f"Something went wrong added {base_name} to it's catalog")
+    
+    logger.debug(f"Results adding archives found in: '{args.add_dir}': result")
+
+
+def backup_def_from_archive(archive: str) -> str:
+    """
+    return the backup definition from archive name
+    """
+    search = re.search("(.*?)_", archive)
+    backup_def = search.group(1)
+    logger.debug(f"backup definition: '{backup_def}' from given archive '{archive}'")
+    return backup_def
+
+
+
+def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> int:
+    backup_def = backup_def_from_archive(archive)
+    database_path = os.path.join(config_settings.backup_dir, f"{backup_def}{DB_SUFFIX}")
+    cat_no = cat_no_for_name(archive, config_settings)
+    if cat_no > 0:
+        command = ['dar_manager', '--base', database_path, "--delete", str(cat_no)]
+        process = run_command(command)
+    else:
+        logger.error(f"archive: '{archive}' not found in in't catalog database: {database_path}")
+        return cat_no
+
+    if process.returncode == 0:
+        logger.info(f"'{archive}' removed from it's catalog")
+    else:
+        logger.error(process.stdout)
+        logger.error(process.sterr)
+
+    return process.returncode
+
 
 
 
@@ -142,7 +322,9 @@ def main():
     parser.add_argument('-d', '--backup-def', type=str, help='Restrict to work only on this backup definition')
     parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog database')
     parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog database')
-    parser.add_argument('--list-db', action='store_true', help='List catalogs in databases')
+    parser.add_argument('--list-catalog', action='store_true', help='List catalogs in databases for all backup definitions')
+    parser.add_argument('--list-catalog-contents', type=int, help="List contents of a catalog. Argument is the 'archive #', '-d <definition>' argument is also required")
+    parser.add_argument('--find-file', type=str, help="List catalogs containing <path>/file. '-d <definition>' argument is also required")
     parser.add_argument('--verbose', action='store_true', help='Be more verbose')
     parser.add_argument('--log-level', type=str, help="`debug` or `trace`, default is `info`", default="info")
     parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
@@ -196,6 +378,13 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
         logger.error("you can't add and remove archives in the same operation, exiting")
         sys.exit(1)
 
+    if args.add_dir and args.add_specific_archive:
+        logger.error("you cannot add both a directory and an archive")
+        sys.exit(1)
+
+    if args.backup_def and not args.backup_def.strip():
+        logger.error(f"No backup definition given to --backup-def")
+
     if args.backup_def:
         backup_def_path = os.path.join(config_settings.backup_d_dir, args.backup_def)
         if not os.path.exists(backup_def_path):
@@ -203,6 +392,16 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
             sys.exit(1)
 
 
+    if args.list_catalog_contents and not args.backup_def:
+        logger.error(f"--list-catalog-contents requires the --backup-def, exiting")
+        sys.exit(1)
+
+    if args.find_file and not args.backup_def:
+        logger.error(f"--find-file requires the --backup-def, exiting")
+        sys.exit(1)
+    
+
+
     # Modify config settings based on the arguments
     if args.alternate_archive_dir:
         if not os.path.exists(args.alternate_archive_dir):
@@ -213,35 +412,49 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
 
     if args.create_db:
         if args.backup_def:
-            create_db(args.backup_def, config_settings)
+            sys.exit(create_db(args.backup_def, config_settings))
         else:
             for root, dirs, files in os.walk(config_settings.backup_d_dir):
                 for file in files:
                     current_backupdef = os.path.basename(file)
-                    create_db(current_backupdef, config_settings)
-        sys.exit(0)
+                    logger.debug(f"Create catalog db for backup definition: '{current_backupdef}'")
+                    result = create_db(current_backupdef, config_settings)
+                    if result != 0:
+                        sys.exit(result)
 
     if args.add_specific_archive:
-        add_specific_archive(args.add_specific_archive, config_settings)
+        sys.exit(add_specific_archive(args.add_specific_archive, config_settings))
 
     if args.add_dir:
-        # Implement add directory logic
-        pass
+        sys.exit(add_directory(args, config_settings))
+
 
     if args.remove_specific_archive:
-        # Implement remove specific archive logic
-        pass
+        sys.exit(remove_specific_archive(args.remove_specific_archive, config_settings))
+
 
 
-    if args.list_db:
+    if args.list_catalog:
         if args.backup_def:
-            list_db(args.backup_def, config_settings)
+            process = list_catalogs(args.backup_def, config_settings)
+            result = process.returncode
         else:
+            result = 0
             for root, dirs, files in os.walk(config_settings.backup_d_dir):
                 for file in files:
                     current_backupdef = os.path.basename(file)
-                    list_db(current_backupdef, config_settings)
-        sys.exit(0)
+                    if list_catalogs(current_backupdef, config_settings).returncode != 0:
+                        result = 1
+        sys.exit(result)
+
+    if args.list_catalog_contents:
+        result = list_catalog_contents(args.list_catalog_contents, args.backup_def, config_settings)
+        sys.exit(result)
+
+    if args.find_file:
+        result = find_file(args.find_file, args.backup_def, config_settings)
+        sys.exit(result)
+
 
 if __name__ == "__main__":
     main()

{dar_backup-0.6.1.dist-info → dar_backup-0.6.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dar-backup
-Version: 0.6.1
+Version: 0.6.3
 Summary: A script to do full, differential and incremental backups using dar.     Some files are restored from the backups during verification, after which par2 redundancy files are created.     The script also has a cleanup feature to remove old backups and par2 files.
 Project-URL: Homepage, https://github.com/per2jensen/dar-backup
 Project-URL: Issues, https://github.com/per2jensen/dar-backup/issues

dar_backup-0.6.3.dist-info/RECORD

@@ -0,0 +1,13 @@
+dar_backup/.darrc,sha256=ggex9N6eETOS6u003_QRRJMeWbveQfkT1lDBt0XpU-I,2112
+dar_backup/__about__.py,sha256=_SsQ0ZcyZbUqlFFT370nQxs8UER9D0oW_EmCr4Q-hx4,21
+dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dar_backup/cleanup.py,sha256=DgmxSUwKrLLIQuYSIY_yTRhIuOMgI6ivjlQuH4u3wX4,9057
+dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
+dar_backup/dar_backup.py,sha256=YZoVu9NJX3_WIkQIG8EMLSK3-VWdslI0c2XKrM2Un38,32214
+dar_backup/manager.py,sha256=Y1dQ7CRGQx5sGoGrjAO62QfJcuW_0Vod-ZGaQmHInFU,19062
+dar_backup/util.py,sha256=6lPCFHr3MDdaLWAW9EDMZ4jdL7pt8rki-5dOXcesmP8,8955
+dar_backup-0.6.3.dist-info/METADATA,sha256=6120hSt2mUxhn3u8PfbJweoPY4ToWFlAPHAgLozaK7Q,22496
+dar_backup-0.6.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dar_backup-0.6.3.dist-info/entry_points.txt,sha256=x9vnW-JEl8mpDJC69f_XBcn0mBSkV1U0cyvFV-NAP1g,126
+dar_backup-0.6.3.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dar_backup-0.6.3.dist-info/RECORD,,

dar_backup-0.6.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-dar_backup/.darrc,sha256=ggex9N6eETOS6u003_QRRJMeWbveQfkT1lDBt0XpU-I,2112
-dar_backup/__about__.py,sha256=XvHFZM0padtrqitt9-p2enlBUGqc6vGvWNLx2iJv09g,21
-dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dar_backup/cleanup.py,sha256=DgmxSUwKrLLIQuYSIY_yTRhIuOMgI6ivjlQuH4u3wX4,9057
-dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
-dar_backup/dar_backup.py,sha256=xnDuqkpo8q_X1FLKID9I_s__FQozl5EIrwNHdIJAd_0,37525
-dar_backup/manager.py,sha256=lkw1ZAIdxY7WedLPKZMnHpuq_QbjkUdcG61ooiwUYpo,10197
-dar_backup/util.py,sha256=6lPCFHr3MDdaLWAW9EDMZ4jdL7pt8rki-5dOXcesmP8,8955
-dar_backup-0.6.1.dist-info/METADATA,sha256=m3P_iVauaMno0fAhc4iJgO_DS9FobsqV5StQzUdU6wY,22496
-dar_backup-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dar_backup-0.6.1.dist-info/entry_points.txt,sha256=x9vnW-JEl8mpDJC69f_XBcn0mBSkV1U0cyvFV-NAP1g,126
-dar_backup-0.6.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dar_backup-0.6.1.dist-info/RECORD,,