dar-backup 0.6.1__py3-none-any.whl → 0.6.2__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.1"
+__version__ = "0.6.2"

dar_backup/dar_backup.py

@@ -17,56 +17,49 @@ from argparse import ArgumentParser
 from datetime import datetime
 from pathlib import Path
 from time import time
+from typing import List
 
 from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import list_backups
 from dar_backup.util import run_command
 from dar_backup.util import setup_logging
-from dar_backup.util import extract_error_lines
 from dar_backup.util import BackupError
-from dar_backup.util import DifferentialBackupError
-from dar_backup.util import IncrementalBackupError
 from dar_backup.util import RestoreError
 
 
 logger = None
 
-
-def backup(backup_file: str, backup_definition: str, darrc: str,  config_settings: ConfigSettings):
+def generic_backup(type: str, command: List[str], backup_file: str, backup_definition: str, darrc: str,  config_settings: ConfigSettings):
     """
-    Performs a full backup using the 'dar' command.
+    Performs a backup using the 'dar' command.
 
     This function initiates a full backup operation by constructing and executing a command
     with the 'dar' utility. It checks if the backup file already exists to avoid overwriting
     previous backups. If the backup file does not exist, it proceeds with the backup operation.
 
     Args:
+        type (str): The type of backup (FULL, DIFF, INCR).
+        command (List[str]): The command to execute for the backup operation.
         backup_file (str): The base name of the backup file. The actual backup will be saved
                            as '{backup_file}.1.dar'.
         backup_definition (str): The path to the backup definition file. This file contains
                                  specific instructions for the 'dar' utility, such as which
                                  directories to include or exclude.
+        darrc (str): The path to the '.darrc' configuration file.
+        config_settings (ConfigSettings): An instance of the ConfigSettings class.
+
 
-    Note:
-        This function logs an error and returns early if the backup file already exists.
-        It logs the command being executed and reports upon successful completion of the backup.
 
     Raises:
         BackupError: If an error occurs during the backup process.
     """
-    if os.path.exists(backup_file + '.1.dar'):
-        logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
-        return
-
-    logger.info(f"===> Starting FULL backup for {backup_definition}")
-    command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition, '-Q', "compress-exclusion",  "verbose"]
+    logger.info(f"===> Starting {type} backup for {backup_definition}")
     logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
     try:
         process = run_command(command, config_settings.command_timeout_secs)
-        logger.info("Back from run_command")
         if process.returncode == 0:
-            logger.info("FULL backup completed successfully.")
+            logger.info(f"{type} backup completed successfully.")
         elif process.returncode == 5:
             logger.warning("Backup completed with some files not backed up, this can happen if files are changed/deleted during the backup.")
         else:
@@ -78,106 +71,6 @@ def backup(backup_file: str, backup_definition: str, darrc: str,  config_setting
         logger.exception(f"Unexpected error during backup")
         raise BackupError(f"Unexpected error during backup: {e}") from e
  
- 
-
-
-
-def differential_backup(backup_file: str, backup_definition: str, base_backup_file: str, darrc: str, config_settings: ConfigSettings):
-    """
-    Creates a differential backup based on a specified base backup.
-
-    This function performs a differential backup by comparing the current state of the data
-    against a specified base backup file. It captures only the changes made since that base
-    backup, resulting in a smaller and faster backup process compared to a full backup.
-
-    Args:
-        backup_file (str): The base name for the differential backup file. The actual backup
-                           will be saved as '{backup_file}.1.dar'.
-        backup_definition (str): The path to the backup definition file. This file contains
-                                 specific instructions for the 'dar' utility, such as which
-                                 directories to include or exclude.
-        base_backup_file (str): The base name of the full backup file that serves as the
-                                reference point for the differential backup.
-
-    Note:
-        This function logs an error and returns early if the differential backup file already exists.
-        It logs the command being executed and reports upon successful completion of the differential backup.
-
-    Raises:
-        DifferentialBackupError: If the differential backup command fails or encounters an unexpected error.
-    """
-    if os.path.exists(backup_file + '.1.dar'):
-        logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
-        return
-
-    logger.info(f"===> Starting DIFF backup for {backup_definition}")
-    command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition, '-A', base_backup_file, '-Q', "compress-exclusion",  "verbose"]
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
-    try:
-        process = run_command(command)
-        if process.returncode == 0:
-            logger.info("DIFF backup completed successfully.")
-        elif process.returncode == 5:
-            logger.warning("Backup completed with some files not backed up, this can happen if files are changed/deleted during the backup.")
-        else:
-            raise Exception(str(process))
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Differential backup command failed: {e}")
-        raise DifferentialBackupError(f"Differential backup command failed: {e}") from e
-    except Exception as e:
-        logger.exception(f"Unexpected error during differential backup")
-        logger.error("Exception details:", exc_info=True)
-        raise DifferentialBackupError(f"Unexpected error during differential backup: {e}") from e
-
-
-def incremental_backup(backup_file: str, backup_definition: str, last_backup_file: str, darrc: str, config_settings: ConfigSettings):
-    """
-    Creates an incremental backup based on the last backup file.
-
-    This function performs an incremental backup by comparing the current state of the data
-    against the last backup file, whether it's a full backup or the most recent incremental backup.
-    It captures only the changes made since that last backup, making it efficient for frequent
-    backups with minimal data changes.
-
-    Args:
-        backup_file (str): The base name for the incremental backup file. The actual backup
-                           will be saved with a unique identifier to distinguish it from other backups.
-        backup_definition (str): The path to the backup definition file. This file contains
-                                 specific instructions for the 'dar' utility, such as which
-                                 directories to include or exclude.
-        last_backup_file (str): The base name of the last backup file (full or incremental) that
-                                serves as the reference point for the incremental backup.
-
-    Note:
-        This function checks if the incremental backup file already exists to prevent overwriting
-        previous backups. It logs the command being executed and reports upon successful completion
-        of the incremental backup.
-    
-    Raises:
-        IncrementalBackupError: If the incremental backup command fails or an unexpected error occurs.
-    """
-    if os.path.exists(backup_file + '.1.dar'):
-        logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
-        return
-
-    logger.info(f"===> Starting INCR backup for {backup_definition}")
-    command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition, '-A', last_backup_file, '-Q', "compress-exclusion",  "verbose"]
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
-    try:
-        process = run_command(command, config_settings.command_timeout_secs)
-        if process.returncode == 0:
-            logger.info("INCR backup completed successfully.")
-        elif process.returncode == 5:
-            logger.warning("Backup completed with some files not backed up, this can happen if files are changed/deleted during the backup.")
-        else:
-            raise Exception(str(process))
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Incremental backup command failed: {e}")
-        raise IncrementalBackupError(f"Incremental backup command failed: {e}") from e
-    except Exception as e:
-        logger.exception(f"Unexpected error during incremental backup")
-        raise IncrementalBackupError(f"Unexpected error during incremental backup: {e}") from e
-
 
 # Function to recursively find <File> tags and build their full paths
 def find_files_with_paths(element: ET, current_path=""):
@@ -331,7 +224,7 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
     Restores a backup file to a specified directory.
 
     Args:
-        backup_name (str): The name of the backup file.
+        backup_name (str): The base name of the backup file, without the "slice number.dar"
         backup_dir (str): The directory where the backup file is located.
         restore_dir (str): The directory where the backup should be restored to.
         selection (str, optional): A selection criteria to restore specific files or directories. Defaults to None.
@@ -342,16 +235,19 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
         if not os.path.exists(restore_dir):
             os.makedirs(restore_dir)
         command.extend(['-R', restore_dir])
+    else:
+        raise RestoreError("Restore directory ('-R <dir>') not specified")
     if selection:
         selection_criteria = shlex.split(selection)
         command.extend(selection_criteria)
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
+    logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
     try:
         process = run_command(command, config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"Restore completed successfully to: '{restore_dir}'")
         else:
-            raise Exception(str(process))
+            logger.error(f"Restore command failed: \n ==> stdout: {process.stdout}, \n ==> stderr: {process.stderr}")
+            raise RestoreError(str(process))
     except subprocess.CalledProcessError as e:
         raise RestoreError(f"Restore command failed: {e}") from e
     except Exception as e:
@@ -405,6 +301,7 @@ def list_contents(backup_name, backup_dir, selection=None):
         None
     """
     backup_path = os.path.join(backup_dir, backup_name)
+
     try:
         command = ['dar', '-l', backup_path, '-am', '-as', '-Q']
         if selection:
@@ -427,6 +324,32 @@ def list_contents(backup_name, backup_dir, selection=None):
 
 
 
+
+def create_backup_command(backup_type: str, backup_file: str, darrc: str, backup_definition_path: str, latest_base_backup: str = None) -> List[str]:
+    """
+    Generate the backup command for the specified backup type.
+
+    Args:
+        backup_type (str): The type of backup (FULL, DIFF, INCR).
+        backup_file (str): The backup file path. Example: /path/to/example_2021-01-01_FULL
+        darrc (str): Path to the .darrc configuration file.
+        backup_definition_path (str): Path to the backup definition file.
+        latest_base_backup (str, optional): Path to the latest base backup for DIFF or INCR types.
+
+    Returns:
+        List[str]: The constructed backup command.
+    """
+    base_command = ['dar', '-c', backup_file, "-N", '-B', darrc, '-B', backup_definition_path, '-Q', "compress-exclusion", "verbose"]
+    
+    if backup_type in ['DIFF', 'INCR']:
+        if not latest_base_backup:
+            raise ValueError(f"Base backup is required for {backup_type} backups.")
+        base_command.extend(['-A', latest_base_backup])
+    
+    return base_command
+
+
+
 def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, backup_type: str):
     """
     Perform backup operation.
@@ -434,32 +357,12 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
     Args:
         args: Command-line arguments.
         config_settings: An instance of the ConfigSettings class.
-        backup_d: Directory containing backup definitions.
-        backup_dir: Directory to store backup files.
-        test_restore_dir: Directory to test restore backup files.
         backup_type: Type of backup (FULL, DIFF, INCR).
-        min_size_verification_mb: Minimum size for verification in MB.
-        max_size_verification_mb: Maximum size for verification in MB.
-        no_files_verification: Flag indicating whether to skip file verification.
-
-    Returns:
-        None
-
-    Raises:
-        FileNotFoundError: If `backup_d` does not exist or a specified backup definition file does not exist.
-        PermissionError: If there is insufficient permission to access directories or files specified.
-        OSError: For various system-related errors, such as exhaustion of file descriptors.
-        ValueError: If there is an issue with the format string in `datetime.now().strftime`.
-        subprocess.CalledProcessError: If a subprocess invoked during the backup process exits with a non-zero status.
-        Exception: Catches any unexpected exceptions that may occur during the backup process.
-
-    Note: 
-      This function assumes that any exceptions raised by the `backup` function or related subprocesses are handled
-      within those functions or propagated up to be handled by the caller of `perform_backup`.
     """
     logger.debug(f"perform_backup({backup_type}) started")
     backup_definitions = []
 
+    # Gather backup definitions
     if args.backup_definition:
         if '_' in args.backup_definition:
             logger.error(f"Skipping backup definition: '{args.backup_definition}' due to '_' in name")
@@ -482,16 +385,15 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                 logger.error(f"Backup file {backup_file}.1.dar already exists. Skipping backup.")
                 continue
 
-            if backup_type == 'FULL':
-                backup(backup_file, backup_definition_path, args.darrc, config_settings)
-            else:
+            latest_base_backup = None
+            if backup_type in ['DIFF', 'INCR']:
                 base_backup_type = 'FULL' if backup_type == 'DIFF' else 'DIFF'
-                
+
                 if args.alternate_reference_archive:
-                    latest_base_backup = os.path.join(config_settings.backup_dir, args.alternate_reference_archive)  # expects alternerate reference archive to be without slice number
+                    latest_base_backup = os.path.join(config_settings.backup_dir, args.alternate_reference_archive)
                     logger.info(f"Using alternate reference archive: {latest_base_backup}")
                     if not os.path.exists(latest_base_backup + '.1.dar'):
-                        logger.error(f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exciting.")
+                        logger.error(f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exiting.")
                         sys.exit(1)
                 else:
                     base_backups = sorted(
@@ -503,33 +405,36 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                         continue
                     latest_base_backup = os.path.join(config_settings.backup_dir, base_backups[-1].rsplit('.', 2)[0])
 
-                if backup_type == 'DIFF':
-                    differential_backup(backup_file, backup_definition_path, latest_base_backup, args.darrc, config_settings)
-                elif backup_type == 'INCR':
-                    incremental_backup(backup_file, backup_definition_path, latest_base_backup, args.darrc, config_settings) 
+            # Generate the backup command
+            command = create_backup_command(backup_type, backup_file, args.darrc, backup_definition_path, latest_base_backup)
+
+            # Perform backup
+            generic_backup(backup_type, command, backup_file, backup_definition_path, args.darrc, config_settings)
 
             logger.info("Starting verification...")
             result = verify(args, backup_file, backup_definition_path, config_settings)
             if result:
                 logger.info("Verification completed successfully.")
             else:
-                logger.error("Verification failed.") 
+                logger.error("Verification failed.")
+
             if config_settings.par2_enabled:
-                logger.info("Generate par2 redundancy files") 
-                generate_par2_files(backup_file, config_settings)  # do this even if verification failed, because verification could fail on an open file.
+                logger.info("Generate par2 redundancy files.")
+                generate_par2_files(backup_file, config_settings, args)
                 logger.info("par2 files completed successfully.")
-        # we want to continue with other backup definitions, thus only logging an error
         except Exception as e:
-            logger.exception(f"Error during {backup_type} backup process, continuing on next backup definition")
-            logger.error("Exception details:", exc_info=True)
+            logger.exception(f"Error during {backup_type} backup process, continuing to next backup definition.")
+
+
 
-def generate_par2_files(backup_file: str, config_settings: ConfigSettings):
+def generate_par2_files(backup_file: str, config_settings: ConfigSettings, args):
     """
     Generate PAR2 files for a given backup file in the specified backup directory.
 
     Args:
         backup_file (str): The name of the backup file.
-        backup_dir (str): The path to the backup directory.
+        config_settings: The configuration settings object.
+        args: The command-line arguments object.
 
     Raises:
         subprocess.CalledProcessError: If the par2 command fails to execute.
@@ -537,19 +442,39 @@ def generate_par2_files(backup_file: str, config_settings: ConfigSettings):
     Returns:
         None
     """
+    # Regular expression to match DAR slice files
+    dar_slice_pattern = re.compile(rf"{re.escape(os.path.basename(backup_file))}\.([0-9]+)\.dar")
+
+    # List of DAR slice files to be processed
+    dar_slices: List[str] = []
+
     for filename in os.listdir(config_settings.backup_dir):
-        if os.path.basename(backup_file) in filename:
-            # Construct the full path to the file
-            file_path = os.path.join(config_settings.backup_dir, filename)
-            # Run the par2 command to generate redundancy files with error correction
-            command = ['par2', 'create', f'-r{config_settings.error_correction_percent}', '-q', '-q', file_path]
-            process = run_command(command, config_settings.command_timeout_secs)
-            if process.returncode != 0:
-                logger.error(f"Error generating par2 files for {file_path}")
-                raise subprocess.CalledProcessError(process.returncode, command)
-            logger.debug(f"par2 files generated for {file_path}")
+        match = dar_slice_pattern.match(filename)
+        if match:
+            dar_slices.append(filename)
 
+    # Sort the DAR slices based on the slice number
+    dar_slices.sort(key=lambda x: int(dar_slice_pattern.match(x).group(1)))
+    number_of_slices = len(dar_slices)
+    counter = 1
 
+    for slice_file in dar_slices:
+        file_path = os.path.join(config_settings.backup_dir, slice_file)
+    
+        if args.verbose or args.log_level == "debug" or args.log_level == "trace":
+            logger.info(f"{counter}/{number_of_slices}: Now generating par2 files for {file_path}")
+
+        # Run the par2 command to generate redundancy files with error correction
+        command = ['par2', 'create', f'-r{config_settings.error_correction_percent}', '-q', '-q', file_path]
+        process = run_command(command, config_settings.command_timeout_secs)
+
+        if process.returncode == 0:
+            if args.verbose or args.log_level == "debug" or args.log_level == "trace":
+                logger.info(f"{counter}/{number_of_slices}: Done")
+        else:
+            logger.error(f"Error generating par2 files for {file_path}")
+            raise subprocess.CalledProcessError(process.returncode, command)
+        counter += 1
 
 
 
@@ -658,8 +583,9 @@ def main():
     parser.add_argument('-l', '--list', action='store_true', help="List available archives.")
     parser.add_argument('--list-contents', help="List the contents of the specified archive.")
     parser.add_argument('--selection', help="dar file selection for listing/restoring specific files/directories.")
-    parser.add_argument('-r', '--restore', nargs=1, type=str, help="Restore specified archive.")
-    parser.add_argument('--restore-dir', nargs=1, type=str, help="Directory to restore files to.")
+#    parser.add_argument('-r', '--restore', nargs=1, type=str, help="Restore specified archive.")
+    parser.add_argument('-r', '--restore', type=str, help="Restore specified archive.")
+    parser.add_argument('--restore-dir',   type=str, help="Directory to restore files to.")
     parser.add_argument('--verbose', action='store_true', help="Print various status messages to screen")
     parser.add_argument('--log-level', type=str, help="`debug` or `trace`", default="info")
     parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
@@ -708,7 +634,10 @@ def main():
             args.verbose and (print(f"Alternate ref archive: {args.alternate_reference_archive}"))
         args.verbose and (print(f"Backup.d dir:      {config_settings.backup_d_dir}"))
         args.verbose and (print(f"Backup dir:        {config_settings.backup_dir}"))
-        args.verbose and (print(f"Test restore dir:  {config_settings.test_restore_dir}"))
+
+        restore_dir = args.restore_dir if args.restore_dir else config_settings.test_restore_dir
+        args.verbose and (print(f"Restore dir:       {restore_dir}"))
+
         args.verbose and (print(f"Logfile location:  {config_settings.logfile_location}"))
         args.verbose and (print(f".darrc location:   {args.darrc}"))
         args.verbose and (print(f"PAR2 enabled:      {config_settings.par2_enabled}"))
@@ -735,10 +664,9 @@ def main():
         elif args.incremental_backup  and not args.full_backup and not args.differential_backup:
             perform_backup(args, config_settings, "INCR")
         elif args.list_contents:
-            print(f"Listing contents of {args.list_contents}")
             list_contents(args.list_contents, config_settings.backup_dir, args.selection)
         elif args.restore:
-            restore_dir = args.restore_dir if args.restore_dir else config_settings.test_restore_dir
+            logger.debug(f"Restoring {args.restore} to {restore_dir}")
             restore_backup(args.restore, config_settings, restore_dir, args.selection)
         else:
             parser.print_help()

{dar_backup-0.6.1.dist-info → dar_backup-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dar-backup
-Version: 0.6.1
+Version: 0.6.2
 Summary: A script to do full, differential and incremental backups using dar.     Some files are restored from the backups during verification, after which par2 redundancy files are created.     The script also has a cleanup feature to remove old backups and par2 files.
 Project-URL: Homepage, https://github.com/per2jensen/dar-backup
 Project-URL: Issues, https://github.com/per2jensen/dar-backup/issues

{dar_backup-0.6.1.dist-info → dar_backup-0.6.2.dist-info}/RECORD

@@ -1,13 +1,13 @@
 dar_backup/.darrc,sha256=ggex9N6eETOS6u003_QRRJMeWbveQfkT1lDBt0XpU-I,2112
-dar_backup/__about__.py,sha256=XvHFZM0padtrqitt9-p2enlBUGqc6vGvWNLx2iJv09g,21
+dar_backup/__about__.py,sha256=d7NGuoje3vHyudKIFR_PmfKozIOKDFvAhGx0QXiyuMw,21
 dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dar_backup/cleanup.py,sha256=DgmxSUwKrLLIQuYSIY_yTRhIuOMgI6ivjlQuH4u3wX4,9057
 dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
-dar_backup/dar_backup.py,sha256=xnDuqkpo8q_X1FLKID9I_s__FQozl5EIrwNHdIJAd_0,37525
+dar_backup/dar_backup.py,sha256=YZoVu9NJX3_WIkQIG8EMLSK3-VWdslI0c2XKrM2Un38,32214
 dar_backup/manager.py,sha256=lkw1ZAIdxY7WedLPKZMnHpuq_QbjkUdcG61ooiwUYpo,10197
 dar_backup/util.py,sha256=6lPCFHr3MDdaLWAW9EDMZ4jdL7pt8rki-5dOXcesmP8,8955
-dar_backup-0.6.1.dist-info/METADATA,sha256=m3P_iVauaMno0fAhc4iJgO_DS9FobsqV5StQzUdU6wY,22496
-dar_backup-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dar_backup-0.6.1.dist-info/entry_points.txt,sha256=x9vnW-JEl8mpDJC69f_XBcn0mBSkV1U0cyvFV-NAP1g,126
-dar_backup-0.6.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dar_backup-0.6.1.dist-info/RECORD,,
+dar_backup-0.6.2.dist-info/METADATA,sha256=PMfP8NhPwzhHRbjJt6_7Unqxbqqj13PW19QdYac6pcc,22496
+dar_backup-0.6.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dar_backup-0.6.2.dist-info/entry_points.txt,sha256=x9vnW-JEl8mpDJC69f_XBcn0mBSkV1U0cyvFV-NAP1g,126
+dar_backup-0.6.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dar_backup-0.6.2.dist-info/RECORD,,