dar-backup 0.6.16__py3-none-any.whl → 0.6.17__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.16"
+__version__ = "0.6.17"

dar_backup/cleanup.py

@@ -29,14 +29,14 @@ from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import extract_error_lines
 from dar_backup.util import list_backups
-from dar_backup.util import run_command
 from dar_backup.util import setup_logging
+from dar_backup.util import get_logger
 
-from dar_backup.util import CommandResult   
-
-
+from dar_backup.command_runner import CommandRunner   
+from dar_backup.command_runner import CommandResult
 
 logger = None 
+runner = None
 
 def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=None):
     """
@@ -134,7 +134,7 @@ def delete_catalog(catalog_name: str, args: NamedTuple) -> bool:
     command = [f"manager", "--remove-specific-archive", catalog_name, "--config-file", args.config_file, '--log-level', 'debug', '--log-stdout']
     logger.info(f"Deleting catalog '{catalog_name}' using config file: '{args.config_file}'")
     try:
-        result:CommandResult = run_command(command)
+        result:CommandResult = runner.run(command)
         if result.returncode == 0:
             logger.info(f"Deleted catalog '{catalog_name}', using config file: '{args.config_file}'")
             logger.debug(f"Stdout: manager.py --remove-specific-archive output:\n{result.stdout}")
@@ -159,7 +159,7 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
 
 
 def main():
-    global logger
+    global logger, runner
 
     parser = argparse.ArgumentParser(description="Cleanup old archives according to AGE configuration.")
     parser.add_argument('-d', '--backup-definition', help="Specific backup definition to cleanup.")
@@ -188,7 +188,8 @@ def main():
 #    command_output_log = os.path.join(config_settings.logfile_location.removesuffix("dar-backup.log"), "dar-backup-commands.log")
     command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
     logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout)
-
+    command_logger = get_logger(command_output_logger = True)
+    runner = CommandRunner(logger=logger, command_logger=command_logger)
 
     logger.info(f"=====================================")
     logger.info(f"cleanup.py started, version: {about.__version__}")
@@ -283,15 +284,6 @@ def main():
     end_time=int(time())
     logger.info(f"END TIME: {end_time}")
 
-#    error_lines = extract_error_lines(config_settings.logfile_location, start_time, end_time)
-#    if len(error_lines) > 0:
-#        args.verbose and print("\033[1m\033[31mErrors\033[0m encountered")
-#        for line in error_lines:
-#            args.verbose and print(line)
-#        sys.exit(1)
-#    else:
-#        args.verbose and print("\033[1m\033[32mSUCCESS\033[0m No errors encountered")
-#        sys.exit(0)
 
 if __name__ == "__main__":
     main()

dar_backup/command_runner.py

@@ -0,0 +1,81 @@
+import subprocess
+import logging
+import threading
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src")))
+from typing import List, Optional
+
+class CommandResult:
+    def __init__(self, returncode: int, stdout: str, stderr: str):
+        self.returncode = returncode
+        self.stdout = stdout
+        self.stderr = stderr
+
+    def __repr__(self):
+        return f"<CommandResult returncode={self.returncode}>"
+
+class CommandRunner:
+    def __init__(
+        self,
+        logger: Optional[logging.Logger] = None,
+        command_logger: Optional[logging.Logger] = None,
+        default_timeout: int = 30
+    ):
+        self.logger = logger or logging.getLogger(__name__)
+        self.command_logger = command_logger or self.logger
+        self.default_timeout = default_timeout
+
+    def run(
+        self,
+        cmd: List[str],
+        *,
+        timeout: Optional[int] = None,
+        check: bool = False,
+        capture_output: bool = True,
+        text: bool = True
+    ) -> CommandResult:
+        timeout = timeout or self.default_timeout
+        self.logger.debug(f"Executing command: {' '.join(cmd)} (timeout={timeout}s)")
+
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE if capture_output else None,
+            stderr=subprocess.PIPE if capture_output else None,
+            text=text,
+            bufsize=1
+        )
+
+        stdout_lines = []
+        stderr_lines = []
+
+        def stream_output(stream, lines, level):
+            for line in iter(stream.readline, ''):
+                lines.append(line)
+                self.command_logger.log(level, line.strip())
+            stream.close()
+
+        threads = []
+        if capture_output and process.stdout:
+            t_out = threading.Thread(target=stream_output, args=(process.stdout, stdout_lines, logging.INFO))
+            t_out.start()
+            threads.append(t_out)
+        if capture_output and process.stderr:
+            t_err = threading.Thread(target=stream_output, args=(process.stderr, stderr_lines, logging.ERROR))
+            t_err.start()
+            threads.append(t_err)
+
+        try:
+            process.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            self.logger.error(f"Command timed out: {' '.join(cmd)}")
+            return CommandResult(-1, ''.join(stdout_lines), ''.join(stderr_lines))
+
+        for t in threads:
+            t.join()
+
+        if check and process.returncode != 0:
+            self.logger.error(f"Command failed with exit code {process.returncode}")
+
+        return CommandResult(process.returncode, ''.join(stdout_lines), ''.join(stderr_lines))

dar_backup/config_settings.py

@@ -63,7 +63,13 @@ class ConfigSettings:
             self.diff_age = int(self.config['AGE']['DIFF_AGE'])
             self.incr_age = int(self.config['AGE']['INCR_AGE'])
             self.error_correction_percent = int(self.config['PAR2']['ERROR_CORRECTION_PERCENT'])
-            self.par2_enabled = self.config['PAR2']['ENABLED'].lower() in ('true', '1', 'yes')
+            val = self.config['PAR2']['ENABLED'].strip().lower()
+            if val in ('true', '1', 'yes'):
+                self.par2_enabled = True
+            elif val in ('false', '0', 'no'):
+                self.par2_enabled = False
+            else:
+                raise ValueError(f"Invalid boolean value for 'ENABLED' in [PAR2]: '{val}'")
             
             # Ensure the directories exist
             Path(self.backup_dir).mkdir(parents=True, exist_ok=True)

dar_backup/dar-backup.conf

@@ -9,7 +9,7 @@ MIN_SIZE_VERIFICATION_MB = 1
 NO_FILES_VERIFICATION = 5
 # timeout in seconds for backup, test, restore and par2 operations
 # The author has such `dar` tasks running for 10-15 hours on the yearly backups, so a value of 24 hours is used.
-# If a timeout is not specified when using the util.run_command(), a default timeout of 30 secs is used.
+# If a timeout is not specified when using the CommandRunner, a default timeout of 30 secs is used.
 COMMAND_TIMEOUT_SECS = 86400
 
 [DIRECTORIES]

dar_backup/dar_backup.py

@@ -25,14 +25,17 @@ from typing import List
 from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import list_backups
-from dar_backup.util import run_command
 from dar_backup.util import setup_logging
 from dar_backup.util import get_logger
 from dar_backup.util import BackupError
 from dar_backup.util import RestoreError
 
+from dar_backup.command_runner import CommandRunner   
+from dar_backup.command_runner import CommandResult
+
 
 logger = None
+runner = None
 
 def generic_backup(type: str, command: List[str], backup_file: str, backup_definition: str, darrc: str,  config_settings: ConfigSettings, args: argparse.Namespace) -> List[str]:
     """
@@ -63,9 +66,8 @@ def generic_backup(type: str, command: List[str], backup_file: str, backup_defin
     result: List[tuple] = []
 
     logger.info(f"===> Starting {type} backup for {backup_definition}")
-    logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
     try:
-        process = run_command(command, config_settings.command_timeout_secs)
+        process = runner.run(command, timeout = config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"{type} backup completed successfully.")
         elif process.returncode == 5:
@@ -75,7 +77,7 @@ def generic_backup(type: str, command: List[str], backup_file: str, backup_defin
 
         if process.returncode == 0 or process.returncode == 5:
             add_catalog_command = ['manager', '--add-specific-archive' ,backup_file, '--config-file', args.config_file]
-            command_result = run_command(add_catalog_command, config_settings.command_timeout_secs)
+            command_result = runner.run(add_catalog_command, timeout = config_settings.command_timeout_secs)
             if command_result.returncode == 0:
                 logger.info(f"Catalog for archive '{backup_file}' added successfully to its manager.")
             else:
@@ -192,8 +194,7 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
     """
     result = True
     command = ['dar', '-t', backup_file, '-Q']
-    logger.debug(f"Running command: {' '.join(map(shlex.quote, command))}")
-    process = run_command(command, config_settings.command_timeout_secs)
+    process = runner.run(command, timeout = config_settings.command_timeout_secs)
     if process.returncode == 0:
         logger.info("Archive integrity test passed.")
     else:
@@ -235,7 +236,7 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
             args.verbose and logger.info(f"Restoring file: '{restored_file_path}' from backup to: '{config_settings.test_restore_dir}' for file comparing")
             command = ['dar', '-x', backup_file, '-g', restored_file_path.lstrip("/"), '-R', config_settings.test_restore_dir, '-Q', '-B', args.darrc, 'restore-options']
             args.verbose and logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
-            process = run_command(command, config_settings.command_timeout_secs)    
+            process = runner.run(command, timeout = config_settings.command_timeout_secs)    
             if process.returncode != 0:
                 raise Exception(str(process))
 
@@ -276,7 +277,7 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
             command.extend(selection_criteria)
         command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
         logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
-        process = run_command(command, config_settings.command_timeout_secs)
+        process = runner.run(command, timeout = config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"Restore completed successfully to: '{restore_dir}'")
         else:
@@ -309,7 +310,7 @@ def get_backed_up_files(backup_name: str, backup_dir: str):
     try:
         command = ['dar', '-l', backup_path, '-am', '-as', "-Txml" , '-Q']
         logger.debug(f"Running command: {' '.join(map(shlex.quote, command))}")
-        command_result = run_command(command)
+        command_result = runner.run(command)
         # Parse the XML data
         file_paths = find_files_with_paths(command_result.stdout)
         return file_paths
@@ -339,8 +340,7 @@ def list_contents(backup_name, backup_dir, selection=None):
         if selection:
             selection_criteria = shlex.split(selection)
             command.extend(selection_criteria)
-        logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
-        process = run_command(command)
+        process = runner.run(command)
         stdout,stderr = process.stdout, process.stderr
         if process.returncode != 0:
             logger.error(f"Error listing contents of backup: '{backup_name}'")
@@ -512,7 +512,7 @@ def generate_par2_files(backup_file: str, config_settings: ConfigSettings, args)
 
         # Run the par2 command to generate redundancy files with error correction
         command = ['par2', 'create', f'-r{config_settings.error_correction_percent}', '-q', '-q', file_path]
-        process = run_command(command, config_settings.command_timeout_secs)
+        process = runner.run(command, timeout = config_settings.command_timeout_secs)
 
         if process.returncode == 0:
             logger.info(f"{counter}/{number_of_slices}: Done")
@@ -629,7 +629,7 @@ def requirements(type: str, config_setting: ConfigSettings):
         config_settings (ConfigSettings): An instance of the ConfigSettings class.
 
     Raises:
-        RuntimeError: If a subprocess  returns anything but zero.
+        RuntimeError: If a subprocess returns anything but zero.
 
         subprocess.CalledProcessError: if CalledProcessError is raised in subprocess.run(), let it bobble up.
     """
@@ -641,7 +641,7 @@ def requirements(type: str, config_setting: ConfigSettings):
         raise RuntimeError(f"requirements: {type} not in: {allowed_types}")
 
 
-    logger.info(f"Performing  {type}")
+    logger.debug(f"Performing  {type}")
     if type in config_setting.config:
         for key in sorted(config_setting.config[type].keys()):
             script = config_setting.config[type][key]
@@ -658,7 +658,7 @@ def requirements(type: str, config_setting: ConfigSettings):
 
 
 def main():
-    global logger
+    global logger, runner
     results: List[(str,int)] = []  # a list op tuples (<msg>, <exit code>)
 
     MIN_PYTHON_VERSION = (3, 9)
@@ -713,6 +713,9 @@ def main():
         print(f"Error: logfile_location in {args.config_file} does not end at 'dar-backup.log', exiting", file=stderr)
 
     logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout)
+    command_logger = get_logger(command_output_logger = True)
+    runner = CommandRunner(logger=logger, command_logger=command_logger)
+
 
     try:
         if not args.darrc:
@@ -741,9 +744,9 @@ def main():
         file_dir =  os.path.normpath(os.path.dirname(__file__))
         args.verbose and (print(f"Script directory:  {file_dir}"))
         args.verbose and (print(f"Config file:       {args.config_file}"))
-        args.verbose and args.full_backup         and (print(f"Type of backup: FULL"))
-        args.verbose and args.differential_backup and (print(f"Type of backup: DIFF"))
-        args.verbose and args.incremental_backup  and (print(f"Type of backup: INCR"))
+        args.verbose and args.full_backup         and (print(f"Type of backup:    FULL"))
+        args.verbose and args.differential_backup and (print(f"Type of backup:    DIFF"))
+        args.verbose and args.incremental_backup  and (print(f"Type of backup:    INCR"))
         args.verbose and args.backup_definition   and (print(f"Backup definition: '{args.backup_definition}'"))
         if args.alternate_reference_archive:
             args.verbose and (print(f"Alternate ref archive: {args.alternate_reference_archive}"))

dar_backup/manager.py

@@ -29,9 +29,12 @@ import sys
 
 from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
-from dar_backup.util import run_command
 from dar_backup.util import setup_logging
 from dar_backup.util import CommandResult
+from dar_backup.util import get_logger
+
+from dar_backup.command_runner import CommandRunner   
+from dar_backup.command_runner import CommandResult
 
 from datetime import datetime
 from time import time
@@ -44,6 +47,7 @@ SCRIPTDIRPATH = os.path.dirname(SCRIPTPATH)
 DB_SUFFIX = ".db"
 
 logger = None
+runner = None
 
 def show_more_help():
     help_text = f"""
@@ -66,7 +70,7 @@ def create_db(backup_def: str, config_settings: ConfigSettings):
     else:
         logger.info(f'Create catalog database: "{database_path}"')
         command = ['dar_manager', '--create' , database_path]
-        process = run_command(command)
+        process = runner.run(command)
         logger.debug(f"return code from 'db created': {process.returncode}")
         if process.returncode == 0:
             logger.info(f'Database created: "{database_path}"')
@@ -104,7 +108,7 @@ def list_catalogs(backup_def: str, config_settings: ConfigSettings) -> NamedTupl
         command=[])
         return commandResult
     command = ['dar_manager', '--base', database_path, '--list']
-    process = run_command(command)
+    process = runner.run(command)
     stdout, stderr = process.stdout, process.stderr 
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
@@ -156,7 +160,7 @@ def list_archive_contents(archive: str, config_settings: ConfigSettings) -> int
         logger.error(f"archive: '{archive}' not found in database: '{database_path}'")
         return 1
     command = ['dar_manager', '--base', database_path, '-u', f"{cat_no}"]
-    process = run_command(command)
+    process = runner.run(command)
     stdout, stderr = process.stdout, process.stderr 
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
@@ -178,7 +182,7 @@ def list_catalog_contents(catalog_number: int, backup_def: str, config_settings:
         logger.error(f'Catalog database not found: "{database_path}"')
         return 1
     command = ['dar_manager', '--base', database_path, '-u', f"{catalog_number}"]
-    process = run_command(command)
+    process = runner.run(command)
     stdout, stderr = process.stdout, process.stderr 
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
@@ -199,7 +203,7 @@ def find_file(file, backup_def, config_settings):
         logger.error(f'Database not found: "{database_path}"')
         return 1
     command = ['dar_manager', '--base', database_path, '-f', f"{file}"]
-    process = run_command(command)
+    process = runner.run(command)
     stdout, stderr = process.stdout, process.stderr 
     if process.returncode != 0:
         logger.error(f'Error finding file: {file} in: "{database_path}"')
@@ -234,7 +238,7 @@ def add_specific_archive(archive: str, config_settings: ConfigSettings, director
     logger.info(f'Add "{archive_path}" to catalog: "{database}"')
     
     command = ['dar_manager', '--base', database_path, "--add", archive_path, "-Q"]
-    process = run_command(command)
+    process = runner.run(command)
     stdout, stderr = process.stdout, process.stderr
 
     if process.returncode == 0:
@@ -338,7 +342,7 @@ def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> in
     cat_no:int = cat_no_for_name(archive, config_settings)
     if cat_no >= 0:
         command = ['dar_manager', '--base', database_path, "--delete", str(cat_no)]
-        process: CommandResult = run_command(command)
+        process: CommandResult = runner.run(command)
         logger.info(f"CommandResult: {process}")
     else:
         logger.warning(f"archive: '{archive}' not found in it's catalog database: {database_path}")
@@ -355,7 +359,7 @@ def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> in
 
 
 def main():
-    global logger 
+    global logger, runner 
 
     MIN_PYTHON_VERSION = (3, 9)
     if sys.version_info < MIN_PYTHON_VERSION:
@@ -404,6 +408,8 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
 #    command_output_log = os.path.join(config_settings.logfile_location.removesuffix("dar-backup.log"), "dar-backup-commands.log")
     command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
     logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout)
+    command_logger = get_logger(command_output_logger = True)
+    runner = CommandRunner(logger=logger, command_logger=command_logger)
 
 
     start_time=int(time())

dar_backup/util.py

@@ -97,109 +97,6 @@ def get_logger(command_output_logger: bool = False) -> logging.Logger:
     return secondary_logger if command_output_logger else logger
 
 
-
-def _stream_reader(pipe, log_funcs, output_accumulator: List[str]):
-    """
-    Reads lines from the subprocess pipe and logs them to multiple destinations.
-    """
-    with pipe:
-        for line in iter(pipe.readline, ''):
-            stripped_line = line.strip()
-            output_accumulator.append(stripped_line)
-            for log_func in log_funcs:
-                log_func(stripped_line)  # Log the output in real-time
-
-
-
-def run_command(command: List[str], timeout: int = 30, no_output_log: bool = False):
-    """
-    Executes a command and streams output only to the secondary log unless no_log is set to True.
-
-
-    Returns:
-        A CommandResult NamedTuple with the following properties:
-        - process: subprocess.CompletedProcess
-        - stdout: str: The full standard output of the command.
-        - stderr: str: The full standard error of the command.
-        - returncode: int: The return code of the command.
-        - timeout: int: The timeout value in seconds used to run the command.
-        - command: list[str]: The command executed.
-
-    Logs:
-        - Logs standard output (`stdout`) and standard error in real-time to the
-          logger.secondary_log (that contains the command output).  
-    
-    Raises:
-        subprocess.TimeoutExpired: If the command execution times out (see `timeout` parameter).
-        Exception: If other exceptions occur during command execution.
-        FileNotFoundError: If the command is not found.
-    """
-    stdout_lines, stderr_lines = [], []
-    process = None
-    stdout_thread, stderr_thread = None, None
-
-    try:
-        logger =         get_logger(command_output_logger=False)
-        command_logger = get_logger(command_output_logger=True)
-
-        if not shutil.which(command[0]):
-            raise FileNotFoundError(f"Command not found: {command[0]}")
-   
-        logger.debug(f"Running command: {command}")
-        command_logger.info(f"Running command: {command}")
-
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-
-        log_funcs = [command_logger.info] if not no_output_log else []
-        err_log_funcs = [command_logger.error] if not no_output_log else []
-
-        stdout_thread = threading.Thread(target=_stream_reader, args=(process.stdout, log_funcs,     stdout_lines))
-        stderr_thread = threading.Thread(target=_stream_reader, args=(process.stderr, err_log_funcs, stderr_lines))
-        
-        stdout_thread.start()
-        stderr_thread.start()
-
-        process.wait(timeout=timeout)
-
-    except FileNotFoundError as e:
-        logger.error(f"Command not found: {command[0]}")
-        return CommandResult(
-            process=None,
-            stdout="",
-            stderr=str(e),
-            returncode=127,
-            timeout=timeout,
-            command=command
-        )
-    except subprocess.TimeoutExpired:
-        if process:
-            process.terminate()
-        logger.error(f"Command: '{command}' timed out and was terminated.")
-        raise
-    except Exception as e:
-        logger.error(f"Error running command: {command}", exc_info=True)
-        raise
-    finally:
-        if stdout_thread and stdout_thread.is_alive():
-            stdout_thread.join()
-        if stderr_thread and stderr_thread.is_alive():
-            stderr_thread.join()
-        if process:
-            if process.stdout:
-                process.stdout.close()
-            if process.stderr:
-                process.stderr.close()
-
-    
-    # Combine captured stdout and stderr lines into single strings
-    stdout = "\n".join(stdout_lines)
-    stderr = "\n".join(stderr_lines)
-
-    #Build the result object
-    result = CommandResult(process=process, stdout=stdout, stderr=stderr, returncode=process.returncode, timeout=timeout, command=command)
-    return result
-
-
 class BackupError(Exception):
     """Exception raised for errors in the backup process."""
     pass

{dar_backup-0.6.16.dist-info → dar_backup-0.6.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dar-backup
-Version: 0.6.16
+Version: 0.6.17
 Summary: A script to do full, differential and incremental backups using dar.     Some files are restored from the backups during verification, after which par2 redundancy files are created.     The script also has a cleanup feature to remove old backups and par2 files.
 Project-URL: Homepage, https://github.com/per2jensen/dar-backup/tree/main/v2
 Project-URL: Changelog, https://github.com/per2jensen/dar-backup/blob/main/v2/Changelog.md
@@ -744,13 +744,15 @@ This is the `Python` based **version 2** of `dar-backup`.
   - [Performance tip due to par2](#performance-tip-due-to-par2)
   - [.darrc sets -vd -vf (since v0.6.4)](#darrc-sets--vd--vf-since-v064)
   - [Separate log file for command output](#separate-log-file-for-command-output)
+  - [Skipping cache directories](#skipping-cache-directories)
 - [Todo](#todo)
 - [Reference](#reference)
-  - [dar-backup](#dar-backup)
-  - [manager](#manager)
-  - [cleanup](#cleanup)
-  - [clean-log](#clean-log)
-  - [installer](#installer)
+  - [Test coverage report](#test-coverage)
+  - [dar-backup](#dar-backup-options)
+  - [manager](#manager-options)
+  - [cleanup](#cleanup-options)
+  - [clean-log](#clean-log-options)
+  - [installer](#installer-options)
   
 ## My use case
 
@@ -807,15 +809,24 @@ On Ubuntu, install the requirements this way:
 
 ### dar-backup
 
-`dar-backup` is built in a way that emphasizes getting backups. It loops over the backup definitions, and in the event of a failure while backing up a backup definition, dar-backup shall log an error and start working on the next backup definition.
+`dar-backup` is built in a way that emphasizes getting backups. It loops over the [backup definitions](#backup-definition-example), and in the event of a failure while backing up a backup definition, dar-backup shall log an error and start working on the next backup definition.
 
 There are 3 levels of backups, FULL, DIFF and INCR.
 
-- The author does a FULL yearly backup once a year. This includes all files in all directories as defined in the backup definition(s).
-- The author makes a DIFF once a month. The DIFF backs up new and changed files compared to the FULL backup.
-- The author takes an INCR backup every 3 days. An INCR backup includes new and changed files compared to the DIFF backup.
---  So, a set of INCR's will contain duplicates (this might change as I become more used to use the catalog databases)
---  No INCR backups will taken until a DIFF backup has been taken.
+- The author does a FULL yearly backup once a year. This includes all files in all directories as defined in the backup definition(s) (assuming `-d` was not given).
+- The author makes a DIFF once a month. The DIFF backs up new and changed files **compared** to the **FULL** backup.
+
+  - No DIFF backups are taken until a FULL backup has been taken for a particular backup definition.
+
+- The author takes an INCR backup every 3 days. An INCR backup includes new and changed files **compared** to the **DIFF** backup.
+  
+  - So, a set of INCR's will contain duplicates (this might change as I become more used to use the catalog databases)
+  
+  - No INCR backups are taken until a DIFF backup has been taken for a particular backup definition.
+
+After each backup of a backup definition, `dar-backup` tests the archive and then performs a few restore operations of random files from the archive (see [dar-backup.conf](#config-file)). The restored files are compared to the originals to check if the restore went well.
+
+`dar-backup` skips doing a backup of a backup definition if an archive is already in place. So, if you for some reason need to take a new backup on the same date, the first archive must be deleted (I recommend using [cleanup](#cleanup-1)).
 
 ### cleanup
 
@@ -823,6 +834,12 @@ The `cleanup` application deletes DIFF and INCR if the archives are older than t
 
 `cleanup` will only remove FULL archives if the option  `--cleanup-specific-archives` is used. It requires the user to confirm deletion of FULL archives.
 
+### manager
+
+`dar`has the concept of catalogs which can be exported and optionally be added to a catalog database. That database makes it much easier to restore the correct version of a backed up file if for example a target date has been set.
+
+`dar-backup` adds archive catalogs to their databases (using the `manager` script). Should the operation fail, `dar-backup` logs an error and continue with testing and restore validation tests.
+
 ## How to run
 
 ### 1 - installation
@@ -839,7 +856,6 @@ Note:
 
 The module `inputimeout` is installed into the venv and used for the confirmation input (with a 30 second timeout)
 
-
 To install, create a venv and run pip:
 
 ```` bash
@@ -902,7 +918,7 @@ manager --create-db
 ### 4 - do FULL backups
 
 Prereq:
-Backup definitions are in place in BACKUP.D_DIR (see config file)
+[Backup definitions](#backup-definition-example) are in place in BACKUP.D_DIR (see [config file](#config-file)).
 
 You are ready to do backups of all your backup definitions.
 
@@ -975,7 +991,7 @@ SCRIPT_1 = df -h
 
 ### .darrc
 
-The package includes a default `.darrc` file which configures `dar`.
+The package includes a default `darrc` file which configures `dar`.
 
 You can override the default `.darrc` using the `--darrc` option.
 
@@ -1106,7 +1122,7 @@ compress-exclusion:
 
 ### Backup definition example
 
-This piece of configuration is a `backup definition`. It is placed in the BACKUP.D_DIR (see config file description).
+This piece of configuration is a [backup definition](#backup-definition-example). It is placed in the BACKUP.D_DIR (see config file description).
 The name of the file is the name of the backup definition.
 
 You can use as many backup definitions as you need.
@@ -1143,6 +1159,7 @@ You can use as many backup definitions as you need.
 
 # bypass directores marked as cache directories
 # http://dar.linux.free.fr/doc/Features.html
+# https://bford.info/cachedir/
 --cache-directory-tagging
 ````
 
@@ -1367,7 +1384,7 @@ deactivate
 
 "dar" in newer versions emits a question about file ownership, which is "answered" with a "no" via the "-Q" option. That in turn leads to an error code 4.
 
-Thus the dar option "--comparison-field=ignore-owner" has been placed in the supplied .darrc file (located in the virtual environment where dar-backup is installed).
+Thus the dar option "--comparison-field=ignore-owner" has been placed in the supplied [.darrc](#darrc) file (located in the virtual environment where dar-backup is installed).
 
 This causes dar to restore without an error.
 
@@ -1382,7 +1399,7 @@ My home directory is on a btrfs filesystem, while /tmp (for the restore test) is
 
 The restore test can result in an exit code 5, due to the different filesystems used. In order to avoid the errors, the "option "--fsa-scope none" can be used. That will restult in FSA's not being restored.
 
-If you need to use this option, un-comment it in the .darrc file (located in the virtual environment where dar-backup is installed)
+If you need to use this option, un-comment it in the [.darrc](#darrc) file (located in the virtual environment where dar-backup is installed)
 
 ## Par2
 
@@ -1453,7 +1470,7 @@ Slice size should be smaller than available RAM, apparently a large performance
 
 ### .darrc sets -vd -vf (since v0.6.4)
 
-These .darrc settings make `dar` print the current directory being processed (-vd) and some stats after (-vf)
+These [.darrc](#darrc) settings make `dar` print the current directory being processed (-vd) and some stats after (-vf)
 
 This is very useful in very long running jobs to get an indication that the backup is proceeding normally.
 
@@ -1465,6 +1482,14 @@ In order to not clutter that log file with the output of commands being run, a n
 
 The secondary log file can get quite cluttered, if you want to remove the clutter, run the `clean-log`script with the `--file` option, or simply delete it.
 
+### Skipping cache directories
+
+The author uses the `--cache-directory-tagging` option in his [backup definitions](#backup-definition-example).
+
+The effect is that directories with the [CACHEDIR.TAG](https://bford.info/cachedir/) file are not backed up. Those directories contain content fetched from the net, which is of an ephemeral nature and probably not what you want to back up.
+
+If the option is not in the backup definition, the cache directories are backed up as any other.
+
 ## Todo
 
 - `installer` to generate, but not deploy systemd units and timers for:
@@ -1475,80 +1500,109 @@ The secondary log file can get quite cluttered, if you want to remove the clutte
 
 ## Reference
 
-### dar-backup
+### test coverage
 
-This script is responsible for managing the backup creation and validation process. It supports the following options:
+Running
+
+```` bash
+pytest --cov=dar_backup tests/
+````
+
+gives for version 0.6.17:
+
+```` code
+---------- coverage: platform linux, python 3.12.3-final-0 -----------
+Name                                                              Stmts   Miss  Cover
+-------------------------------------------------------------------------------------
+venv/lib/python3.12/site-packages/dar_backup/__about__.py             1      0   100%
+venv/lib/python3.12/site-packages/dar_backup/__init__.py              0      0   100%
+venv/lib/python3.12/site-packages/dar_backup/clean_log.py            68     14    79%
+venv/lib/python3.12/site-packages/dar_backup/cleanup.py             196     53    73%
+venv/lib/python3.12/site-packages/dar_backup/config_settings.py      66      8    88%
+venv/lib/python3.12/site-packages/dar_backup/dar_backup.py          464     99    79%
+venv/lib/python3.12/site-packages/dar_backup/installer.py            46     46     0%
+venv/lib/python3.12/site-packages/dar_backup/manager.py             316     72    77%
+venv/lib/python3.12/site-packages/dar_backup/util.py                162     34    79%
+-------------------------------------------------------------------------------------
+TOTAL                                                              1319    326    75%
+````
+
+### dar-backup options
+
+This script does backups, validation and restoring. It has the following options:
 
 ``` code
---full-backup                         Perform a full backup.
---differential-backup                 Perform a differential backup.
---incremental-backup                  Perform an incremental backup.
---backup-definition <name>            Specify the backup definition file.
---alternate-reference-archive <file>  Use a different archive for DIFF/INCR backups.
---config-file <path>                  Specify the path to the configuration file.
---darrc <path>                        Specify an optional path to .darrc.
---examples                            Show examples of using dar-backup.py.
---list                                List available backups.
---list-contents <archive>             List the contents of a specified archive.
---selection <params>                  Define file selection for listing/restoring.
---restore <archive>                   Restore a specified archive.
---restore-dir <path>                  Directory to restore files to.
---verbose                             Enable verbose output.
---suppress-dar-msg                    Filter out this from the darrc: "-vt", "-vs", "-vd", "-vf", "-va"
---log-level <level>                   `debug` or `trace`, default is `info`", default="info".
---log-stdout                          Also print log messages to stdout.
---do-not-compare                      Do not compare restores to file system.
---version                             Show version and license information.
+-F, --full-backup                    Perform a full backup.
+-D, --differential-backup            Perform a differential backup.
+-I, --incremental-backup             Perform an incremental backup.
+-d, --backup-definition <name>       Specify the backup definition file.
+--alternate-reference-archive <file> Use a different archive for DIFF/INCR backups.
+-c, --config-file <path>             Specify the path to the configuration file.
+--darrc <path>                       Specify an optional path to .darrc.
+--examples                           Show examples of using dar-backup.py.
+-l, --list                           List available backups.
+--list-contents <archive>            List the contents of a specified archive.
+--selection <params>                 Define file selection for listing/restoring.
+--restore <archive>                  Restore a specified archive.
+-r, --restore-dir <path>             Directory to restore files to.
+--verbose                            Enable verbose output.
+--suppress-dar-msg                   Filter out this from the darrc: "-vt", "-vs", "-vd", "-vf", "-va"
+--log-level <level>                  `debug` or `trace`, default is `info`.
+--log-stdout                         Also print log messages to stdout.
+--do-not-compare                     Do not compare restores to file system.
+-v --version                         Show version and license information.
 ```
 
-### manager
+### manager options
 
-This script manages `dar` databases and catalogs. Available options include:
+This script manages `dar` databases and catalogs. Available options:
 
 ``` code
---create-db                           Create missing databases for all backup definitions.
---alternate-archive-dir <path>        Use this directory instead of BACKUP_DIR in the config file.
---add-dir <path>                      Add all archive catalogs in this directory to databases.
--d, --backup-def <name>               Restrict to work only on this backup definition.
---add-specific-archive <archive>      Add this archive to the catalog database.
---remove-specific-archive <archive>   Remove this archive from the catalog database.
--l, --list-catalogs                   List catalogs in databases for all backup definitions.
---list-catalog-contents <num>         List contents of a catalog by catalog number.
---list-archive-contents <archive>     List contents of an archive’s catalog, given the archive name.
---find-file <file>                    Search catalogs for a specific file.
---verbose                             Enable verbose output.
---log-level <level>                   `debug` or `trace`, default is `info`", default="info".
+-c, --config-file                    Path to dar-backup.conf
+--create-db                          Create missing databases for all backup definitions.
+--alternate-archive-dir <path>       Use this directory instead of BACKUP_DIR in the config file.
+--add-dir <path>                     Add all archive catalogs in this directory to databases.
+-d, --backup-def <name>              Restrict to work only on this backup definition.
+--add-specific-archive <archive>     Add this archive to the catalog database.
+--remove-specific-archive <archive>  Remove this archive from the catalog database.
+-l, --list-catalogs                  List catalogs in databases for all backup definitions.
+--list-catalog-contents <num>        List contents of a catalog by catalog number.
+--list-archive-contents <archive>    List contents of an archive’s catalog, given the archive name.
+--find-file <file>                   Search catalogs for a specific file.
+--verbose                            Enable verbose output.
+--log-level <level>                  `debug` or `trace`, default is `info`", default="info".
 ```
 
-### cleanup
+### cleanup options
 
-This script cleans up old backups and manages storage. Supported options:
+This script cleans up old backups and par2 files. Supported options:
 
 ``` code
--d, --backup-definition                       Backup definition to cleanup.
--c, --config-file                             Path to 'dar-backup.conf', default='~/.config/dar-backup/dar-backup.conf.
--v, --version                                 Show version & license information.
---alternate-archive-dir                       Clean up in this directory instead of the default one.
---cleanup-specific-archives <archive>, ...    Comma separated list of archives to cleanup.
--l,  --list                                   List available archives.
---verbose                                     Print various status messages to screen.
---log-level <level>                           `debug` or `trace`, default is `info`", default="info".
---log-stdout                                  Print log messages to stdout.
+-d, --backup-definition                           Backup definition to cleanup.
+-c, --config-file                                 Path to 'dar-backup.conf'
+-v, --version                                     Show version & license information.
+--alternate-archive-dir                           Clean up in this directory instead of the default one.
+--cleanup-specific-archives "<archive>, <>, ..."  Comma separated list of archives to cleanup.
+-l, --list                                       List available archives (filter using the -d option).
+--verbose                                         Print various status messages to screen.
+--log-level <level>                               `debug` or `trace`, default is `info`", default="info".
+--log-stdout                                      Print log messages to stdout.
+--test-mode                                       This is used when running pytest test cases
 ```
 
-### clean-log
+### clean-log options
 
 This script removes excessive logging output from `dar` logs, improving readability and efficiency. Available options:
 
 ``` code
 -f, --file <path>          Specify the log file(s) to be cleaned.
--c, --config-file <path>   Specify the configuration file (default: ~/.config/dar-backup/dar-backup.conf).
+-c, --config-file <path>   Path to dar-backup.conf.
 --dry-run                  Show which lines would be removed without modifying the file.
 -v, --version              Display version and licensing information.
 -h, --help                 Displays usage info
 ```
 
-### installer
+### installer options
 
 Sets up `dar-backup`for a user.
 

dar_backup-0.6.17.dist-info/RECORD

@@ -0,0 +1,17 @@
+dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
+dar_backup/__about__.py,sha256=vii4GL7MExpBC8tvQjQXAsEgfxDE9p438_97wKl4XCc,22
+dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dar_backup/clean_log.py,sha256=cGhtKYnQJ2ceNQfw5XcCln_WNBasbmlfhO3kRydjDNk,5196
+dar_backup/cleanup.py,sha256=HA8SmwrqDGjfFg4L0CuUQ5N0YJFT6qYbCuvh5mQseF0,13148
+dar_backup/command_runner.py,sha256=74Fsylz1NN-dn8lbdRhkL6LA1r527QJeojBlniGrPuo,2708
+dar_backup/config_settings.py,sha256=Rh4T35-w_5tpRAViMfv3YP3GBpG4mQy7Do8cNBzYAR0,4912
+dar_backup/dar-backup.conf,sha256=64O3bGlzqupneT2gVeaETJ1qS6-3Exet9Zto27jgwPQ,897
+dar_backup/dar_backup.py,sha256=NHBm3zsOhCHnCVoPO0ysD3uMdIQMe62AIdr0yCQ_6BY,37952
+dar_backup/installer.py,sha256=ehp4KSgTc8D9Edsyve5v3NY2MuDbuTFYQQPgou8woV8,4331
+dar_backup/manager.py,sha256=4NeIVgrhIzOS8UePUCdvtswEG55ue0tXWAK7SjD3tpo,21897
+dar_backup/util.py,sha256=6dJXFOjIIZqerbNVFxJZ6gQ4ZVAxyY-RxHcO--9bxwg,8462
+dar_backup-0.6.17.dist-info/METADATA,sha256=-TmZ95gGD9VX1Uz0so4G8sWWX0RE6Aq8qkfKThkDFnY,79979
+dar_backup-0.6.17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dar_backup-0.6.17.dist-info/entry_points.txt,sha256=Z7P5BUbhtJxo8_nB9qNIMay2eGDbsMKB3Fjwv3GMa4g,202
+dar_backup-0.6.17.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dar_backup-0.6.17.dist-info/RECORD,,

dar_backup-0.6.16.dist-info/RECORD

@@ -1,16 +0,0 @@
-dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
-dar_backup/__about__.py,sha256=iZmWvp4Ehnidf9YJv6Dpn5Sma5S84lPbQtLoYZ2OuiI,22
-dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dar_backup/clean_log.py,sha256=cGhtKYnQJ2ceNQfw5XcCln_WNBasbmlfhO3kRydjDNk,5196
-dar_backup/cleanup.py,sha256=EoT_jTiJmy_HoWxxlWuRn21GoMnr5ioN8yM7qiNRPKE,13351
-dar_backup/config_settings.py,sha256=uicCq6FnpxPFzbv7xfYSXNnQf1tfLk1Z3VIO9M71fsE,4659
-dar_backup/dar-backup.conf,sha256=-wXqP4vj5TS7cCfMJN1nbk-1Sqkq00Tg22ySQXynUF4,902
-dar_backup/dar_backup.py,sha256=PI154FIXZiU36iAyLZCCxAciDeBSwoBXYQh0n5JmNEs,37895
-dar_backup/installer.py,sha256=ehp4KSgTc8D9Edsyve5v3NY2MuDbuTFYQQPgou8woV8,4331
-dar_backup/manager.py,sha256=sQl0xdWwBgui11S9Ekg0hOSC4gt89nz_Z8Bt8IPXCDw,21640
-dar_backup/util.py,sha256=F6U-e-WugxCxLPVoiWsM6_YO8VrDw1wdgGvtnGnig2I,12279
-dar_backup-0.6.16.dist-info/METADATA,sha256=xaYjZK6XAWOc_NHsnktylhdvqWLqzmo94GT1MoR6qLw,76678
-dar_backup-0.6.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dar_backup-0.6.16.dist-info/entry_points.txt,sha256=Z7P5BUbhtJxo8_nB9qNIMay2eGDbsMKB3Fjwv3GMa4g,202
-dar_backup-0.6.16.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dar_backup-0.6.16.dist-info/RECORD,,