dar-backup 0.6.10__py3-none-any.whl → 0.6.12__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.10"
+__version__ = "0.6.12"

dar_backup/clean_log.py

@@ -122,7 +122,7 @@ def main():
 
     args = parser.parse_args()
 
-    config_settings = ConfigSettings(os.path.expanduser(args.config_file))
+    config_settings = ConfigSettings(os.path.expanduser(os.path.expandvars(args.config_file)))
 
     if not args.file:
         args.file = [config_settings.logfile_location]

dar_backup/cleanup.py

@@ -171,7 +171,7 @@ def main():
     parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
     args = parser.parse_args()
 
-    args.config_file = os.path.expanduser(args.config_file)
+    args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
     
 
     if args.version:

dar_backup/config_settings.py

@@ -1,8 +1,11 @@
-from dataclasses import dataclass, field
+
 import configparser
-from pathlib import Path
-import sys
 import logging
+import sys
+
+from dataclasses import dataclass, field, fields
+from os.path import expandvars, expanduser
+from pathlib import Path
 
 @dataclass
 class ConfigSettings:
@@ -20,21 +23,35 @@ class ConfigSettings:
         backup_d_dir (str): The directory for backup.d.
         diff_age (int): The age for differential backups before deletion.
         incr_age (int): The age for incremental backups before deletion.
+        error_correction_percent (int): The error correction percentage for PAR2.
+        par2_enabled (bool): Whether PAR2 is enabled.
     """
 
-    def __init__(self, config_file: str):
+    config_file: str
+    logfile_location: str = field(init=False)
+    max_size_verification_mb: int = field(init=False)
+    min_size_verification_mb: int = field(init=False)
+    no_files_verification: int = field(init=False)
+    command_timeout_secs: int = field(init=False)
+    backup_dir: str = field(init=False)
+    test_restore_dir: str = field(init=False)
+    backup_d_dir: str = field(init=False)
+    diff_age: int = field(init=False)
+    incr_age: int = field(init=False)
+    error_correction_percent: int = field(init=False)
+    par2_enabled: bool = field(init=False)
+
+    def __post_init__(self):
         """
-        Initializes the ConfigSettings instance by reading the specified configuration file.
-        
-        Args:
-            config_file (str): The path to the configuration file.
+        Initializes the ConfigSettings instance by reading the specified configuration file
+        and expands environment variables for all string fields.
         """
-        if config_file is None:
+        if self.config_file is None:
             raise ValueError("`config_file` must be specified.")
         
         self.config = configparser.ConfigParser()
         try:
-            self.config.read(config_file)
+            self.config.read(self.config_file)
             self.logfile_location = self.config['MISC']['LOGFILE_LOCATION']
             self.max_size_verification_mb = int(self.config['MISC']['MAX_SIZE_VERIFICATION_MB'])
             self.min_size_verification_mb = int(self.config['MISC']['MIN_SIZE_VERIFICATION_MB'])
@@ -46,12 +63,18 @@ class ConfigSettings:
             self.diff_age = int(self.config['AGE']['DIFF_AGE'])
             self.incr_age = int(self.config['AGE']['INCR_AGE'])
             self.error_correction_percent = int(self.config['PAR2']['ERROR_CORRECTION_PERCENT'])
-            self.par2_enabled = bool(self.config['PAR2']['ENABLED'])
+            self.par2_enabled = self.config['PAR2']['ENABLED'].lower() in ('true', '1', 'yes')
+            
             # Ensure the directories exist
             Path(self.backup_dir).mkdir(parents=True, exist_ok=True)
             Path(self.test_restore_dir).mkdir(parents=True, exist_ok=True)
             Path(self.backup_d_dir).mkdir(parents=True, exist_ok=True)
 
+            # Expand environment variables for all string fields
+            for field in fields(self):
+                if isinstance(getattr(self, field.name), str):
+                    setattr(self, field.name, expanduser(expandvars(getattr(self, field.name))))
+
         except FileNotFoundError as e:
             logging.error(f"Configuration file not found: {self.config_file}")
             logging.error(f"Error details: {e}")

dar_backup/dar-backup.conf

@@ -0,0 +1,33 @@
+# This config file is intended to demo `dar-backup`.
+#
+# The `installer` puts it in ~/.config/dar-backup/dar-backup.conf
+
+[MISC]
+LOGFILE_LOCATION = ~/dar-backup/dar-backup.log
+MAX_SIZE_VERIFICATION_MB = 20
+MIN_SIZE_VERIFICATION_MB = 1
+NO_FILES_VERIFICATION = 5
+# timeout in seconds for backup, test, restore and par2 operations
+# The author has such `dar` tasks running for 10-15 hours on the yearly backups, so a value of 24 hours is used.
+# If a timeout is not specified when using the util.run_command(), a default timeout of 30 secs is used.
+COMMAND_TIMEOUT_SECS = 86400
+
+[DIRECTORIES]
+BACKUP_DIR = ~/dar-backup/backups
+BACKUP.D_DIR = ~/.config/dar-backup/backup.d/
+TEST_RESTORE_DIR = ~/dar-backup/restore/
+
+[AGE]
+# age settings are in days
+DIFF_AGE = 100
+INCR_AGE = 40
+
+[PAR2]
+ERROR_CORRECTION_PERCENT = 5
+ENABLED = True
+
+[PREREQ]
+#SCRIPT_1 = <pre-script 1>
+
+[POSTREQ]
+#SCRIPT_1 = <post-script 1>

dar_backup/dar_backup.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import datetime
 import filecmp
 
 import os
@@ -9,13 +8,16 @@ import random
 import re
 import shlex
 import subprocess
-import sys
 import xml.etree.ElementTree as ET
 
 
 from argparse import ArgumentParser
 from datetime import datetime
 from pathlib import Path
+from sys import exit
+from sys import stderr
+from sys import argv
+from sys import version_info
 from time import time
 from typing import List
 
@@ -81,30 +83,31 @@ def generic_backup(type: str, command: List[str], backup_file: str, backup_defin
         raise BackupError(f"Unexpected error during backup: {e}") from e
  
 
-# Function to recursively find <File> tags and build their full paths
-def find_files_with_paths(element: ET, current_path=""):
+
+def find_files_with_paths(xml_root: ET.Element):
     """
-    Recursively finds files within a directory element and returns a list of file paths with their sizes.
+    Finds files within an XML element and returns a list of file paths with their sizes.
 
     Args:
-        element (Element): The directory element to search within.
-        current_path (str, optional): The current path of the directory element. Defaults to "".
+        xml_root (Element): The root element of the XML.
 
     Returns:
         list: A list of tuples containing file paths and their sizes.
     """
-    logger.debug(f"Recursively generate list of tuples with file paths and sizes for File elements in dar xml output")
+    logger.debug("Generating list of tuples with file paths and sizes for File elements in dar xml output")
     files = []
-    if element.tag == "Directory":
-        current_path = f"{current_path}/{element.get('name')}"
-    for child in element:
-        if child.tag == "File":
-            file_path = (f"{current_path}/{child.get('name')}", child.get('size'))  # tuple (filepath, size)
+    current_path = []
+
+    for elem in xml_root.iter():
+        if elem.tag == "directory":
+            current_path.append(elem.get('name'))
+        elif elem.tag == "file":
+            file_path = ("/".join(current_path + [elem.get('name')]), elem.get('size'))
             files.append(file_path)
-        elif child.tag == "Directory":
-            files.extend(find_files_with_paths(child, current_path))
-    return files
+        elif elem.tag == "directory" and elem.get('name') in current_path:
+            current_path.pop()
 
+    return files
 
 
 def find_files_between_min_and_max_size(backed_up_files: list[(str, str)], config_settings: ConfigSettings):
@@ -238,20 +241,20 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
         restore_dir (str): The directory where the backup should be restored to.
         selection (str, optional): A selection criteria to restore specific files or directories. Defaults to None.
     """
-    backup_file = os.path.join(config_settings.backup_dir, backup_name)
-    command = ['dar', '-x', backup_file, '-Q', '-D']
-    if restore_dir:
-        if not os.path.exists(restore_dir):
-            os.makedirs(restore_dir)
-        command.extend(['-R', restore_dir])
-    else:
-        raise RestoreError("Restore directory ('-R <dir>') not specified")
-    if selection:
-        selection_criteria = shlex.split(selection)
-        command.extend(selection_criteria)
-    command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
-    logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
     try:
+        backup_file = os.path.join(config_settings.backup_dir, backup_name)
+        command = ['dar', '-x', backup_file, '-Q', '-D']
+        if restore_dir:
+            if not os.path.exists(restore_dir):
+                os.makedirs(restore_dir)
+            command.extend(['-R', restore_dir])
+        else:
+            raise RestoreError("Restore directory ('-R <dir>') not specified")
+        if selection:
+            selection_criteria = shlex.split(selection)
+            command.extend(selection_criteria)
+        command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
+        logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
         process = run_command(command, config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"Restore completed successfully to: '{restore_dir}'")
@@ -260,6 +263,9 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
             raise RestoreError(str(process))
     except subprocess.CalledProcessError as e:
         raise RestoreError(f"Restore command failed: {e}") from e
+    except OSError as e:
+        logger.error(f"Failed to create restore directory: {e}")
+        raise RestoreError("Could not create restore directory")
     except Exception as e:
         raise RestoreError(f"Unexpected error during restore: {e}") from e
 
@@ -285,7 +291,7 @@ def get_backed_up_files(backup_name: str, backup_dir: str):
         # Parse the XML data
         root = ET.fromstring(process.stdout)
         output = None  # help gc
-        # Extract full paths and file size for all <File> elements
+        # Extract full paths and file size for all <file> elements
         file_paths = find_files_with_paths(root)
         root = None # help gc
         logger.trace(str(process))
@@ -404,7 +410,7 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                     logger.info(f"Using alternate reference archive: {latest_base_backup}")
                     if not os.path.exists(latest_base_backup + '.1.dar'):
                         logger.error(f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exiting.")
-                        sys.exit(1)
+                        exit(1)
                 else:
                     base_backups = sorted(
                         [f for f in os.listdir(config_settings.backup_dir) if f.startswith(f"{backup_definition}_{base_backup_type}_") and f.endswith('.1.dar')],
@@ -489,7 +495,7 @@ def generate_par2_files(backup_file: str, config_settings: ConfigSettings, args)
 
 
 def show_version():
-    script_name = os.path.basename(sys.argv[0])
+    script_name = os.path.basename(argv[0])
     print(f"{script_name} {about.__version__}") 
     print(f"dar-backup.py source code is here: https://github.com/per2jensen/dar-backup")
     print('''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
@@ -577,9 +583,9 @@ def main():
     global logger 
 
     MIN_PYTHON_VERSION = (3, 9)
-    if sys.version_info < MIN_PYTHON_VERSION:
-        sys.stderr.write(f"Error: This script requires Python {'.'.join(map(str, MIN_PYTHON_VERSION))} or higher.\n")
-        sys.exit(1)
+    if version_info < MIN_PYTHON_VERSION:
+        stderr.write(f"Error: This script requires Python {'.'.join(map(str, MIN_PYTHON_VERSION))} or higher.\n")
+        exit(1)
 
     parser = argparse.ArgumentParser(description="Backup and verify using dar backup definitions.")
     parser.add_argument('-F', '--full-backup', action='store_true', help="Perform a full backup.")
@@ -603,15 +609,25 @@ def main():
     parser.add_argument('-v', '--version', action='store_true', help="Show version and license information.")
     args = parser.parse_args()
 
-    args.config_file = os.path.expanduser(args.config_file)
-    config_settings = ConfigSettings(args.config_file)
-
     if args.version:
         show_version()
-        sys.exit(0)
+        exit(0)
     elif args.examples:
         show_examples()
-        sys.exit(0)
+        exit(0)
+
+    if not args.config_file:
+        print(f"Config file not specified, exiting", file=stderr)
+        exit(1) 
+    
+    config_settings_path = os.path.expanduser(os.path.expandvars(args.config_file))
+    if not os.path.exists(config_settings_path):
+        print(f"Config file {args.config_file} does not exist.", file=stderr)
+        exit(127)
+
+    args.config_file = config_settings_path
+    config_settings = ConfigSettings(args.config_file)
+
 
     logger = setup_logging(config_settings.logfile_location, args.log_level, args.log_stdout)
 
@@ -619,10 +635,13 @@ def main():
         current_script_dir = os.path.dirname(os.path.abspath(__file__))
         args.darrc = os.path.join(current_script_dir, ".darrc")
 
-    if os.path.exists(args.darrc) and os.path.isfile(args.darrc):
+    darrc_file = os.path.expanduser(os.path.expandvars(args.darrc))
+    if os.path.exists(darrc_file) and os.path.isfile(darrc_file):
         logger.debug(f"Using .darrc: {args.darrc}")                
     else:
-        logger.error(f"Supplied .darrc: '{args.darrc}' does not exist or is not a file")
+        logger.error(f"Supplied .darrc: '{args.darrc}' does not exist or is not a file, exiting", file=stderr)
+        exit(127)
+
 
 
     try:
@@ -657,10 +676,10 @@ def main():
         # sanity check
         if args.backup_definition and not os.path.exists(os.path.join(config_settings.backup_d_dir, args.backup_definition)):
             logger.error(f"Backup definition: '{args.backup_definition}' does not exist, exiting")
-            sys.exit(1)
+            exit(127)
         if args.backup_definition and '_' in args.backup_definition:
             logger.error(f"Backup definition: '{args.backup_definition}' contains '_', exiting")
-            sys.exit(1)
+            exit(1)
 
 
         requirements('PREREQ', config_settings)
@@ -684,12 +703,12 @@ def main():
         requirements('POSTREQ', config_settings)
 
         args.verbose and print("\033[1m\033[32mSUCCESS\033[0m No errors encountered")
-        sys.exit(0)
+        exit(0)
     except Exception as e:
         logger.exception("An error occurred")
         logger.error("Exception details:", exc_info=True)
         args.verbose and print("\033[1m\033[31mErrors\033[0m encountered")
-        sys.exit(1) 
+        exit(1) 
     finally:
         end_time=int(time())
         logger.info(f"END TIME: {end_time}")

dar_backup/installer.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""
+installer.py source code is here: https://github.com/per2jensen/dar-backup/tree/main/v2/src/dar_backup/installer.py
+This script is part of dar-backup, a backup solution for Linux using dar and systemd.
+
+Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, 
+not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See section 15 and section 16 in the supplied "LICENSE" file
+
+This script can be used to configure dar-backup on your system.
+It is non-destructive and will not overwrite any existing files or directories.
+"""
+
+import argparse
+import os
+import shutil
+import sys 
+
+from . import __about__ as about
+from pathlib import Path
+
+LICENSE = '''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See section 15 and section 16 in the supplied "LICENSE" file.'''
+
+CONFIG_DIR = os.path.expanduser("~/.config/dar-backup")
+DAR_BACKUP_DIR = os.path.expanduser("~/dar-backup/")
+
+BACKUP_DEFINITION = '''
+# Demo of a `dar-backup` definition file
+# This back definition file configures a backup of ~/.config/dar-backup
+# `dar-backup` puts the backups in ~/dar-backup/backups
+# ------------------------------------------------------------------------
+
+# Switch to ordered selection mode, which means that the following options
+# will be considered top to bottom
+-am
+
+# Backup Root dir
+-R @@HOME_DIR@@
+
+# Directories to backup below the Root dir
+-g .config/dar-backup
+
+# Examples of directories to exclude below the Root dir
+ -P mnt
+ -P .private
+ -P .cache   
+
+# compression level
+ -z5
+
+ # no overwrite, if you rerun a backup, 'dar' halts and asks what to do
+ -n
+ 
+ # size of each slice in the archive
+ --slice 10G
+
+# bypass directores marked as cache directories
+# http://dar.linux.free.fr/doc/Features.html
+--cache-directory-tagging
+'''
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Set up `dar-backup` on your system.",
+    )
+    parser.add_argument(
+        "-i", "--install",
+        action="store_true",
+        help="Deploy a simple config file, use ~/dar-backup/ for log file, archives and restore tests."
+    )
+
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show which lines would be removed without modifying the file."
+    )
+
+
+    parser.add_argument(
+        "-v", "--version",
+        action="version",
+        version=f"%(prog)s version {about.__version__}, {LICENSE}"
+    )
+
+    args = parser.parse_args()
+
+    if args.install:
+        errors = []
+        if os.path.exists(CONFIG_DIR):
+            errors.append(f"Config directory '{CONFIG_DIR}' already exists.")
+
+        if os.path.exists(DAR_BACKUP_DIR):
+            errors.append(f"Directory '{DAR_BACKUP_DIR}' already exists.")
+
+        if len(errors) > 0:
+            for error in errors:
+                print(f"Error: {error}")
+            sys.exit(1) 
+
+        os.makedirs(DAR_BACKUP_DIR, exist_ok=False)
+        os.makedirs(os.path.join(DAR_BACKUP_DIR, "backups"), exist_ok=False)
+        os.makedirs(os.path.join(DAR_BACKUP_DIR, "restore"), exist_ok=False)
+        os.makedirs(CONFIG_DIR, exist_ok=False)
+        os.makedirs(os.path.join(CONFIG_DIR, "backup.d"), exist_ok=False)
+        print(f"Directories created: `{DAR_BACKUP_DIR}` and `{CONFIG_DIR}`")
+
+        script_dir = Path(__file__).parent
+        source_file = script_dir / "dar-backup.conf"
+        destination_file = Path(CONFIG_DIR) / "dar-backup.conf"
+        shutil.copy(source_file, destination_file)
+        print(f"Config file deployed to {destination_file}")
+
+
+        backup_definition = BACKUP_DEFINITION.replace("@@HOME_DIR@@", os.path.expanduser("~"))
+        with open(os.path.join(CONFIG_DIR, "backup.d", "default"), "w") as f:
+            f.write(backup_definition)
+        print(f"Default backup definition file deployed to {os.path.join(CONFIG_DIR, 'backup.d', 'default')}")
+        print("1. Now run `manager --create` to create the catalog database.")
+        print("2. Then you can run `dar-backup --full-backup` to create a backup.")
+        print("3. List backups with `dar-backup --list`")
+        print("4. List contents of a backup with `dar-backup --list-contents <backup-name>`")
+
+
+if __name__ == "__main__":
+    main()

dar_backup/manager.py

@@ -395,7 +395,7 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
         sys.exit(0)
 
     # setup logging
-    args.config_file = os.path.expanduser(args.config_file)
+    args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
     config_settings = ConfigSettings(args.config_file)
     if not os.path.dirname(config_settings.logfile_location):
         print(f"Directory for log file '{config_settings.logfile_location}' does not exist, exiting")

dar_backup/util.py

@@ -14,6 +14,7 @@ import os
 import re
 import subprocess
 import shlex
+import shutil
 import sys
 import threading
 import traceback
@@ -123,10 +124,113 @@ def  _stream_reader(pipe, log_func, output_accumulator: List[str]):
             log_func(stripped_line)  # Log the output in real time
 
 
+
 def run_command(command: List[str], timeout: int = 30) -> CommandResult:
     """
     Executes a given command via subprocess, logs its output in real time, and returns the result.
 
+    Args:
+        command (list): The command to be executed, represented as a list of strings.
+        timeout (int): The maximum time in seconds to wait for the command to complete. Defaults to 30 seconds.
+
+    Returns:
+        A CommandResult NamedTuple with the following properties:
+        - process: subprocess.CompletedProcess
+        - stdout: str: The full standard output of the command.
+        - stderr: str: The full standard error of the command.
+        - returncode: int: The return code of the command.
+        - timeout: int: The timeout value in seconds used to run the command.
+        - command: list[str]: The command executed.
+
+    Logs:
+        - Logs standard output (`stdout`) in real-time at the INFO log level.
+        - Logs standard error (`stderr`) in real-time at the ERROR log level.
+
+    Raises:
+        subprocess.TimeoutExpired: If the command execution times out (see `timeout` parameter).
+        Exception: If other exceptions occur during command execution.
+        FileNotFoundError: If the command is not found.
+
+    Notes:
+        - While the command runs, its `stdout` and `stderr` streams are logged in real-time.
+        - The returned `stdout` and `stderr` capture the complete output, even though the output is also logged.
+        - The command is forcibly terminated if it exceeds the specified timeout.
+    """
+    stdout_lines = []  # To accumulate stdout
+    stderr_lines = []  # To accumulate stderr
+    process = None  # Track the process for cleanup
+    stdout_thread = None
+    stderr_thread = None
+
+    try:
+        # Check if the command exists before executing
+        if not shutil.which(command[0]):
+            raise FileNotFoundError(f"Command not found: {command[0]}")
+
+        logger.debug(f"Running command: {command}")
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+        # Start threads to read and log stdout and stderr
+        stdout_thread = threading.Thread(target=_stream_reader, args=(process.stdout, logger.info, stdout_lines))
+        stderr_thread = threading.Thread(target=_stream_reader, args=(process.stderr, logger.error, stderr_lines))
+        
+        stdout_thread.start()
+        stderr_thread.start()
+
+        # Wait for process to complete or timeout
+        process.wait(timeout=timeout)
+
+    except FileNotFoundError as e:
+        logger.error(f"Command not found: {command[0]}")
+        return CommandResult(
+            process=None,
+            stdout="",
+            stderr=str(e),
+            returncode=127,
+            timeout=timeout,
+            command=command
+        )
+    except subprocess.TimeoutExpired:
+        if process:
+            process.terminate()
+        logger.error(f"Command: '{command}' timed out and was terminated.")
+        raise
+    except Exception as e:
+        logger.error(f"Error running command: {command}", exc_info=True)
+        raise
+    finally:
+        # Ensure threads are joined to clean up (only if they were started)
+        if stdout_thread and stdout_thread.is_alive():
+            stdout_thread.join()
+        if stderr_thread and stderr_thread.is_alive():
+            stderr_thread.join()
+
+        # Ensure process streams are closed
+        if process and process.stdout:
+            process.stdout.close()
+        if process and process.stderr:
+            process.stderr.close()
+
+    # Combine captured stdout and stderr lines into single strings
+    stdout = "\n".join(stdout_lines)
+    stderr = "\n".join(stderr_lines)
+
+    # Build the result object
+    result = CommandResult(
+        process=process,
+        stdout=stdout,
+        stderr=stderr,
+        returncode=process.returncode,
+        timeout=timeout,
+        command=command
+    )
+    logger.debug(f"Command result: {result}")
+    return result
+
+def run_command2(command: List[str], timeout: int = 30) -> CommandResult:
+    """
+    Executes a given command via subprocess, logs its output in real time, and returns the result.
+
     Args:
         command (list): The command to be executed, represented as a list of strings.
         timeout (int): The maximum time in seconds to wait for the command to complete. Defaults to 30 seconds.
@@ -158,6 +262,10 @@ def run_command(command: List[str], timeout: int = 30) -> CommandResult:
     process = None  # Track the process for cleanup
 
     try:
+        # Check if the command exists before executing
+        if not shutil.which(command[0]):
+            raise FileNotFoundError(f"Command not found: {command[0]}")
+
         logger.debug(f"Running command: {command}")
         process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 
@@ -171,6 +279,16 @@ def run_command(command: List[str], timeout: int = 30) -> CommandResult:
         # Wait for process to complete or timeout
         process.wait(timeout=timeout)
 
+    except FileNotFoundError as e:
+        logger.error(f"Command not found: {command[0]}")
+        return CommandResult(
+            process=None,
+            stdout="",
+            stderr=str(e),
+            returncode=127,
+            timeout=timeout,
+            command=command
+        )
     except subprocess.TimeoutExpired:
         if process:
             process.terminate()

{dar_backup-0.6.10.dist-info → dar_backup-0.6.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dar-backup
-Version: 0.6.10
+Version: 0.6.12
 Summary: A script to do full, differential and incremental backups using dar.     Some files are restored from the backups during verification, after which par2 redundancy files are created.     The script also has a cleanup feature to remove old backups and par2 files.
 Project-URL: Homepage, https://github.com/per2jensen/dar-backup/tree/main/v2
 Project-URL: Changelog, https://github.com/per2jensen/dar-backup/blob/main/v2/Changelog.md
@@ -690,9 +690,9 @@ Classifier: Topic :: System :: Archiving :: Backup
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 
-# Full, differential or incremental backups using 'dar' 
+# Full, differential or incremental backups using 'dar'
 
-  The wonderful 'dar' [Disk Archiver] (https://github.com/Edrusb/DAR) is used for
+  The wonderful 'dar' [Disk Archiver](https://github.com/Edrusb/DAR) is used for
   the heavy lifting, together with the par2 suite in these scripts.
 
 ## Table of Contents
@@ -706,12 +706,11 @@ Description-Content-Type: text/markdown
 - [Requirements](#requirements)
 - [Config file](#config-file)
 - [How to run](#how-to-run)
-  - [1](#1)
-  - [2](#2)
-  - [3](#3)
-  - [4](#4)
-  - [5](#5)
-  - [6](#6)
+  - [1 - installation](#1---installation)
+  - [2 - configuration](#2---configuration)
+  - [3 - generate catalog databases](#3---generate-catalog-databases)
+  - [4 - do FULL backups](#4---do-full-backups)
+  - [5 - deactivate venv](#5---deactivate-venv)
 - [.darrc](#darrc)
 - [Systemctl examples](#systemctl-examples)
   - [Service: dar-back --incremental-backup](#service-dar-back---incremental-backup)
@@ -723,12 +722,19 @@ Description-Content-Type: text/markdown
   - [Exclude .xmp files from that date](#exclude-xmp-files-from-that-date)
 - [Restoring](#restoring)
   - [Default location for restores](#default-location-for-restores)
-  - [--restore-dir option](#restore-dir-option)
+  - [--restore-dir option](#--restore-dir-option)
   - [A single file](#a-single-file)
   - [A directory](#a-directory)
   - [.NEF from a specific date](#nef-from-a-specific-date)
+  - [Restore test fails with exit code 4](#restore-test-fails-with-exit-code-4)
+  - [Restore test fails with exit code 5](#restore-test-fails-with-exit-code-5)
+- [Par2](#par2)
+  - [Par2 to verify/repair](#par2-to-verifyrepair)
+  - [Par2 create redundancy files](#par2-create-redundancy-files)
 - [Points of interest](#points-of-interest)
+  - [Merge FULL with DIFF, creating new FULL](#merge-full-with-diff-creating-new-full)
   - [dar manager databases](#dar-manager-databases)
+  - [Performance tip due to par2](#performance-tip-due-to-par2)
   - [.darrc sets -vd -vf (since v0.6.4)](#darrc-sets--vd--vf-since-v064)
 - [Reference](#reference)
   - [dar-backup.py](#dar-backuppy)
@@ -791,105 +797,9 @@ On Ubuntu, install the requirements this way:
 
 The default configuration is expected here: ~/.config/dar-backup/dar-backup.conf
 
-## How to run 
+## How to run
 
-### 1
-
-Config file default location is $HOME/.config/dar-backup/dar-backup.conf
-
-Example:
-
-```` code
-[MISC]
-LOGFILE_LOCATION=/home/user/dar-backup.log
-MAX_SIZE_VERIFICATION_MB = 20
-MIN_SIZE_VERIFICATION_MB = 1
-NO_FILES_VERIFICATION = 5
-
-# timeout in seconds for backup, test, restore and par2 operations
-# The author has such `dar` tasks running for 10-15 hours on the yearly backups, so a value of 24 hours is used.
-# If a timeout is not specified when using the util.run_command(), a default timeout of 30 secs is used.
-COMMAND_TIMEOUT_SECS = 86400
-
-[DIRECTORIES]
-BACKUP_DIR = /home/user/mnt/dir/
-BACKUP.D_DIR = /home/user/.config/dar-backup/backup.d/
-TEST_RESTORE_DIR = /tmp/dar-backup/restore/
-
-[AGE]
-# age settings are in days
-DIFF_AGE = 100
-INCR_AGE = 40
-
-[PAR2]
-ERROR_CORRECTION_PERCENT = 5
-# False means "do not generate par2 redundancy files"
-ENABLED = True  
-
-[PREREQ]
-# SCRIPT_1 = /home/user/programmer/dar-backup/prereq/mount-server.sh
-# SCRIPT_2 = <something>
-# ...
-
-[POSTREQ]
-# SCRIPT_1 = /home/user/programmer/dar-backup/postreq/umount-server.sh
-# SCRIPT_2 = <something>
-# ...
-
-````
-
-### 2
-
-Put your backup definitions in the directory $BACKUP.D_DIR (defined in the config file)
-
-The name of the file is the `backup definition` name.
-
-Make as many backup definitions as you need. Run them all in one go, or run one at a time using the `-d` option.
-
-The `dar` [documentation](http://dar.linux.free.fr/doc/man/dar.html#COMMANDS%20AND%20OPTIONS) has good information on file selection.
-
-Example of backup definition for a home directory
-
-```` code
-
-# Switch to ordered selection mode, which means that the following
-# options will be considered top to bottom
- -am
-
-
-# Backup Root dir
- -R /home/user
-
-# Directories to backup below the Root dir
-# if you want to take a backup of /home/user/Documents only, uncomment next line
-#  -g Documents 
-
-# Some directories to exclude below the Root dir (here Root directory is `/home/user` as set in the -R option)
- -P mnt
- -P tmp
- -P .cache
- -P .config/Code/CachedData
- -P .config/Code/Cache
- -P ".config/Code/Service Worker"
- -P .config/Code/logs
- -P snap/firefox/common/.cache
- 
-# compression level
- -z5
-
- # no overwrite, if you rerun a backup, 'dar' halts and asks what to do (and Quits due to the "-Q" given by dar-backup)
- -n
- 
- # size of each slice in the archive
- --slice 10G
-
-
-# bypass directores marked as cache directories
-# http://dar.linux.free.fr/doc/Features.html
---cache-directory-tagging
-````
-
-### 3
+### 1 - installation
 
 Installation is currently in a venv. These commands are installed in the venv:
 
@@ -897,6 +807,7 @@ Installation is currently in a venv. These commands are installed in the venv:
 - cleanup
 - manager
 - clean-log
+- installer
 
 To install, create a venv and run pip:
 
@@ -918,25 +829,48 @@ Typing `db` at the command line gives this
 
 ```` bash
 (venv) user@machine:~$ db
-dar-backup 0.6.9
+dar-backup 0.6.12
 dar-backup.py source code is here: https://github.com/per2jensen/dar-backup
 Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
 THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 See section 15 and section 16 in the supplied "LICENSE" file.
 ````
 
-### 4
+### 2 - configuration
+
+The dar-backup installer is non-destructive and stops if some of the default directories exist.
+
+Run the installer
+
+```` bash
+installer --install
+````
+
+The output is
+
+```` text
+Directories created: `/home/user/dar-backup/` and `/home/user/.config/dar-backup`
+Config file deployed to /home/user/.config/dar-backup/dar-backup.conf
+Default backup definition deployed to /home/user/.config/dar-backup/backup.d/default
+1. Now run `manager --create` to create the catalog database.
+2. Then you can run `dar-backup --full-backup` to create a backup.
+3. List backups with `dar-backup --list`
+4. List contents of a backup with `dar-backup --list-contents <backup-name>`
+````
+
+### 3 - generate catalog databases
+
+Generate the archive catalog database(s).
 
-Generate the archive catalog database(s). 
 `dar-backup` expects the catalog databases to be in place, it does not automatically create them (by design)
 
 ```` bash
-manager --create-db
+manager --create
 ````
 
-### 5
+### 4 - do FULL backups
 
-You are ready to do backups of all your backup definitions, if your backup definitions are 
+You are ready to do backups of all your backup definitions, if your backup definitions are
 in place in BACKUP.D_DIR (see config file)
 
 ```` bash
@@ -953,9 +887,9 @@ If you want a backup of a single definition, use the `-d <backup definition>` op
 dar-backup --full-backup -d <your backup definition>
 ````
 
-### 6
+### 5 - deactivate venv
 
-Deactivate the virtual environment
+Deactivate the virtual environment (venv)
 
 ```` bash
 deactivate
@@ -1311,14 +1245,92 @@ dar-backup --restore <archive_name>  --selection "-X '*.xmp' -I '*2024-06-16*' -
 deactivate
 ```
 
+### restore test fails with exit code 4
+
+"dar" in newer versions emits a question about file ownership, which is "answered" with a "no" via the "-Q" option. That in turn leads to an error code 4.
+
+Thus the dar option "--comparison-field=ignore-owner" has been placed in the supplied .darrc file (located in the virtual environment where dar-backup is installed). 
+
+This causes dar to restore without an error.
+
+It is a good option when using dar as a non-privileged user.
+
+### restore test fails with exit code 5
+
+If exit code 5 is emitted on the restore test, FSA (File System specific Attributes) could be the cause.
+
+That (might) occur if you backup a file stored on one type of filesystem, and restore it on another type.
+My home directory is on a btrfs filesystem, while /tmp (for the restore test) is on zfs.
+
+The restore test can result in an exit code 5, due to the different filesystems used. In order to avoid the errors, the "option "--fsa-scope none" can be used. That will restult in FSA's not being restored.
+
+If you need to use this option, un-comment it in the .darrc file (located in the virtual environment where dar-backup is installed)
+
+## Par2
+
+### Par2 to verify/repair
+
+You can run a par2 verification on an archive like this:
+
+```` bash
+for file in <archive>*.dar.par2; do
+  par2 verify "$file"
+done
+````
+
+if there are problems with a slice, try to repair it like this:
+
+```` bash
+  par2 repair <archive>.<slice number>.dar.par2
+````
+
+### Par2 create redundancy files
+
+If you have merged archives, you will need to create the .par2 redundency files manually.
+Here is an example
+
+```` bash
+for file in <some-archive>_FULL_yyyy-mm-dd.*; do
+  par2 c -r5 -n1 "$file"
+done
+````
+
+where "c" is create, -r5 is 5% redundency and -n1 is 1 redundency file
+
 ## Points of interest
 
+### Merge FULL with DIFF, creating new FULL
+
+Over time, the DIFF archives become larger and larger. At some point one wishes to create a new FULL archive to do DIFF's on.
+One way to do that, is to let dar create a FULL archive from scratch, another is to merge a FULL archive with a DIFF, and from there do DIFF's until they once again gets too large for your taste.
+
+I do backups of my homedir. Here it is shown how a FULL archive is merged with a DIFF, creating a new FULL archive.
+
+```` bash
+dar --merge pj-homedir_FULL_2021-09-12  -A pj-homedir_FULL_2021-06-06  -@pj-homedir_DIFF_2021-08-29 -s 12G
+
+# test the new FULL archive
+dar -t pj-homedir_FULL_2021-09-12
+
+# create Par2 redundancy files
+for file in pj-homedir_FULL_yyyy-mm-dd.*.dar; do
+  par2 c -r5 -n1 "$file"
+done
+
+````
+
 ### dar manager databases
 
 `dar-backup` now saves archive catalogs in dar catalog databases.
 
 This makes it easier to restore to a given date when having many FULL, DIFF and INCR archives.
 
+### Performance tip due to par2
+
+This [dar benchmark page](https://dar.sourceforge.io/doc/benchmark.html) has an interesting note on the slice size.
+
+Slice size should be smaller than available RAM, apparently a large performance hit can be avoided keeping the the par2 data in memory.
+
 ### .darrc sets -vd -vf (since v0.6.4)
 
 These .darrc settings make `dar` print the current directory being processed (-vd) and some stats after (-vf)

dar_backup-0.6.12.dist-info/RECORD

@@ -0,0 +1,16 @@
+dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
+dar_backup/__about__.py,sha256=AY38r3HUSyMqkCPP-vaHASQmjQF5-PRm11j_QYtx28w,22
+dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dar_backup/clean_log.py,sha256=cGhtKYnQJ2ceNQfw5XcCln_WNBasbmlfhO3kRydjDNk,5196
+dar_backup/cleanup.py,sha256=NaZMrTdtYv4uJSw3jwDQWc5F5jMfIdfIQdrcPGAVcnM,11439
+dar_backup/config_settings.py,sha256=uicCq6FnpxPFzbv7xfYSXNnQf1tfLk1Z3VIO9M71fsE,4659
+dar_backup/dar-backup.conf,sha256=-wXqP4vj5TS7cCfMJN1nbk-1Sqkq00Tg22ySQXynUF4,902
+dar_backup/dar_backup.py,sha256=Cye8gS0E0mNKaUzcjqsdsuTyyeZYCspRMBIdcGbsEik,33171
+dar_backup/installer.py,sha256=0TgC_O-T7Y3sLn_NIQ9lBYt8GJqLZzxPqkmbjElfgkM,4491
+dar_backup/manager.py,sha256=MkrB0AL0MefE_cUAvdzQb_0bzvqsPmYi5s0M3EPw-z8,21379
+dar_backup/util.py,sha256=E-sEBQZY1hmdeVx5xNE22zKQ0BXDee1eI9F1-w7Fq1Q,15756
+dar_backup-0.6.12.dist-info/METADATA,sha256=YEbPNJr_ntP03BVKSACo2MU-Qko_9quyzmuqrhryBz0,69768
+dar_backup-0.6.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dar_backup-0.6.12.dist-info/entry_points.txt,sha256=Z7P5BUbhtJxo8_nB9qNIMay2eGDbsMKB3Fjwv3GMa4g,202
+dar_backup-0.6.12.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dar_backup-0.6.12.dist-info/RECORD,,

{dar_backup-0.6.10.dist-info → dar_backup-0.6.12.dist-info}/entry_points.txt

@@ -2,4 +2,5 @@
 clean-log = dar_backup.clean_log:main
 cleanup = dar_backup.cleanup:main
 dar-backup = dar_backup.dar_backup:main
+installer = dar_backup.installer:main
 manager = dar_backup.manager:main

dar_backup-0.6.10.dist-info/RECORD

@@ -1,14 +0,0 @@
-dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
-dar_backup/__about__.py,sha256=G-zfU8sHQRTZnZK0te1cvbxqeLhT1z86XeAYFNUON6Q,22
-dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dar_backup/clean_log.py,sha256=VXKA2BMyQmaC6R08Bq9a3wP3mczdFb_moy6HkL-mnF8,5176
-dar_backup/cleanup.py,sha256=9yEdRR84XPtEvBGc2QfwGBQl2tdTPttjetHeiSc_TsM,11419
-dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
-dar_backup/dar_backup.py,sha256=hDy7aXU-XiWOtW40Pxql441liNkSYKGU76eOwy8m7fU,32714
-dar_backup/manager.py,sha256=HDa8eYF89QFhlBRR4EWRzzmswOW00S_w8ToZ5SARO_o,21359
-dar_backup/util.py,sha256=SSSJYM9lQZfubhTUBlX1xDGWmCpYEF3ePARmlY544xM,11283
-dar_backup-0.6.10.dist-info/METADATA,sha256=CEgbqp93sB_cPnEDWuLn7gphZjdrn4zP_5dShC2Buv8,67980
-dar_backup-0.6.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dar_backup-0.6.10.dist-info/entry_points.txt,sha256=p6c4uQLjlTIVP1Od2iorGefrVUH0IWZdFRMl63mNaRg,164
-dar_backup-0.6.10.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dar_backup-0.6.10.dist-info/RECORD,,