dar-backup 0.6.11__py3-none-any.whl → 0.6.13__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.11"
+__version__ = "0.6.13"

dar_backup/clean_log.py

@@ -122,7 +122,7 @@ def main():
 
     args = parser.parse_args()
 
-    config_settings = ConfigSettings(os.path.expanduser(args.config_file))
+    config_settings = ConfigSettings(os.path.expanduser(os.path.expandvars(args.config_file)))
 
     if not args.file:
         args.file = [config_settings.logfile_location]

dar_backup/cleanup.py

@@ -171,7 +171,7 @@ def main():
     parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
     args = parser.parse_args()
 
-    args.config_file = os.path.expanduser(args.config_file)
+    args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
     
 
     if args.version:
@@ -181,7 +181,11 @@ def main():
     config_settings = ConfigSettings(args.config_file)
 
     start_time=int(time())
-    logger = setup_logging(config_settings.logfile_location, args.log_level, args.log_stdout)
+
+#    command_output_log = os.path.join(config_settings.logfile_location.removesuffix("dar-backup.log"), "dar-backup-commands.log")
+    command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
+    logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout)
+
 
     logger.info(f"=====================================")
     logger.info(f"cleanup.py started, version: {about.__version__}")

dar_backup/config_settings.py

@@ -1,8 +1,11 @@
-from dataclasses import dataclass, field
+
 import configparser
-from pathlib import Path
-import sys
 import logging
+import sys
+
+from dataclasses import dataclass, field, fields
+from os.path import expandvars, expanduser
+from pathlib import Path
 
 @dataclass
 class ConfigSettings:
@@ -20,21 +23,35 @@ class ConfigSettings:
         backup_d_dir (str): The directory for backup.d.
         diff_age (int): The age for differential backups before deletion.
         incr_age (int): The age for incremental backups before deletion.
+        error_correction_percent (int): The error correction percentage for PAR2.
+        par2_enabled (bool): Whether PAR2 is enabled.
     """
 
-    def __init__(self, config_file: str):
+    config_file: str
+    logfile_location: str = field(init=False)
+    max_size_verification_mb: int = field(init=False)
+    min_size_verification_mb: int = field(init=False)
+    no_files_verification: int = field(init=False)
+    command_timeout_secs: int = field(init=False)
+    backup_dir: str = field(init=False)
+    test_restore_dir: str = field(init=False)
+    backup_d_dir: str = field(init=False)
+    diff_age: int = field(init=False)
+    incr_age: int = field(init=False)
+    error_correction_percent: int = field(init=False)
+    par2_enabled: bool = field(init=False)
+
+    def __post_init__(self):
         """
-        Initializes the ConfigSettings instance by reading the specified configuration file.
-        
-        Args:
-            config_file (str): The path to the configuration file.
+        Initializes the ConfigSettings instance by reading the specified configuration file
+        and expands environment variables for all string fields.
         """
-        if config_file is None:
+        if self.config_file is None:
             raise ValueError("`config_file` must be specified.")
         
         self.config = configparser.ConfigParser()
         try:
-            self.config.read(config_file)
+            self.config.read(self.config_file)
             self.logfile_location = self.config['MISC']['LOGFILE_LOCATION']
             self.max_size_verification_mb = int(self.config['MISC']['MAX_SIZE_VERIFICATION_MB'])
             self.min_size_verification_mb = int(self.config['MISC']['MIN_SIZE_VERIFICATION_MB'])
@@ -46,12 +63,18 @@ class ConfigSettings:
             self.diff_age = int(self.config['AGE']['DIFF_AGE'])
             self.incr_age = int(self.config['AGE']['INCR_AGE'])
             self.error_correction_percent = int(self.config['PAR2']['ERROR_CORRECTION_PERCENT'])
-            self.par2_enabled = bool(self.config['PAR2']['ENABLED'])
+            self.par2_enabled = self.config['PAR2']['ENABLED'].lower() in ('true', '1', 'yes')
+            
             # Ensure the directories exist
             Path(self.backup_dir).mkdir(parents=True, exist_ok=True)
             Path(self.test_restore_dir).mkdir(parents=True, exist_ok=True)
             Path(self.backup_d_dir).mkdir(parents=True, exist_ok=True)
 
+            # Expand environment variables for all string fields
+            for field in fields(self):
+                if isinstance(getattr(self, field.name), str):
+                    setattr(self, field.name, expanduser(expandvars(getattr(self, field.name))))
+
         except FileNotFoundError as e:
             logging.error(f"Configuration file not found: {self.config_file}")
             logging.error(f"Error details: {e}")

dar_backup/dar-backup.conf

@@ -0,0 +1,33 @@
+# This config file is intended to demo `dar-backup`.
+#
+# The `installer` puts it in ~/.config/dar-backup/dar-backup.conf
+
+[MISC]
+LOGFILE_LOCATION = ~/dar-backup/dar-backup.log
+MAX_SIZE_VERIFICATION_MB = 20
+MIN_SIZE_VERIFICATION_MB = 1
+NO_FILES_VERIFICATION = 5
+# timeout in seconds for backup, test, restore and par2 operations
+# The author has such `dar` tasks running for 10-15 hours on the yearly backups, so a value of 24 hours is used.
+# If a timeout is not specified when using the util.run_command(), a default timeout of 30 secs is used.
+COMMAND_TIMEOUT_SECS = 86400
+
+[DIRECTORIES]
+BACKUP_DIR = ~/dar-backup/backups
+BACKUP.D_DIR = ~/.config/dar-backup/backup.d/
+TEST_RESTORE_DIR = ~/dar-backup/restore/
+
+[AGE]
+# age settings are in days
+DIFF_AGE = 100
+INCR_AGE = 40
+
+[PAR2]
+ERROR_CORRECTION_PERCENT = 5
+ENABLED = True
+
+[PREREQ]
+#SCRIPT_1 = <pre-script 1>
+
+[POSTREQ]
+#SCRIPT_1 = <post-script 1>

dar_backup/dar_backup.py

@@ -7,14 +7,18 @@ import os
 import random
 import re
 import shlex
+import shutil
 import subprocess
-import sys
 import xml.etree.ElementTree as ET
-
+import tempfile
 
 from argparse import ArgumentParser
 from datetime import datetime
 from pathlib import Path
+from sys import exit
+from sys import stderr
+from sys import argv
+from sys import version_info
 from time import time
 from typing import List
 
@@ -27,6 +31,7 @@ from dar_backup.util import BackupError
 from dar_backup.util import RestoreError
 
 
+RESULT = True
 logger = None
 
 def generic_backup(type: str, command: List[str], backup_file: str, backup_definition: str, darrc: str,  config_settings: ConfigSettings, args: argparse.Namespace):
@@ -96,12 +101,12 @@ def find_files_with_paths(xml_root: ET.Element):
     current_path = []
 
     for elem in xml_root.iter():
-        if elem.tag == "directory":
+        if elem.tag == "Directory":
             current_path.append(elem.get('name'))
-        elif elem.tag == "file":
+        elif elem.tag == "File":
             file_path = ("/".join(current_path + [elem.get('name')]), elem.get('size'))
             files.append(file_path)
-        elif elem.tag == "directory" and elem.get('name') in current_path:
+        elif elem.tag == "Directory" and elem.get('Name') in current_path:
             current_path.pop()
 
     return files
@@ -189,10 +194,10 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
         logger.info(f"No files between {config_settings.min_size_verification_mb}MB and {config_settings.max_size_verification_mb}MB for verification, skipping")
         return result
 
+    # find Root path in backup definition
     with open(backup_definition, 'r') as f:
         backup_definition_content = f.readlines()
         logger.debug(f"Backup definition: '{backup_definition}', content:\n{backup_definition_content}")
-    # Initialize a variable to hold the path after "-R"
     root_path = None
     for line in backup_definition_content:
         line = line.strip()
@@ -200,7 +205,10 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
             root_path = line.split("-R", 1)[1].strip()
             break
     if root_path is None:
-        logger.warning("No Root (-R) path specified in the backup definition file.")
+        msg = f"No Root (-R) path found in the backup definition file: '{backup_definition}', restore verification skipped"
+        raise BackupError(msg)
+
+
 
     no_files_verification = config_settings.no_files_verification
     if len(files) < config_settings.no_files_verification:
@@ -407,7 +415,7 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                     logger.info(f"Using alternate reference archive: {latest_base_backup}")
                     if not os.path.exists(latest_base_backup + '.1.dar'):
                         logger.error(f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exiting.")
-                        sys.exit(1)
+                        exit(1)
                 else:
                     base_backups = sorted(
                         [f for f in os.listdir(config_settings.backup_dir) if f.startswith(f"{backup_definition}_{base_backup_type}_") and f.endswith('.1.dar')],
@@ -435,9 +443,10 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
             logger.info("Generate par2 redundancy files.")
             generate_par2_files(backup_file, config_settings, args)
             logger.info("par2 files completed successfully.")
-            
 
         except Exception as e:
+            global RESULT
+            RESULT = False
             logger.exception(f"Error during {backup_type} backup process, continuing to next backup definition.")
 
 
@@ -491,8 +500,46 @@ def generate_par2_files(backup_file: str, config_settings: ConfigSettings, args)
 
 
 
+def filter_darrc_file(darrc_path):
+    """
+    Filters the .darrc file to remove lines containing the options: -vt, -vs, -vd, -vf, and -va.
+    The filtered version is stored in a uniquely named file in the home directory of the user running the script.
+    The file permissions are set to 440.
+    
+    :param darrc_path: Path to the original .darrc file.
+    :return: Path to the filtered .darrc file.
+    """
+    # Define options to filter out
+    options_to_remove = {"-vt", "-vs", "-vd", "-vf", "-va"}
+
+    # Get the user's home directory
+    home_dir = os.path.expanduser("~")
+
+    # Create a unique file name in the home directory
+    filtered_darrc_path = os.path.join(home_dir, f"filtered_darrc_{next(tempfile._get_candidate_names())}.darrc")
+
+    try:
+        with open(darrc_path, "r") as infile, open(filtered_darrc_path, "w") as outfile:
+            for line in infile:
+                # Check if any unwanted option is in the line
+                if not any(option in line for option in options_to_remove):
+                    outfile.write(line)
+        
+        # Set file permissions to 440 (read-only for owner and group, no permissions for others)
+        os.chmod(filtered_darrc_path, 0o440)
+
+        return filtered_darrc_path
+
+    except Exception as e:
+        # If anything goes wrong, clean up the temp file if it was created
+        if os.path.exists(filtered_darrc_path):
+            os.remove(filtered_darrc_path)
+        raise RuntimeError(f"Error filtering .darrc file: {e}")
+
+
+
 def show_version():
-    script_name = os.path.basename(sys.argv[0])
+    script_name = os.path.basename(argv[0])
     print(f"{script_name} {about.__version__}") 
     print(f"dar-backup.py source code is here: https://github.com/per2jensen/dar-backup")
     print('''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
@@ -577,14 +624,14 @@ def requirements(type: str, config_setting: ConfigSettings):
 
 
 def main():
-    global logger 
+    global logger, RESULT 
 
     MIN_PYTHON_VERSION = (3, 9)
-    if sys.version_info < MIN_PYTHON_VERSION:
-        sys.stderr.write(f"Error: This script requires Python {'.'.join(map(str, MIN_PYTHON_VERSION))} or higher.\n")
-        sys.exit(1)
+    if version_info < MIN_PYTHON_VERSION:
+        stderr.write(f"Error: This script requires Python {'.'.join(map(str, MIN_PYTHON_VERSION))} or higher.\n")
+        exit(1)
 
-    parser = argparse.ArgumentParser(description="Backup and verify using dar backup definitions.")
+    parser = argparse.ArgumentParser(description="Backup, verify & redundancy using dar and par2.")
     parser.add_argument('-F', '--full-backup', action='store_true', help="Perform a full backup.")
     parser.add_argument('-D', '--differential-backup', action='store_true', help="Perform differential backup.")
     parser.add_argument('-I', '--incremental-backup', action='store_true', help="Perform incremental backup.")
@@ -600,43 +647,55 @@ def main():
     parser.add_argument('-r', '--restore', type=str, help="Restore specified archive.")
     parser.add_argument('--restore-dir',   type=str, help="Directory to restore files to.")
     parser.add_argument('--verbose', action='store_true', help="Print various status messages to screen")
+    parser.add_argument('--suppress-dar-msg', action='store_true', help="cancel dar options in .darrc: -vt, -vs, -vd, -vf and -va")
     parser.add_argument('--log-level', type=str, help="`debug` or `trace`", default="info")
     parser.add_argument('--log-stdout', action='store_true', help='also print log messages to stdout')
     parser.add_argument('--do-not-compare', action='store_true', help="do not compare restores to file system")
     parser.add_argument('-v', '--version', action='store_true', help="Show version and license information.")
     args = parser.parse_args()
 
-    args.config_file = os.path.expanduser(args.config_file)
-    config_settings = ConfigSettings(args.config_file)
-
     if args.version:
         show_version()
-        sys.exit(0)
+        exit(0)
     elif args.examples:
         show_examples()
-        sys.exit(0)
-
-    logger = setup_logging(config_settings.logfile_location, args.log_level, args.log_stdout)
-
-    if not args.darrc:
-        current_script_dir = os.path.dirname(os.path.abspath(__file__))
-        args.darrc = os.path.join(current_script_dir, ".darrc")
-
-    if os.path.exists(args.darrc) and os.path.isfile(args.darrc):
-        logger.debug(f"Using .darrc: {args.darrc}")                
-    else:
-        logger.error(f"Supplied .darrc: '{args.darrc}' does not exist or is not a file")
+        exit(0)
 
     if not args.config_file:
-        logger.error(f"Config file not specified, exiting")
-        sys.exit(1) 
+        print(f"Config file not specified, exiting", file=stderr)
+        exit(1) 
     
-    if not os.path.exists(args.config_file):
-        logger.error(f"Config file {args.config_file} does not exist.")
-        sys.exit(1)
+    config_settings_path = os.path.expanduser(os.path.expandvars(args.config_file))
+    if not os.path.exists(config_settings_path):
+        print(f"Config file {args.config_file} does not exist.", file=stderr)
+        exit(127)
+
+    args.config_file = config_settings_path
+    config_settings = ConfigSettings(args.config_file)
 
+    command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
+    if command_output_log == config_settings.logfile_location:
+        print(f"Error: logfile_location in {args.config_file} does not end at 'dar-backup.log', exiting", file=stderr)
+
+    logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout)
 
     try:
+        if not args.darrc:
+            current_script_dir = os.path.dirname(os.path.abspath(__file__))
+            args.darrc = os.path.join(current_script_dir, ".darrc")
+
+        darrc_file = os.path.expanduser(os.path.expandvars(args.darrc))
+        if os.path.exists(darrc_file) and os.path.isfile(darrc_file):
+            logger.debug(f"Using .darrc: {args.darrc}")                
+        else:
+            logger.error(f"Supplied .darrc: '{args.darrc}' does not exist or is not a file, exiting", file=stderr)
+            exit(127)
+
+        if args.suppress_dar_msg:
+            logger.info("Suppressing dar messages: -vt, -vs, -vd, -vf and -va")
+            args.darrc = filter_darrc_file(args.darrc)
+            logger.debug(f"Filtered .darrc file saved at: {args.darrc}")
+
         start_time=int(time())
         logger.info(f"=====================================")
         logger.info(f"dar-backup.py started, version: {about.__version__}")
@@ -664,14 +723,13 @@ def main():
         args.verbose and (print(f"PAR2 enabled:      {config_settings.par2_enabled}"))
         args.verbose and (print(f"--do-not-compare:  {args.do_not_compare}"))
 
-
         # sanity check
         if args.backup_definition and not os.path.exists(os.path.join(config_settings.backup_d_dir, args.backup_definition)):
             logger.error(f"Backup definition: '{args.backup_definition}' does not exist, exiting")
-            sys.exit(1)
+            exit(127)
         if args.backup_definition and '_' in args.backup_definition:
             logger.error(f"Backup definition: '{args.backup_definition}' contains '_', exiting")
-            sys.exit(1)
+            exit(1)
 
 
         requirements('PREREQ', config_settings)
@@ -694,19 +752,24 @@ def main():
 
         requirements('POSTREQ', config_settings)
 
-        args.verbose and print("\033[1m\033[32mSUCCESS\033[0m No errors encountered")
-        sys.exit(0)
     except Exception as e:
         logger.exception("An error occurred")
         logger.error("Exception details:", exc_info=True)
         args.verbose and print("\033[1m\033[31mErrors\033[0m encountered")
-        sys.exit(1) 
+        RESULT = False
     finally:
         end_time=int(time())
         logger.info(f"END TIME: {end_time}")
-
-#    error_lines = extract_error_lines(config_settings.logfile_location, start_time, end_time)
+        # Clean up
+        if args.suppress_dar_msg and os.path.exists(args.darrc) and os.path.dirname(args.darrc) == os.path.expanduser("~"):
+            if args.darrc.startswith("filtered_darrc_"):
+                os.remove(args.darrc)
+                logger.info(f"Removed filtered .darrc: {args.darrc}")
 
 
+    if RESULT:
+        exit(0)
+    else:
+        exit(1)
 if __name__ == "__main__":
     main()

dar_backup/installer.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+installer.py source code is here: https://github.com/per2jensen/dar-backup/tree/main/v2/src/dar_backup/installer.py
+This script is part of dar-backup, a backup solution for Linux using dar and systemd.
+
+Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, 
+not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See section 15 and section 16 in the supplied "LICENSE" file
+
+This script can be used to configure dar-backup on your system.
+It is non-destructive and will not overwrite any existing files or directories.
+"""
+
+import argparse
+import os
+import shutil
+import sys 
+
+from . import __about__ as about
+from pathlib import Path
+
+LICENSE = '''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See section 15 and section 16 in the supplied "LICENSE" file.'''
+
+CONFIG_DIR = os.path.expanduser("~/.config/dar-backup")
+DAR_BACKUP_DIR = os.path.expanduser("~/dar-backup/")
+
+BACKUP_DEFINITION = '''
+# Demo of a `dar-backup` definition file
+# This back definition file configures a backup of ~/.config/dar-backup
+# `dar-backup` puts the backups in ~/dar-backup/backups
+# ------------------------------------------------------------------------
+
+# Switch to ordered selection mode, which means that the following options
+# will be considered top to bottom
+-am
+
+# Backup Root dir
+-R @@HOME_DIR@@
+
+# Directories to backup below the Root dir
+-g .config/dar-backup
+
+# Examples of directories to exclude below the Root dir
+ -P mnt
+ -P .private
+ -P .cache   
+
+# compression level
+ -z5
+
+ # no overwrite, if you rerun a backup, 'dar' halts and asks what to do
+ -n
+ 
+ # size of each slice in the archive
+ --slice 10G
+
+# bypass directores marked as cache directories
+# http://dar.linux.free.fr/doc/Features.html
+--cache-directory-tagging
+'''
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Set up `dar-backup` on your system.",
+    )
+    parser.add_argument(
+        "-i", "--install",
+        action="store_true",
+        help="Deploy a simple config file, use ~/dar-backup/ for log file, archives and restore tests."
+    )
+
+    parser.add_argument(
+        "-v", "--version",
+        action="version",
+        version=f"%(prog)s version {about.__version__}, {LICENSE}"
+    )
+
+    args = parser.parse_args()
+
+    if args.install:
+        errors = []
+        if os.path.exists(CONFIG_DIR):
+            errors.append(f"Config directory '{CONFIG_DIR}' already exists.")
+
+        if os.path.exists(DAR_BACKUP_DIR):
+            errors.append(f"Directory '{DAR_BACKUP_DIR}' already exists.")
+
+        if len(errors) > 0:
+            for error in errors:
+                print(f"Error: {error}")
+            sys.exit(1) 
+
+        os.makedirs(DAR_BACKUP_DIR, exist_ok=False)
+        os.makedirs(os.path.join(DAR_BACKUP_DIR, "backups"), exist_ok=False)
+        os.makedirs(os.path.join(DAR_BACKUP_DIR, "restore"), exist_ok=False)
+        os.makedirs(CONFIG_DIR, exist_ok=False)
+        os.makedirs(os.path.join(CONFIG_DIR, "backup.d"), exist_ok=False)
+        print(f"Directories created: `{DAR_BACKUP_DIR}` and `{CONFIG_DIR}`")
+
+        script_dir = Path(__file__).parent
+        source_file = script_dir / "dar-backup.conf"
+        destination_file = Path(CONFIG_DIR) / "dar-backup.conf"
+        shutil.copy(source_file, destination_file)
+        print(f"Config file deployed to {destination_file}")
+
+
+        backup_definition = BACKUP_DEFINITION.replace("@@HOME_DIR@@", os.path.expanduser("~"))
+        with open(os.path.join(CONFIG_DIR, "backup.d", "default"), "w") as f:
+            f.write(backup_definition)
+        print(f"Default backup definition file deployed to {os.path.join(CONFIG_DIR, 'backup.d', 'default')}")
+        print("1. Now run `manager --create` to create the catalog database.")
+        print("2. Then you can run `dar-backup --full-backup` to create a backup.")
+        print("3. List backups with `dar-backup --list`")
+        print("4. List contents of a backup with `dar-backup --list-contents <backup-name>`")
+
+
+if __name__ == "__main__":
+    main()

dar_backup/manager.py

@@ -335,7 +335,7 @@ def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> in
     """
     backup_def = backup_def_from_archive(archive)
     database_path = os.path.join(config_settings.backup_dir, f"{backup_def}{DB_SUFFIX}")
-    cat_no = cat_no_for_name(archive, config_settings)
+    cat_no:int = cat_no_for_name(archive, config_settings)
     if cat_no >= 0:
         command = ['dar_manager', '--base', database_path, "--delete", str(cat_no)]
         process: CommandResult = run_command(command)
@@ -395,12 +395,16 @@ See section 15 and section 16 in the supplied "LICENSE" file.''')
         sys.exit(0)
 
     # setup logging
-    args.config_file = os.path.expanduser(args.config_file)
+    args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
     config_settings = ConfigSettings(args.config_file)
     if not os.path.dirname(config_settings.logfile_location):
         print(f"Directory for log file '{config_settings.logfile_location}' does not exist, exiting")
         sys.exit(1) 
-    logger = setup_logging(config_settings.logfile_location, args.log_level, args.log_stdout)
+
+#    command_output_log = os.path.join(config_settings.logfile_location.removesuffix("dar-backup.log"), "dar-backup-commands.log")
+    command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
+    logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout)
+
 
     start_time=int(time())
     logger.info(f"=====================================")