dar-backup 0.6.9__py3-none-any.whl → 0.6.11__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.9"
+__version__ = "0.6.11"

dar_backup/dar_backup.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import datetime
 import filecmp
 
 import os
@@ -81,30 +80,31 @@ def generic_backup(type: str, command: List[str], backup_file: str, backup_defin
         raise BackupError(f"Unexpected error during backup: {e}") from e
  
 
-# Function to recursively find <File> tags and build their full paths
-def find_files_with_paths(element: ET, current_path=""):
+
+def find_files_with_paths(xml_root: ET.Element):
     """
-    Recursively finds files within a directory element and returns a list of file paths with their sizes.
+    Finds files within an XML element and returns a list of file paths with their sizes.
 
     Args:
-        element (Element): The directory element to search within.
-        current_path (str, optional): The current path of the directory element. Defaults to "".
+        xml_root (Element): The root element of the XML.
 
     Returns:
         list: A list of tuples containing file paths and their sizes.
     """
-    logger.debug(f"Recursively generate list of tuples with file paths and sizes for File elements in dar xml output")
+    logger.debug("Generating list of tuples with file paths and sizes for File elements in dar xml output")
     files = []
-    if element.tag == "Directory":
-        current_path = f"{current_path}/{element.get('name')}"
-    for child in element:
-        if child.tag == "File":
-            file_path = (f"{current_path}/{child.get('name')}", child.get('size'))  # tuple (filepath, size)
+    current_path = []
+
+    for elem in xml_root.iter():
+        if elem.tag == "directory":
+            current_path.append(elem.get('name'))
+        elif elem.tag == "file":
+            file_path = ("/".join(current_path + [elem.get('name')]), elem.get('size'))
             files.append(file_path)
-        elif child.tag == "Directory":
-            files.extend(find_files_with_paths(child, current_path))
-    return files
+        elif elem.tag == "directory" and elem.get('name') in current_path:
+            current_path.pop()
 
+    return files
 
 
 def find_files_between_min_and_max_size(backed_up_files: list[(str, str)], config_settings: ConfigSettings):
@@ -238,20 +238,20 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
         restore_dir (str): The directory where the backup should be restored to.
         selection (str, optional): A selection criteria to restore specific files or directories. Defaults to None.
     """
-    backup_file = os.path.join(config_settings.backup_dir, backup_name)
-    command = ['dar', '-x', backup_file, '-Q', '-D']
-    if restore_dir:
-        if not os.path.exists(restore_dir):
-            os.makedirs(restore_dir)
-        command.extend(['-R', restore_dir])
-    else:
-        raise RestoreError("Restore directory ('-R <dir>') not specified")
-    if selection:
-        selection_criteria = shlex.split(selection)
-        command.extend(selection_criteria)
-    command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
-    logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
     try:
+        backup_file = os.path.join(config_settings.backup_dir, backup_name)
+        command = ['dar', '-x', backup_file, '-Q', '-D']
+        if restore_dir:
+            if not os.path.exists(restore_dir):
+                os.makedirs(restore_dir)
+            command.extend(['-R', restore_dir])
+        else:
+            raise RestoreError("Restore directory ('-R <dir>') not specified")
+        if selection:
+            selection_criteria = shlex.split(selection)
+            command.extend(selection_criteria)
+        command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
+        logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
         process = run_command(command, config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"Restore completed successfully to: '{restore_dir}'")
@@ -260,6 +260,9 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
             raise RestoreError(str(process))
     except subprocess.CalledProcessError as e:
         raise RestoreError(f"Restore command failed: {e}") from e
+    except OSError as e:
+        logger.error(f"Failed to create restore directory: {e}")
+        raise RestoreError("Could not create restore directory")
     except Exception as e:
         raise RestoreError(f"Unexpected error during restore: {e}") from e
 
@@ -285,7 +288,7 @@ def get_backed_up_files(backup_name: str, backup_dir: str):
         # Parse the XML data
         root = ET.fromstring(process.stdout)
         output = None  # help gc
-        # Extract full paths and file size for all <File> elements
+        # Extract full paths and file size for all <file> elements
         file_paths = find_files_with_paths(root)
         root = None # help gc
         logger.trace(str(process))
@@ -624,6 +627,14 @@ def main():
     else:
         logger.error(f"Supplied .darrc: '{args.darrc}' does not exist or is not a file")
 
+    if not args.config_file:
+        logger.error(f"Config file not specified, exiting")
+        sys.exit(1) 
+    
+    if not os.path.exists(args.config_file):
+        logger.error(f"Config file {args.config_file} does not exist.")
+        sys.exit(1)
+
 
     try:
         start_time=int(time())

dar_backup/util.py

@@ -14,6 +14,7 @@ import os
 import re
 import subprocess
 import shlex
+import shutil
 import sys
 import threading
 import traceback
@@ -123,10 +124,113 @@ def  _stream_reader(pipe, log_func, output_accumulator: List[str]):
             log_func(stripped_line)  # Log the output in real time
 
 
+
 def run_command(command: List[str], timeout: int = 30) -> CommandResult:
     """
     Executes a given command via subprocess, logs its output in real time, and returns the result.
 
+    Args:
+        command (list): The command to be executed, represented as a list of strings.
+        timeout (int): The maximum time in seconds to wait for the command to complete. Defaults to 30 seconds.
+
+    Returns:
+        A CommandResult NamedTuple with the following properties:
+        - process: subprocess.CompletedProcess
+        - stdout: str: The full standard output of the command.
+        - stderr: str: The full standard error of the command.
+        - returncode: int: The return code of the command.
+        - timeout: int: The timeout value in seconds used to run the command.
+        - command: list[str]: The command executed.
+
+    Logs:
+        - Logs standard output (`stdout`) in real-time at the INFO log level.
+        - Logs standard error (`stderr`) in real-time at the ERROR log level.
+
+    Raises:
+        subprocess.TimeoutExpired: If the command execution times out (see `timeout` parameter).
+        Exception: If other exceptions occur during command execution.
+        FileNotFoundError: If the command is not found.
+
+    Notes:
+        - While the command runs, its `stdout` and `stderr` streams are logged in real-time.
+        - The returned `stdout` and `stderr` capture the complete output, even though the output is also logged.
+        - The command is forcibly terminated if it exceeds the specified timeout.
+    """
+    stdout_lines = []  # To accumulate stdout
+    stderr_lines = []  # To accumulate stderr
+    process = None  # Track the process for cleanup
+    stdout_thread = None
+    stderr_thread = None
+
+    try:
+        # Check if the command exists before executing
+        if not shutil.which(command[0]):
+            raise FileNotFoundError(f"Command not found: {command[0]}")
+
+        logger.debug(f"Running command: {command}")
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+        # Start threads to read and log stdout and stderr
+        stdout_thread = threading.Thread(target=_stream_reader, args=(process.stdout, logger.info, stdout_lines))
+        stderr_thread = threading.Thread(target=_stream_reader, args=(process.stderr, logger.error, stderr_lines))
+        
+        stdout_thread.start()
+        stderr_thread.start()
+
+        # Wait for process to complete or timeout
+        process.wait(timeout=timeout)
+
+    except FileNotFoundError as e:
+        logger.error(f"Command not found: {command[0]}")
+        return CommandResult(
+            process=None,
+            stdout="",
+            stderr=str(e),
+            returncode=127,
+            timeout=timeout,
+            command=command
+        )
+    except subprocess.TimeoutExpired:
+        if process:
+            process.terminate()
+        logger.error(f"Command: '{command}' timed out and was terminated.")
+        raise
+    except Exception as e:
+        logger.error(f"Error running command: {command}", exc_info=True)
+        raise
+    finally:
+        # Ensure threads are joined to clean up (only if they were started)
+        if stdout_thread and stdout_thread.is_alive():
+            stdout_thread.join()
+        if stderr_thread and stderr_thread.is_alive():
+            stderr_thread.join()
+
+        # Ensure process streams are closed
+        if process and process.stdout:
+            process.stdout.close()
+        if process and process.stderr:
+            process.stderr.close()
+
+    # Combine captured stdout and stderr lines into single strings
+    stdout = "\n".join(stdout_lines)
+    stderr = "\n".join(stderr_lines)
+
+    # Build the result object
+    result = CommandResult(
+        process=process,
+        stdout=stdout,
+        stderr=stderr,
+        returncode=process.returncode,
+        timeout=timeout,
+        command=command
+    )
+    logger.debug(f"Command result: {result}")
+    return result
+
+def run_command2(command: List[str], timeout: int = 30) -> CommandResult:
+    """
+    Executes a given command via subprocess, logs its output in real time, and returns the result.
+
     Args:
         command (list): The command to be executed, represented as a list of strings.
         timeout (int): The maximum time in seconds to wait for the command to complete. Defaults to 30 seconds.
@@ -158,6 +262,10 @@ def run_command(command: List[str], timeout: int = 30) -> CommandResult:
     process = None  # Track the process for cleanup
 
     try:
+        # Check if the command exists before executing
+        if not shutil.which(command[0]):
+            raise FileNotFoundError(f"Command not found: {command[0]}")
+
         logger.debug(f"Running command: {command}")
         process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 
@@ -171,6 +279,16 @@ def run_command(command: List[str], timeout: int = 30) -> CommandResult:
         # Wait for process to complete or timeout
         process.wait(timeout=timeout)
 
+    except FileNotFoundError as e:
+        logger.error(f"Command not found: {command[0]}")
+        return CommandResult(
+            process=None,
+            stdout="",
+            stderr=str(e),
+            returncode=127,
+            timeout=timeout,
+            command=command
+        )
     except subprocess.TimeoutExpired:
         if process:
             process.terminate()

{dar_backup-0.6.9.dist-info → dar_backup-0.6.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dar-backup
-Version: 0.6.9
+Version: 0.6.11
 Summary: A script to do full, differential and incremental backups using dar.     Some files are restored from the backups during verification, after which par2 redundancy files are created.     The script also has a cleanup feature to remove old backups and par2 files.
 Project-URL: Homepage, https://github.com/per2jensen/dar-backup/tree/main/v2
 Project-URL: Changelog, https://github.com/per2jensen/dar-backup/blob/main/v2/Changelog.md
@@ -690,71 +690,123 @@ Classifier: Topic :: System :: Archiving :: Backup
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 
-# Full, differential or incremental backups using 'dar' 
+# Full, differential or incremental backups using 'dar'
 
-  The wonderful 'dar' [Disk Archiver] (https://github.com/Edrusb/DAR) is used for 
+  The wonderful 'dar' [Disk Archiver](https://github.com/Edrusb/DAR) is used for
   the heavy lifting, together with the par2 suite in these scripts.
 
-
-#  My use case
-
- I have cloud storage mounted on a directory within my home dir. The filesystem is [FUSE based](https://www.kernel.org/doc/html/latest/filesystems/fuse.html), which gives it a few special features
- - a non-privileged user (me :-)) can perform a mount
- - a privileged user cannot look into the filesystem --> a backup script running as root is not suitable
+## Table of Contents
+
+- [Full, differential or incremental backups using 'dar'](#full-differential-or-incremental-backups-using-dar)
+- [My use case](#my-use-case)
+- [License](#license)
+- [Status](#status)
+  - [Breaking change in version 0.6.0](#breaking-change-in-version-060)
+- [Homepage - Github](#homepage---github)
+- [Requirements](#requirements)
+- [Config file](#config-file)
+- [How to install & run](#how-to-run)
+  - [1 - config file](#1---config-file)
+  - [2 - backup definitions](#2---backup-definitions)
+  - [3 - installation](#3---installation)
+  - [4 - generate catalog databases](#4---generate-catalog-databases)
+  - [5 - do FULL backups](#5---do-full-backups)
+  - [6 - deactivate venv](#6---deactivate-venv)
+- [.darrc](#darrc)
+- [Systemctl examples](#systemctl-examples)
+  - [Service: dar-back --incremental-backup](#service-dar-back---incremental-backup)
+  - [Timer: dar-back --incremental-backup](#timer-dar-back---incremental-backup)
+- [List contents of an archive](#list-contents-of-an-archive)
+- [dar file selection examples](#dar-file-selection-exmaples)
+  - [Select a directory](#select-a-directory)
+  - [Select file dates in the directory](#select-file-dates-in-the-directory)
+  - [Exclude .xmp files from that date](#exclude-xmp-files-from-that-date)
+- [Restoring](#restoring)
+  - [Default location for restores](#default-location-for-restores)
+  - [--restore-dir option](#--restore-dir-option)
+  - [A single file](#a-single-file)
+  - [A directory](#a-directory)
+  - [.NEF from a specific date](#nef-from-a-specific-date)
+  - [Restore test fails with exit code 4](#restore-test-fails-with-exit-code-4)
+  - [Restore test fails with exit code 5](#restore-test-fails-with-exit-code-5)
+- [Par2](#par2)
+  - [Par2 to verify/repair](#par2-to-verifyrepair)
+  - [Par2 create redundancy files](#par2-create-redundancy-files)
+- [Points of interest](#points-of-interest)
+  - [Merge FULL with DIFF, creating new FULL](#merge-full-with-diff-creating-new-full)
+  - [dar manager databases](#dar-manager-databases)
+  - [Performance tip due to par2](#performance-tip-due-to-par2)
+  - [.darrc sets -vd -vf (since v0.6.4)](#darrc-sets--vd--vf-since-v064)
+- [Reference](#reference)
+  - [dar-backup.py](#dar-backuppy)
+  - [manager.py](#managerpy)
+  - [cleanup.py](#cleanuppy)
+  - [clean-log.py](#clean-logpy)
+
+## My use case
+
+I have cloud storage mounted on a directory within my home dir. The filesystem is [FUSE based](https://www.kernel.org/doc/html/latest/filesystems/fuse.html), which gives it a few special features
+
+- a non-privileged user (me :-)) can perform a mount
+- a privileged user cannot look into the filesystem --> a backup script running as root is not suitable
 
  I needed the following:
- - Backup my cloud storage to something local (cloud is convenient, but I want control over my backups)
- - Backup primarily photos, video and different types of documents
- - Have a simple non-complicated way of restoring, possibly years into the future. 'dar' fits that scenario with a single statically linked binary (kept with the archives). There is no need install/configure anything - restoring is simple and works well.
- - During backup archives must be tested and a restore test (however small) performed
- - Archives stored on a server with a reliable file system (easy to mount a directory over sshfs)
- - Easy to verify archive's integrity, after being moved around.
+
+- Backup my cloud storage to something local (cloud is convenient, but I want control over my backups)
+- Backup primarily photos, video and different types of documents
+- Have a simple non-complicated way of restoring, possibly years into the future. 'dar' fits that scenario with a single statically linked binary (kept with the archives). There is no need install/configure anything - restoring is simple and works well.
+- During backup archives must be tested and a restore test (however small) performed
+- Archives stored on a server with a reliable file system (easy to mount a directory over sshfs)
+- Easy to verify archive's integrity, after being moved around.
 
  I do not need the encryption features of dar, as all storage is already encrypted.
- 
 
-# License
+## License
 
   These scripts are licensed under the GPLv3 license.
   Read more here: https://www.gnu.org/licenses/gpl-3.0.en.html, or have a look at the ["LICENSE"](https://github.com/per2jensen/dar-backup/blob/main/LICENSE) file in this repository.
 
+## Status
 
-# Status
 As of August 8, 2024 I am using the alpha versions of `dar-backup` (alpha-0.5.9 onwards) in my automated backup routine.
 
 As of February 13, 2025, I have changed the status from alpha --> beta, as the featureset is in place and the alphas have worked well for a very long time.
 
-
-**Breaking change in version 0.6.0**
+### Breaking change in version 0.6.0
 
 Version 0.6.0 and forwards requires the config variable *COMMAND_TIMEOUT_SECS* in the config file.
 
-# Homepage - Github
-This 'dar-backup' package lives at: https://github.com/per2jensen/dar-backup
+## Homepage - Github
+
+This 'dar-backup' package lives at: https://github.com/per2jensen/dar-backup/tree/main/v2
 
 This python version is v2 of dar-backup, the first is made in bash.
 
-# Requirements
-  - dar
-  - par2
-  - python3 :-)
+## Requirements
+
+- dar
+- par2
+- python3 :-)
 
 On Ubuntu, install the requirements this way:
-  ````
+
+```` bash
     sudo apt install dar par2 python3
-  ````
+````
 
-# Config file
+## Config file
 
 The default configuration is expected here: ~/.config/dar-backup/dar-backup.conf
 
-# How to run 
+## How to run 
+
+### 1 - config file
 
-## 1
 Config file default location is $HOME/.config/dar-backup/dar-backup.conf
 
 Example:
-````
+
+```` code
 [MISC]
 LOGFILE_LOCATION=/home/user/dar-backup.log
 MAX_SIZE_VERIFICATION_MB = 20
@@ -791,9 +843,10 @@ ENABLED = True
 # SCRIPT_2 = <something>
 # ...
 
-````    
+````
+
+### 2 - backup definitions
 
-## 2 
 Put your backup definitions in the directory $BACKUP.D_DIR (defined in the config file)
 
 The name of the file is the `backup definition` name.
@@ -803,7 +856,8 @@ Make as many backup definitions as you need. Run them all in one go, or run one
 The `dar` [documentation](http://dar.linux.free.fr/doc/man/dar.html#COMMANDS%20AND%20OPTIONS) has good information on file selection.
 
 Example of backup definition for a home directory
-````    
+
+```` code
 
 # Switch to ordered selection mode, which means that the following
 # options will be considered top to bottom
@@ -817,7 +871,7 @@ Example of backup definition for a home directory
 # if you want to take a backup of /home/user/Documents only, uncomment next line
 #  -g Documents 
 
-# Some directories to exclude below the Root dir
+# Some directories to exclude below the Root dir (here Root directory is `/home/user` as set in the -R option)
  -P mnt
  -P tmp
  -P .cache
@@ -836,131 +890,93 @@ Example of backup definition for a home directory
  # size of each slice in the archive
  --slice 10G
 
-# see https://github.com/per2jensen/dar-backup?tab=readme-ov-file#restore-test-exit-code-4
---comparison-field=ignore-owner
 
 # bypass directores marked as cache directories
 # http://dar.linux.free.fr/doc/Features.html
 --cache-directory-tagging
-````    
+````
 
+### 3 - installation
 
-## 3
 Installation is currently in a venv. These commands are installed in the venv:
+
 - dar-back
 - cleanup
 - manager
+- clean-log
 
 To install, create a venv and run pip:
-````    
+
+```` bash
 mkdir $HOME/tmp
 cd $HOME/tmp
 python3 -m venv venv    # create the virtual environment 
 . venv/bin/activate     # activate the virtual env
 pip install dar-backup  # run pip to install `dar-backup`
-````    
-
+````
 
 I have an alias in ~/.bashrc pointing to my venv:
-````    
+
+```` bash
 alias db=". ~/tmp/venv/bin/activate; dar-backup -v"
-````    
+````
 
 Typing `db` at the command line gives this
-````    
+
+```` bash
 (venv) user@machine:~$ db
-dar-backup 0.5.17
+dar-backup 0.6.9
 dar-backup.py source code is here: https://github.com/per2jensen/dar-backup
 Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
 THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 See section 15 and section 16 in the supplied "LICENSE" file.
 ````
 
-`dar-backup -h` gives the usage output:
-````
-usage: dar-backup [-h] [-F] [-D] [-I] [-d BACKUP_DEFINITION]
-                  [--alternate-reference-archive ALTERNATE_REFERENCE_ARCHIVE] [-c CONFIG_FILE] [--darrc DARRC]
-                  [--examples] [-l] [--list-contents LIST_CONTENTS] [--selection SELECTION] [-r RESTORE]
-                  [--restore-dir RESTORE_DIR] [--verbose] [--log-level LOG_LEVEL] [--log-stdout]
-                  [--do-not-compare] [-v]
-
-Backup and verify using dar backup definitions.
-
-options:
-  -h, --help            show this help message and exit
-  -F, --full-backup     Perform a full backup.
-  -D, --differential-backup
-                        Perform differential backup.
-  -I, --incremental-backup
-                        Perform incremental backup.
-  -d BACKUP_DEFINITION, --backup-definition BACKUP_DEFINITION
-                        Specific 'recipe' to select directories and files.
-  --alternate-reference-archive ALTERNATE_REFERENCE_ARCHIVE
-                        DIFF or INCR compared to specified archive.
-  -c CONFIG_FILE, --config-file CONFIG_FILE
-                        Path to 'dar-backup.conf'
-  --darrc DARRC         Optional path to .darrc
-  --examples            Examples of using dar-backup.py.
-  -l, --list            List available archives.
-  --list-contents LIST_CONTENTS
-                        List the contents of the specified archive.
-  --selection SELECTION
-                        dar file selection for listing/restoring specific files/directories.
-  -r RESTORE, --restore RESTORE
-                        Restore specified archive.
-  --restore-dir RESTORE_DIR
-                        Directory to restore files to.
-  --verbose             Print various status messages to screen
-  --log-level LOG_LEVEL
-                        `debug` or `trace`
-  --log-stdout          also print log messages to stdout
-  --do-not-compare      do not compare restores to file system
-  -v, --version         Show version and license information.
-````    
-
-## 4
+### 4 - generate catalog databases
+
 Generate the archive catalog database(s). 
 `dar-backup` expects the catalog databases to be in place, it does not automatically create them (by design)
 
-````    
-manager --create-db --config-file <path to config file> --log-level debug --log-stdout
-````    
-
-
+```` bash
+manager --create-db
+````
 
+### 5 - do FULL backups
 
-## 5
 You are ready to do backups of all your backup definitions, if your backup definitions are 
 in place in BACKUP.D_DIR (see config file)
-````    
+
+```` bash
 dar-backup --full-backup 
-````    
+````
+
 If you want to see dar-backup's log entries in the terminal, use the `--log-stdout` option. This is also useful if dar-backup is started by systemd.
 
 If you want more log messages, use the `--log-level debug` option.
 
-
 If you want a backup of a single definition, use the `-d <backup definition>` option. The definition's name is the filename of the definition in the `backup.d` config directory.
-````    
+
+```` bash
 dar-backup --full-backup -d <your backup definition>
 ````
 
-## 6 
+### 6 - deactivate venv
 
-Deactivate the virtual environment
-````
+Deactivate the virtual environment (venv)
+
+```` bash
 deactivate
 ````
 
+## .darrc
 
-# .darrc
 The package includes a default `.darrc` file which configures `dar`.
 
 You can override the default `.darrc` using the `--darrc` option.
 
 The default `.darrc` contents are as follows:
 
-````
+```` code
 #  .darrc configuration file for `dar` as used by the `dar-backup` script.
 #  `dar-backup` lives here: https://github.com/per2jensen/dar-backup
 
@@ -998,19 +1014,21 @@ verbose:
 # -va
 
 
-extract:
+restore-options:
 # don't restore File Specific Attributes
 #--fsa-scope none
 
 # ignore owner, useful when used by a non-privileged user
 --comparison-field=ignore-owner
 
+
+# Exclude specific file types from compression
+compress-exclusion:
+
 # First setting case insensitive mode on:
 -an
 -ag
 
-# Exclude specific file types from compression
-compress-exclusion:
 -Z    "*.gz"
 -Z    "*.bz2"
 -Z    "*.xz"
@@ -1081,14 +1099,15 @@ compress-exclusion:
 -acase
 ````
 
-# Systemctl examples
+## Systemctl examples
+
 I have dar-backup scheduled to run via systemd --user settings.
 
 The files are located in: ~/.config/systemd/user
 
 Once the .service and .timer files are in place, timers must be enabled and started.
 
-````
+```` bash
 systemctl --user enable dar-inc-backup.timer
 systemctl --user start  dar-inc-backup.timer
 systemctl --user daemon-reload
@@ -1096,15 +1115,15 @@ systemctl --user daemon-reload
 
 Verify your timers are set up as you want:
 
-````
+```` bash
 systemctl --user list-timers
 ````
 
-
 ## Service: dar-back --incremental-backup
 
 File:  dar-inc-backup.service
-````
+
+```` code
 [Unit]
 Description=dar-backup INC
 StartLimitIntervalSec=120
@@ -1119,7 +1138,8 @@ ExecStart=/bin/bash -c '. /home/user/programmer/dar-backup.py/venv/bin/activate
 ## Timer: dar-back --incremental-backup
 
 File:  dar-inc-backup.timer
-````
+
+```` code
 [Unit]
 Description=dar-backup INC timer
 
@@ -1131,15 +1151,17 @@ Persistent=true
 WantedBy=timers.target
 ````
 
+## list contents of an archive
 
-# list contents of an archive
-```
+```` bash
 . <the virtual evn>/bin/activate
 dar-backup --list-contents example_FULL_2024-06-23 --selection "-X '*.xmp' -I '*2024-06-16*' -g home/pj/tmp/LUT-play"
 deactivate
-```
+````
+
 gives
-```
+
+``` code
 [Data ][D][ EA  ][FSA][Compr][S]| Permission | User  | Group | Size    |          Date                 |    filename
 --------------------------------+------------+-------+-------+---------+-------------------------------+------------
 [Saved][-]       [-L-][   0%][ ]  drwxr-xr-x   root	root	113 Mio	Sat May 11 16:16:48 2024	home
@@ -1149,17 +1171,17 @@ gives
 [Saved][ ]       [-L-][   0%][X]  -rw-rw-r--   pj	pj	49 Mio	Sun Jun 16 12:52:22 2024	home/pj/tmp/LUT-play/2024-06-16_12:52:22,15.NEF
 ```
 
+## dar file selection exmaples
 
+### select a directory
 
-
-# dar file selection exmaples
-
-## select a directory
-```
+``` bash
 dar -l /tmp/example_FULL_2024-06-23  -g home/pj/tmp/LUT-play
 ```
+
 gives
-```
+
+```` code
 [Data ][D][ EA  ][FSA][Compr][S]| Permission | User  | Group | Size    |          Date                 |    filename
 --------------------------------+------------+-------+-------+---------+-------------------------------+------------
 [Saved][-]       [-L-][   0%][ ]  drwxr-xr-x   root	root	113 Mio	Sat May 11 16:16:48 2024	home
@@ -1186,15 +1208,17 @@ gives
 [Saved][ ]       [-L-][  92%][ ]  -rw-rw-r--   pj	pj	24 kio	Sat Jun 22 21:51:15 2024	home/pj/tmp/LUT-play/2024-06-16_12:52:22,15_16.NEF.xmp
 [Saved][ ]       [-L-][  92%][ ]  -rw-rw-r--   pj	pj	24 kio	Sat Jun 22 21:50:48 2024	home/pj/tmp/LUT-play/2024-06-16_12:52:22,15_17.NEF.xmp
 [Saved][ ]       [-L-][  92%][ ]  -rw-rw-r--   pj	pj	24 kio	Sat Jun 22 21:50:19 2024	home/pj/tmp/LUT-play/2024-06-16_12:52:22,15_18.NEF.xmp
-```
+````
 
+### select file dates in the directory:
 
-## select file dates in the directory:
-```
+``` bash
 dar -l /tmp/example_FULL_2024-06-23  -I '*2024-06-16*' -g home/pj/tmp/LUT-play
 ```
+
 gives
-```
+
+``` code
 [Data ][D][ EA  ][FSA][Compr][S]| Permission | User  | Group | Size    |          Date                 |    filename
 --------------------------------+------------+-------+-------+---------+-------------------------------+------------
 [Saved][-]       [-L-][   0%][ ]  drwxr-xr-x   root	root	113 Mio	Sat May 11 16:16:48 2024	home
@@ -1223,12 +1247,16 @@ gives
 [Saved][ ]       [-L-][  92%][ ]  -rw-rw-r--   pj	pj	24 kio	Sat Jun 22 21:50:19 2024	home/pj/tmp/LUT-play/2024-06-16_12:52:22,15_18.NEF.xmp
 ```
 
-## exclude .xmp files from that date
-```
+### exclude .xmp files from that date
+
+``` bash
 dar -l /tmp/example_FULL_2024-06-23 -X '*.xmp' -I '*2024-06-16*' -g home/pj/tmp/LUT-play
+
 ```
+
 gives
-```
+
+``` code
 [Data ][D][ EA  ][FSA][Compr][S]| Permission | User  | Group | Size    |          Date                 |    filename
 --------------------------------+------------+-------+-------+---------+-------------------------------+------------
 [Saved][-]       [-L-][   0%][ ]  drwxr-xr-x   root	root	113 Mio	Sat May 11 16:16:48 2024	home
@@ -1240,16 +1268,16 @@ gives
 
 Nice :-)
 
+## Restoring
 
+### default location for restores
 
-# Restoring
-
-## default location for restores
 dar-backup will use the TEST_RESTORE_DIR location as the Root for restores, if the --restore-dir option has not been supplied.
 
 See example below to see where files are restored to.
 
-## --restore-dir option
+### --restore-dir option
+
 When the --restore-dir option is used for restoring, a directory must be supplied.
 
 The directory supplied functions as the Root of the restore operation.
@@ -1257,41 +1285,200 @@ The directory supplied functions as the Root of the restore operation.
 **Example**:
 
 A backup has been taken using this backup definition:
-```
+
+``` code
 -R /
 -g home/user/Documents
 ```
 
 When restoring and using `/tmp` for --restore-dir, the restored files can be found in `/tmp/home/user/Documents`
 
-## a single file
-```
+### a single file
+
+``` code
 . <the virtual env>/bin/activate
 dar-backup --restore <archive_name> --selection "-g path/to/file"
 deactivate
+
 ```
-## a directory
-```
+
+### a directory
+
+``` bash
 . <the virtual env>/bin/activate
 dar-backup --restore <archive_name> --selection "-g path/to/directory"
 deactivate
 ```
 
+### .NEF from a specific date
 
-## .NEF from a specific date
-```
+``` bash
 . <the virtual env>/bin/activate
 dar-backup --restore <archive_name>  --selection "-X '*.xmp' -I '*2024-06-16*' -g home/pj/tmp/LUT-play"
 deactivate
 ```
 
+### restore test fails with exit code 4
 
-# Points of interest
- 
-## .darrc sets -vd -vf (since v0.6.4)
+"dar" in newer versions emits a question about file ownership, which is "answered" with a "no" via the "-Q" option. That in turn leads to an error code 4.
+
+Thus the dar option "--comparison-field=ignore-owner" has been placed in the supplied .darrc file (located in the virtual environment where dar-backup is installed). 
+
+This causes dar to restore without an error.
+
+It is a good option when using dar as a non-privileged user.
+
+### restore test fails with exit code 5
+
+If exit code 5 is emitted on the restore test, FSA (File System specific Attributes) could be the cause.
+
+That (might) occur if you backup a file stored on one type of filesystem, and restore it on another type.
+My home directory is on a btrfs filesystem, while /tmp (for the restore test) is on zfs.
+
+The restore test can result in an exit code 5, due to the different filesystems used. In order to avoid the errors, the "option "--fsa-scope none" can be used. That will restult in FSA's not being restored.
+
+If you need to use this option, un-comment it in the .darrc file (located in the virtual environment where dar-backup is installed)
+
+## Par2
+
+### Par2 to verify/repair
+
+You can run a par2 verification on an archive like this:
+
+```` bash
+for file in <archive>*.dar.par2; do
+  par2 verify "$file"
+done
+````
+
+if there are problems with a slice, try to repair it like this:
+
+```` bash
+  par2 repair <archive>.<slice number>.dar.par2
+````
+
+### Par2 create redundancy files
+
+If you have merged archives, you will need to create the .par2 redundency files manually.
+Here is an example
+
+```` bash
+for file in <some-archive>_FULL_yyyy-mm-dd.*; do
+  par2 c -r5 -n1 "$file"
+done
+````
+
+where "c" is create, -r5 is 5% redundency and -n1 is 1 redundency file
+
+## Points of interest
+
+### Merge FULL with DIFF, creating new FULL
+
+Over time, the DIFF archives become larger and larger. At some point one wishes to create a new FULL archive to do DIFF's on.
+One way to do that, is to let dar create a FULL archive from scratch, another is to merge a FULL archive with a DIFF, and from there do DIFF's until they once again gets too large for your taste.
+
+I do backups of my homedir. Here it is shown how a FULL archive is merged with a DIFF, creating a new FULL archive.
+
+```` bash
+dar --merge pj-homedir_FULL_2021-09-12  -A pj-homedir_FULL_2021-06-06  -@pj-homedir_DIFF_2021-08-29 -s 12G
+
+# test the new FULL archive
+dar -t pj-homedir_FULL_2021-09-12
+
+# create Par2 redundancy files
+for file in pj-homedir_FULL_yyyy-mm-dd.*.dar; do
+  par2 c -r5 -n1 "$file"
+done
+
+````
+
+### dar manager databases
+
+`dar-backup` now saves archive catalogs in dar catalog databases.
+
+This makes it easier to restore to a given date when having many FULL, DIFF and INCR archives.
+
+### Performance tip due to par2
+
+This [dar benchmark page](https://dar.sourceforge.io/doc/benchmark.html) has an interesting note on the slice size.
+
+Slice size should be smaller than available RAM, apparently a large performance hit can be avoided keeping the the par2 data in memory.
+
+### .darrc sets -vd -vf (since v0.6.4)
 
 These .darrc settings make `dar` print the current directory being processed (-vd) and some stats after (-vf)
 This is very useful in very long running jobs to get an indication that the backup is proceeding normally.
 
-if --log-stdout is used the information would be picked up by systemd and logged by journald
+if --log-stdout is used the information would be picked up by systemd and logged by journald.½
+
+The log file can get quite cluttered, if you want the clutter to be removed, run the `clean-log`script.
+
+## Reference
+
+### dar-backup.py
+
+This script is responsible for managing the backup creation and validation process. It supports the following options:
+
+``` code
+--full-backup                     Perform a full backup.
+--differential-backup             Perform a differential backup.
+--incremental-backup              Perform an incremental backup.
+--backup-definition <name>        Specify the backup definition file.
+--alternate-reference-archive <file>  Use a different archive for DIFF/INCR backups.
+--config-file <path>              Specify the path to the configuration file.
+--darrc <path>                    Specify an optional path to .darrc.
+--examples                        Show examples of using dar-backup.py.
+--list                            List available backups.
+--list-contents <archive>         List the contents of a specified archive.
+--selection <params>              Define file selection for listing/restoring.
+--restore <archive>               Restore a specified archive.
+--restore-dir <path>              Directory to restore files to.
+--verbose                         Enable verbose output.
+--log-level <level>               Set log level (debug, trace, etc.).
+--log-stdout                      Also print log messages to stdout.
+--do-not-compare                  Do not compare restores to file system.
+--version                         Show version and license information.
+```
+
+### manager.py
+
+This script manages `dar` databases and catalogs. Available options include:
+
+``` code
+--create-db                     Create missing databases for all backup definitions.
+--alternate-archive-dir <path>   Use this directory instead of BACKUP_DIR in the config file.
+--add-dir <path>                 Add all archive catalogs in this directory to databases.
+-d, --backup-def <name>          Restrict to work only on this backup definition.
+--add-specific-archive <file>    Add this archive to the catalog database.
+--remove-specific-archive <file> Remove this archive from the catalog database.
+-l, --list-catalogs              List catalogs in databases for all backup definitions.
+--list-catalog-contents <num>    List contents of a catalog by archive number.
+--list-archive-contents <file>   List contents of an archive’s catalog.
+--find-file <file>               Search catalogs for a specific file.
+--verbose                        Enable verbose output.
+--log-level <level>              Set log level.
+```
+
+### cleanup.py
+
+This script cleans up old backups and manages storage. Supported options:
+
+``` code
+--prune-older-than <days>   Remove backups older than the specified number of days.
+--keep-last <num>           Retain only the last <num> backups.
+--dry-run                   Show what would be deleted without actually deleting.
+--backup-dir <path>         Specify the backup directory.
+--verbose                   Enable verbose output.
+--help                      Show this help message and exit.
+```
+
+### clean-log.py
 
+This script removes excessive logging output from `dar` logs, improving readability and efficiency. Available options:
+
+``` code
+-f, --file <path>          Specify the log file(s) to be cleaned.
+-c, --config-file <path>   Specify the configuration file (default: ~/.config/dar-backup/dar-backup.conf).
+--dry-run                  Show which lines would be removed without modifying the file.
+-v, --version              Display version and licensing information.
+```

dar_backup-0.6.11.dist-info/RECORD

@@ -0,0 +1,14 @@
+dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
+dar_backup/__about__.py,sha256=WAW-bfIrNx18qYeBN4lihEQibyosiKElDt_p7DfZELo,22
+dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dar_backup/clean_log.py,sha256=VXKA2BMyQmaC6R08Bq9a3wP3mczdFb_moy6HkL-mnF8,5176
+dar_backup/cleanup.py,sha256=9yEdRR84XPtEvBGc2QfwGBQl2tdTPttjetHeiSc_TsM,11419
+dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
+dar_backup/dar_backup.py,sha256=o4DvaVsqWHb1PxHIXfdr2qyqW6KRRGNcrR2_CRbPy1I,32942
+dar_backup/manager.py,sha256=HDa8eYF89QFhlBRR4EWRzzmswOW00S_w8ToZ5SARO_o,21359
+dar_backup/util.py,sha256=E-sEBQZY1hmdeVx5xNE22zKQ0BXDee1eI9F1-w7Fq1Q,15756
+dar_backup-0.6.11.dist-info/METADATA,sha256=N7ICBJQEVDC01tza9lRAkmOXCIAILoDrg-mrHG926f0,71626
+dar_backup-0.6.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dar_backup-0.6.11.dist-info/entry_points.txt,sha256=p6c4uQLjlTIVP1Od2iorGefrVUH0IWZdFRMl63mNaRg,164
+dar_backup-0.6.11.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dar_backup-0.6.11.dist-info/RECORD,,

dar_backup-0.6.9.dist-info/RECORD

@@ -1,14 +0,0 @@
-dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
-dar_backup/__about__.py,sha256=42fbYUSZonpHevrA9hTI2EzfgLNO9LBK95poG3HeDEQ,21
-dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dar_backup/clean_log.py,sha256=VXKA2BMyQmaC6R08Bq9a3wP3mczdFb_moy6HkL-mnF8,5176
-dar_backup/cleanup.py,sha256=9yEdRR84XPtEvBGc2QfwGBQl2tdTPttjetHeiSc_TsM,11419
-dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
-dar_backup/dar_backup.py,sha256=hDy7aXU-XiWOtW40Pxql441liNkSYKGU76eOwy8m7fU,32714
-dar_backup/manager.py,sha256=HDa8eYF89QFhlBRR4EWRzzmswOW00S_w8ToZ5SARO_o,21359
-dar_backup/util.py,sha256=SSSJYM9lQZfubhTUBlX1xDGWmCpYEF3ePARmlY544xM,11283
-dar_backup-0.6.9.dist-info/METADATA,sha256=mqAzHZp_-X9pw8w77c4FA7QwXnKikkSodJ-oJnaFT3M,64610
-dar_backup-0.6.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dar_backup-0.6.9.dist-info/entry_points.txt,sha256=p6c4uQLjlTIVP1Od2iorGefrVUH0IWZdFRMl63mNaRg,164
-dar_backup-0.6.9.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dar_backup-0.6.9.dist-info/RECORD,,