dar-backup 0.6.10__py3-none-any.whl → 0.6.11__py3-none-any.whl

dar_backup/__about__.py

@@ -1 +1 @@
-__version__ = "0.6.10"
+__version__ = "0.6.11"

dar_backup/dar_backup.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import datetime
 import filecmp
 
 import os
@@ -81,30 +80,31 @@ def generic_backup(type: str, command: List[str], backup_file: str, backup_defin
         raise BackupError(f"Unexpected error during backup: {e}") from e
  
 
-# Function to recursively find <File> tags and build their full paths
-def find_files_with_paths(element: ET, current_path=""):
+
+def find_files_with_paths(xml_root: ET.Element):
     """
-    Recursively finds files within a directory element and returns a list of file paths with their sizes.
+    Finds files within an XML element and returns a list of file paths with their sizes.
 
     Args:
-        element (Element): The directory element to search within.
-        current_path (str, optional): The current path of the directory element. Defaults to "".
+        xml_root (Element): The root element of the XML.
 
     Returns:
         list: A list of tuples containing file paths and their sizes.
     """
-    logger.debug(f"Recursively generate list of tuples with file paths and sizes for File elements in dar xml output")
+    logger.debug("Generating list of tuples with file paths and sizes for File elements in dar xml output")
     files = []
-    if element.tag == "Directory":
-        current_path = f"{current_path}/{element.get('name')}"
-    for child in element:
-        if child.tag == "File":
-            file_path = (f"{current_path}/{child.get('name')}", child.get('size'))  # tuple (filepath, size)
+    current_path = []
+
+    for elem in xml_root.iter():
+        if elem.tag == "directory":
+            current_path.append(elem.get('name'))
+        elif elem.tag == "file":
+            file_path = ("/".join(current_path + [elem.get('name')]), elem.get('size'))
             files.append(file_path)
-        elif child.tag == "Directory":
-            files.extend(find_files_with_paths(child, current_path))
-    return files
+        elif elem.tag == "directory" and elem.get('name') in current_path:
+            current_path.pop()
 
+    return files
 
 
 def find_files_between_min_and_max_size(backed_up_files: list[(str, str)], config_settings: ConfigSettings):
@@ -238,20 +238,20 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
         restore_dir (str): The directory where the backup should be restored to.
         selection (str, optional): A selection criteria to restore specific files or directories. Defaults to None.
     """
-    backup_file = os.path.join(config_settings.backup_dir, backup_name)
-    command = ['dar', '-x', backup_file, '-Q', '-D']
-    if restore_dir:
-        if not os.path.exists(restore_dir):
-            os.makedirs(restore_dir)
-        command.extend(['-R', restore_dir])
-    else:
-        raise RestoreError("Restore directory ('-R <dir>') not specified")
-    if selection:
-        selection_criteria = shlex.split(selection)
-        command.extend(selection_criteria)
-    command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
-    logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
     try:
+        backup_file = os.path.join(config_settings.backup_dir, backup_name)
+        command = ['dar', '-x', backup_file, '-Q', '-D']
+        if restore_dir:
+            if not os.path.exists(restore_dir):
+                os.makedirs(restore_dir)
+            command.extend(['-R', restore_dir])
+        else:
+            raise RestoreError("Restore directory ('-R <dir>') not specified")
+        if selection:
+            selection_criteria = shlex.split(selection)
+            command.extend(selection_criteria)
+        command.extend(['-B', darrc,  'restore-options'])  # the .darrc `restore-options` section
+        logger.info(f"Running restore command: {' '.join(map(shlex.quote, command))}")
         process = run_command(command, config_settings.command_timeout_secs)
         if process.returncode == 0:
             logger.info(f"Restore completed successfully to: '{restore_dir}'")
@@ -260,6 +260,9 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
             raise RestoreError(str(process))
     except subprocess.CalledProcessError as e:
         raise RestoreError(f"Restore command failed: {e}") from e
+    except OSError as e:
+        logger.error(f"Failed to create restore directory: {e}")
+        raise RestoreError("Could not create restore directory")
     except Exception as e:
         raise RestoreError(f"Unexpected error during restore: {e}") from e
 
@@ -285,7 +288,7 @@ def get_backed_up_files(backup_name: str, backup_dir: str):
         # Parse the XML data
         root = ET.fromstring(process.stdout)
         output = None  # help gc
-        # Extract full paths and file size for all <File> elements
+        # Extract full paths and file size for all <file> elements
         file_paths = find_files_with_paths(root)
         root = None # help gc
         logger.trace(str(process))
@@ -624,6 +627,14 @@ def main():
     else:
         logger.error(f"Supplied .darrc: '{args.darrc}' does not exist or is not a file")
 
+    if not args.config_file:
+        logger.error(f"Config file not specified, exiting")
+        sys.exit(1) 
+    
+    if not os.path.exists(args.config_file):
+        logger.error(f"Config file {args.config_file} does not exist.")
+        sys.exit(1)
+
 
     try:
         start_time=int(time())

dar_backup/util.py

@@ -14,6 +14,7 @@ import os
 import re
 import subprocess
 import shlex
+import shutil
 import sys
 import threading
 import traceback
@@ -123,10 +124,113 @@ def  _stream_reader(pipe, log_func, output_accumulator: List[str]):
             log_func(stripped_line)  # Log the output in real time
 
 
+
 def run_command(command: List[str], timeout: int = 30) -> CommandResult:
     """
     Executes a given command via subprocess, logs its output in real time, and returns the result.
 
+    Args:
+        command (list): The command to be executed, represented as a list of strings.
+        timeout (int): The maximum time in seconds to wait for the command to complete. Defaults to 30 seconds.
+
+    Returns:
+        A CommandResult NamedTuple with the following properties:
+        - process: subprocess.CompletedProcess
+        - stdout: str: The full standard output of the command.
+        - stderr: str: The full standard error of the command.
+        - returncode: int: The return code of the command.
+        - timeout: int: The timeout value in seconds used to run the command.
+        - command: list[str]: The command executed.
+
+    Logs:
+        - Logs standard output (`stdout`) in real-time at the INFO log level.
+        - Logs standard error (`stderr`) in real-time at the ERROR log level.
+
+    Raises:
+        subprocess.TimeoutExpired: If the command execution times out (see `timeout` parameter).
+        Exception: If other exceptions occur during command execution.
+        FileNotFoundError: If the command is not found.
+
+    Notes:
+        - While the command runs, its `stdout` and `stderr` streams are logged in real-time.
+        - The returned `stdout` and `stderr` capture the complete output, even though the output is also logged.
+        - The command is forcibly terminated if it exceeds the specified timeout.
+    """
+    stdout_lines = []  # To accumulate stdout
+    stderr_lines = []  # To accumulate stderr
+    process = None  # Track the process for cleanup
+    stdout_thread = None
+    stderr_thread = None
+
+    try:
+        # Check if the command exists before executing
+        if not shutil.which(command[0]):
+            raise FileNotFoundError(f"Command not found: {command[0]}")
+
+        logger.debug(f"Running command: {command}")
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+        # Start threads to read and log stdout and stderr
+        stdout_thread = threading.Thread(target=_stream_reader, args=(process.stdout, logger.info, stdout_lines))
+        stderr_thread = threading.Thread(target=_stream_reader, args=(process.stderr, logger.error, stderr_lines))
+        
+        stdout_thread.start()
+        stderr_thread.start()
+
+        # Wait for process to complete or timeout
+        process.wait(timeout=timeout)
+
+    except FileNotFoundError as e:
+        logger.error(f"Command not found: {command[0]}")
+        return CommandResult(
+            process=None,
+            stdout="",
+            stderr=str(e),
+            returncode=127,
+            timeout=timeout,
+            command=command
+        )
+    except subprocess.TimeoutExpired:
+        if process:
+            process.terminate()
+        logger.error(f"Command: '{command}' timed out and was terminated.")
+        raise
+    except Exception as e:
+        logger.error(f"Error running command: {command}", exc_info=True)
+        raise
+    finally:
+        # Ensure threads are joined to clean up (only if they were started)
+        if stdout_thread and stdout_thread.is_alive():
+            stdout_thread.join()
+        if stderr_thread and stderr_thread.is_alive():
+            stderr_thread.join()
+
+        # Ensure process streams are closed
+        if process and process.stdout:
+            process.stdout.close()
+        if process and process.stderr:
+            process.stderr.close()
+
+    # Combine captured stdout and stderr lines into single strings
+    stdout = "\n".join(stdout_lines)
+    stderr = "\n".join(stderr_lines)
+
+    # Build the result object
+    result = CommandResult(
+        process=process,
+        stdout=stdout,
+        stderr=stderr,
+        returncode=process.returncode,
+        timeout=timeout,
+        command=command
+    )
+    logger.debug(f"Command result: {result}")
+    return result
+
+def run_command2(command: List[str], timeout: int = 30) -> CommandResult:
+    """
+    Executes a given command via subprocess, logs its output in real time, and returns the result.
+
     Args:
         command (list): The command to be executed, represented as a list of strings.
         timeout (int): The maximum time in seconds to wait for the command to complete. Defaults to 30 seconds.
@@ -158,6 +262,10 @@ def run_command(command: List[str], timeout: int = 30) -> CommandResult:
     process = None  # Track the process for cleanup
 
     try:
+        # Check if the command exists before executing
+        if not shutil.which(command[0]):
+            raise FileNotFoundError(f"Command not found: {command[0]}")
+
         logger.debug(f"Running command: {command}")
         process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 
@@ -171,6 +279,16 @@ def run_command(command: List[str], timeout: int = 30) -> CommandResult:
         # Wait for process to complete or timeout
         process.wait(timeout=timeout)
 
+    except FileNotFoundError as e:
+        logger.error(f"Command not found: {command[0]}")
+        return CommandResult(
+            process=None,
+            stdout="",
+            stderr=str(e),
+            returncode=127,
+            timeout=timeout,
+            command=command
+        )
     except subprocess.TimeoutExpired:
         if process:
             process.terminate()

{dar_backup-0.6.10.dist-info → dar_backup-0.6.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dar-backup
-Version: 0.6.10
+Version: 0.6.11
 Summary: A script to do full, differential and incremental backups using dar.     Some files are restored from the backups during verification, after which par2 redundancy files are created.     The script also has a cleanup feature to remove old backups and par2 files.
 Project-URL: Homepage, https://github.com/per2jensen/dar-backup/tree/main/v2
 Project-URL: Changelog, https://github.com/per2jensen/dar-backup/blob/main/v2/Changelog.md
@@ -690,9 +690,9 @@ Classifier: Topic :: System :: Archiving :: Backup
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 
-# Full, differential or incremental backups using 'dar' 
+# Full, differential or incremental backups using 'dar'
 
-  The wonderful 'dar' [Disk Archiver] (https://github.com/Edrusb/DAR) is used for
+  The wonderful 'dar' [Disk Archiver](https://github.com/Edrusb/DAR) is used for
   the heavy lifting, together with the par2 suite in these scripts.
 
 ## Table of Contents
@@ -705,30 +705,37 @@ Description-Content-Type: text/markdown
 - [Homepage - Github](#homepage---github)
 - [Requirements](#requirements)
 - [Config file](#config-file)
-- [How to run](#how-to-run)
-  - [1](#1)
-  - [2](#2)
-  - [3](#3)
-  - [4](#4)
-  - [5](#5)
-  - [6](#6)
+- [How to install & run](#how-to-run)
+  - [1 - config file](#1---config-file)
+  - [2 - backup definitions](#2---backup-definitions)
+  - [3 - installation](#3---installation)
+  - [4 - generate catalog databases](#4---generate-catalog-databases)
+  - [5 - do FULL backups](#5---do-full-backups)
+  - [6 - deactivate venv](#6---deactivate-venv)
 - [.darrc](#darrc)
 - [Systemctl examples](#systemctl-examples)
   - [Service: dar-back --incremental-backup](#service-dar-back---incremental-backup)
   - [Timer: dar-back --incremental-backup](#timer-dar-back---incremental-backup)
 - [List contents of an archive](#list-contents-of-an-archive)
-- [dar file selection examples](#dar-file-selection-examples)
+- [dar file selection examples](#dar-file-selection-exmaples)
   - [Select a directory](#select-a-directory)
   - [Select file dates in the directory](#select-file-dates-in-the-directory)
   - [Exclude .xmp files from that date](#exclude-xmp-files-from-that-date)
 - [Restoring](#restoring)
   - [Default location for restores](#default-location-for-restores)
-  - [--restore-dir option](#restore-dir-option)
+  - [--restore-dir option](#--restore-dir-option)
   - [A single file](#a-single-file)
   - [A directory](#a-directory)
   - [.NEF from a specific date](#nef-from-a-specific-date)
+  - [Restore test fails with exit code 4](#restore-test-fails-with-exit-code-4)
+  - [Restore test fails with exit code 5](#restore-test-fails-with-exit-code-5)
+- [Par2](#par2)
+  - [Par2 to verify/repair](#par2-to-verifyrepair)
+  - [Par2 create redundancy files](#par2-create-redundancy-files)
 - [Points of interest](#points-of-interest)
+  - [Merge FULL with DIFF, creating new FULL](#merge-full-with-diff-creating-new-full)
   - [dar manager databases](#dar-manager-databases)
+  - [Performance tip due to par2](#performance-tip-due-to-par2)
   - [.darrc sets -vd -vf (since v0.6.4)](#darrc-sets--vd--vf-since-v064)
 - [Reference](#reference)
   - [dar-backup.py](#dar-backuppy)
@@ -793,7 +800,7 @@ The default configuration is expected here: ~/.config/dar-backup/dar-backup.conf
 
 ## How to run 
 
-### 1
+### 1 - config file
 
 Config file default location is $HOME/.config/dar-backup/dar-backup.conf
 
@@ -838,7 +845,7 @@ ENABLED = True
 
 ````
 
-### 2
+### 2 - backup definitions
 
 Put your backup definitions in the directory $BACKUP.D_DIR (defined in the config file)
 
@@ -889,7 +896,7 @@ Example of backup definition for a home directory
 --cache-directory-tagging
 ````
 
-### 3
+### 3 - installation
 
 Installation is currently in a venv. These commands are installed in the venv:
 
@@ -925,7 +932,7 @@ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW,
 See section 15 and section 16 in the supplied "LICENSE" file.
 ````
 
-### 4
+### 4 - generate catalog databases
 
 Generate the archive catalog database(s). 
 `dar-backup` expects the catalog databases to be in place, it does not automatically create them (by design)
@@ -934,7 +941,7 @@ Generate the archive catalog database(s).
 manager --create-db
 ````
 
-### 5
+### 5 - do FULL backups
 
 You are ready to do backups of all your backup definitions, if your backup definitions are 
 in place in BACKUP.D_DIR (see config file)
@@ -953,9 +960,9 @@ If you want a backup of a single definition, use the `-d <backup definition>` op
 dar-backup --full-backup -d <your backup definition>
 ````
 
-### 6
+### 6 - deactivate venv
 
-Deactivate the virtual environment
+Deactivate the virtual environment (venv)
 
 ```` bash
 deactivate
@@ -1311,14 +1318,92 @@ dar-backup --restore <archive_name>  --selection "-X '*.xmp' -I '*2024-06-16*' -
 deactivate
 ```
 
+### restore test fails with exit code 4
+
+"dar" in newer versions emits a question about file ownership, which is "answered" with a "no" via the "-Q" option. That in turn leads to an error code 4.
+
+Thus the dar option "--comparison-field=ignore-owner" has been placed in the supplied .darrc file (located in the virtual environment where dar-backup is installed). 
+
+This causes dar to restore without an error.
+
+It is a good option when using dar as a non-privileged user.
+
+### restore test fails with exit code 5
+
+If exit code 5 is emitted on the restore test, FSA (File System specific Attributes) could be the cause.
+
+That (might) occur if you backup a file stored on one type of filesystem, and restore it on another type.
+My home directory is on a btrfs filesystem, while /tmp (for the restore test) is on zfs.
+
+The restore test can result in an exit code 5, due to the different filesystems used. In order to avoid the errors, the "option "--fsa-scope none" can be used. That will restult in FSA's not being restored.
+
+If you need to use this option, un-comment it in the .darrc file (located in the virtual environment where dar-backup is installed)
+
+## Par2
+
+### Par2 to verify/repair
+
+You can run a par2 verification on an archive like this:
+
+```` bash
+for file in <archive>*.dar.par2; do
+  par2 verify "$file"
+done
+````
+
+if there are problems with a slice, try to repair it like this:
+
+```` bash
+  par2 repair <archive>.<slice number>.dar.par2
+````
+
+### Par2 create redundancy files
+
+If you have merged archives, you will need to create the .par2 redundency files manually.
+Here is an example
+
+```` bash
+for file in <some-archive>_FULL_yyyy-mm-dd.*; do
+  par2 c -r5 -n1 "$file"
+done
+````
+
+where "c" is create, -r5 is 5% redundency and -n1 is 1 redundency file
+
 ## Points of interest
 
+### Merge FULL with DIFF, creating new FULL
+
+Over time, the DIFF archives become larger and larger. At some point one wishes to create a new FULL archive to do DIFF's on.
+One way to do that, is to let dar create a FULL archive from scratch, another is to merge a FULL archive with a DIFF, and from there do DIFF's until they once again gets too large for your taste.
+
+I do backups of my homedir. Here it is shown how a FULL archive is merged with a DIFF, creating a new FULL archive.
+
+```` bash
+dar --merge pj-homedir_FULL_2021-09-12  -A pj-homedir_FULL_2021-06-06  -@pj-homedir_DIFF_2021-08-29 -s 12G
+
+# test the new FULL archive
+dar -t pj-homedir_FULL_2021-09-12
+
+# create Par2 redundancy files
+for file in pj-homedir_FULL_yyyy-mm-dd.*.dar; do
+  par2 c -r5 -n1 "$file"
+done
+
+````
+
 ### dar manager databases
 
 `dar-backup` now saves archive catalogs in dar catalog databases.
 
 This makes it easier to restore to a given date when having many FULL, DIFF and INCR archives.
 
+### Performance tip due to par2
+
+This [dar benchmark page](https://dar.sourceforge.io/doc/benchmark.html) has an interesting note on the slice size.
+
+Slice size should be smaller than available RAM, apparently a large performance hit can be avoided keeping the the par2 data in memory.
+
 ### .darrc sets -vd -vf (since v0.6.4)
 
 These .darrc settings make `dar` print the current directory being processed (-vd) and some stats after (-vf)

{dar_backup-0.6.10.dist-info → dar_backup-0.6.11.dist-info}/RECORD

@@ -1,14 +1,14 @@
 dar_backup/.darrc,sha256=-aerqivZmOsW_XBCh9IfbYTUvw0GkzDSr3Vx4GcNB1g,2113
-dar_backup/__about__.py,sha256=G-zfU8sHQRTZnZK0te1cvbxqeLhT1z86XeAYFNUON6Q,22
+dar_backup/__about__.py,sha256=WAW-bfIrNx18qYeBN4lihEQibyosiKElDt_p7DfZELo,22
 dar_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dar_backup/clean_log.py,sha256=VXKA2BMyQmaC6R08Bq9a3wP3mczdFb_moy6HkL-mnF8,5176
 dar_backup/cleanup.py,sha256=9yEdRR84XPtEvBGc2QfwGBQl2tdTPttjetHeiSc_TsM,11419
 dar_backup/config_settings.py,sha256=CBMUhLOOZ-x7CRdS3vBDk4TYaGqC4N1Ot8IMH-qPaI0,3617
-dar_backup/dar_backup.py,sha256=hDy7aXU-XiWOtW40Pxql441liNkSYKGU76eOwy8m7fU,32714
+dar_backup/dar_backup.py,sha256=o4DvaVsqWHb1PxHIXfdr2qyqW6KRRGNcrR2_CRbPy1I,32942
 dar_backup/manager.py,sha256=HDa8eYF89QFhlBRR4EWRzzmswOW00S_w8ToZ5SARO_o,21359
-dar_backup/util.py,sha256=SSSJYM9lQZfubhTUBlX1xDGWmCpYEF3ePARmlY544xM,11283
-dar_backup-0.6.10.dist-info/METADATA,sha256=CEgbqp93sB_cPnEDWuLn7gphZjdrn4zP_5dShC2Buv8,67980
-dar_backup-0.6.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dar_backup-0.6.10.dist-info/entry_points.txt,sha256=p6c4uQLjlTIVP1Od2iorGefrVUH0IWZdFRMl63mNaRg,164
-dar_backup-0.6.10.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dar_backup-0.6.10.dist-info/RECORD,,
+dar_backup/util.py,sha256=E-sEBQZY1hmdeVx5xNE22zKQ0BXDee1eI9F1-w7Fq1Q,15756
+dar_backup-0.6.11.dist-info/METADATA,sha256=N7ICBJQEVDC01tza9lRAkmOXCIAILoDrg-mrHG926f0,71626
+dar_backup-0.6.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dar_backup-0.6.11.dist-info/entry_points.txt,sha256=p6c4uQLjlTIVP1Od2iorGefrVUH0IWZdFRMl63mNaRg,164
+dar_backup-0.6.11.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dar_backup-0.6.11.dist-info/RECORD,,