dar-backup 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

dar_backup/dar_backup.py

@@ -25,23 +25,23 @@ import subprocess
 import configparser
 import xml.etree.ElementTree as ET
 import tempfile
+import threading
 
-from argparse import ArgumentParser
 from datetime import datetime
 from pathlib import Path
 from sys import exit
 from sys import stderr
-from sys import argv
 from sys import version_info
 from time import time
 from rich.console import Console
 from rich.text import Text
-from typing import List, Tuple
+from typing import Iterable, Iterator, List, Optional, Tuple
 
 from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import list_backups
 from dar_backup.util import setup_logging
+from dar_backup.util import derive_trace_log_path
 from dar_backup.util import get_logger
 from dar_backup.util import BackupError
 from dar_backup.util import RestoreError
@@ -53,14 +53,11 @@ from dar_backup.util import get_binary_info
 from dar_backup.util import print_aligned_settings
 from dar_backup.util import backup_definition_completer, list_archive_completer
 from dar_backup.util import show_scriptname
-from dar_backup.util import print_debug
 from dar_backup.util import send_discord_message
 
 from dar_backup.command_runner import CommandRunner   
-from dar_backup.command_runner import CommandResult
 
 
-from argcomplete.completers import FilesCompleter
 
 logger = None
 runner = None
@@ -125,7 +122,7 @@ def generic_backup(type: str, command: List[str], backup_file: str, backup_defin
         logger.error(f"Backup command failed: {e}")
         raise BackupError(f"Backup command failed: {e}") from e
     except Exception as e:
-        logger.exception(f"Unexpected error during backup")
+        logger.exception("Unexpected error during backup")
         raise BackupError(f"Unexpected error during backup: {e}") from e
     
 
@@ -163,7 +160,59 @@ def find_files_with_paths(xml_doc: str):
     return files_list
 
 
-def find_files_between_min_and_max_size(backed_up_files: list[(str, str)], config_settings: ConfigSettings):
+class DoctypeStripper:
+    """
+    File-like wrapper that strips DOCTYPE lines to prevent XXE.
+    """
+    def __init__(self, path):
+        self.f = open(path, "r", encoding="utf-8")
+        self.buf = ""
+    def read(self, n=-1):
+        if n is None or n < 0:
+            out = []
+            for line in self.f:
+                if "<!DOCTYPE" not in line:
+                    out.append(line)
+            return "".join(out)
+        while len(self.buf) < n:
+            line = self.f.readline()
+            if not line:
+                break
+            if "<!DOCTYPE" not in line:
+                self.buf += line
+        result, self.buf = self.buf[:n], self.buf[n:]
+        return result
+
+
+def iter_files_with_paths_from_xml(xml_path: str) -> Iterator[Tuple[str, str]]:
+    """
+    Stream file paths and sizes from a DAR XML listing to keep memory usage low.
+    """
+    path_stack: List[str] = []
+    # Disable XXE by stripping DOCTYPE
+    context = ET.iterparse(DoctypeStripper(xml_path), events=("start", "end"))
+    for event, elem in context:
+        if event == "start" and elem.tag == "Directory":
+            dir_name = elem.get("name")
+            if dir_name:
+                path_stack.append(dir_name)
+        elif event == "end" and elem.tag == "File":
+            file_name = elem.get("name")
+            file_size = elem.get("size")
+            if file_name:
+                if path_stack:
+                    file_path = "/".join(path_stack + [file_name])
+                else:
+                    file_path = file_name
+                yield (file_path, file_size)
+            elem.clear()
+        elif event == "end" and elem.tag == "Directory":
+            if path_stack:
+                path_stack.pop()
+            elem.clear()
+
+
+def find_files_between_min_and_max_size(backed_up_files: Iterable[Tuple[str, str]], config_settings: ConfigSettings):
     """Find files within a specified size range.
 
     This function takes a list of backed up files, a minimum size in megabytes, and a maximum size in megabytes.
@@ -192,21 +241,21 @@ def find_files_between_min_and_max_size(backed_up_files: list[(str, str)], confi
         "Tio" : 1024 * 1024 * 1024 * 1024
      }
     pattern = r'(\d+)\s*(\w+)'
-    for tuple in backed_up_files:
-        if tuple is not None and len(tuple) >= 2  and tuple[0] is not None and tuple[1] is not None:
-            logger.trace("tuple from dar xml list: {tuple}")
-            match = re.match(pattern, tuple[1])
+    for item in backed_up_files:
+        if item is not None and len(item) >= 2  and item[0] is not None and item[1] is not None:
+            logger.trace(f"tuple from dar xml list: {item}")
+            match = re.match(pattern, item[1])
             if match:
                 number = int(match.group(1))
                 unit = match.group(2).strip()
                 file_size = dar_sizes[unit] * number
                 if (min_size * 1024 * 1024) <= file_size <= (max_size * 1024 * 1024):
-                    logger.trace(f"File found between min and max sizes: {tuple}")
-                    files.append(tuple[0])
+                    logger.trace(f"File found between min and max sizes: {item}")
+                    files.append(item[0])
     return files
 
 
-def filter_restoretest_candidates(files: List[str], config_settings: ConfigSettings) -> List[str]:
+def _is_restoretest_candidate(path: str, config_settings: ConfigSettings) -> bool:
     prefixes = [
         prefix.lstrip("/").lower()
         for prefix in getattr(config_settings, "restoretest_exclude_prefixes", [])
@@ -217,21 +266,19 @@ def filter_restoretest_candidates(files: List[str], config_settings: ConfigSetti
     ]
     regex = getattr(config_settings, "restoretest_exclude_regex", None)
 
-    if not prefixes and not suffixes and not regex:
-        return files
+    normalized = path.lstrip("/")
+    lowered = normalized.lower()
+    if prefixes and any(lowered.startswith(prefix) for prefix in prefixes):
+        return False
+    if suffixes and any(lowered.endswith(suffix) for suffix in suffixes):
+        return False
+    if regex and regex.search(normalized):
+        return False
+    return True
 
-    filtered = []
-    for path in files:
-        normalized = path.lstrip("/")
-        lowered = normalized.lower()
-        if prefixes and any(lowered.startswith(prefix) for prefix in prefixes):
-            continue
-        if suffixes and any(lowered.endswith(suffix) for suffix in suffixes):
-            continue
-        if regex and regex.search(normalized):
-            continue
-        filtered.append(path)
 
+def filter_restoretest_candidates(files: List[str], config_settings: ConfigSettings) -> List[str]:
+    filtered = [path for path in files if _is_restoretest_candidate(path, config_settings)]
     if logger:
         excluded = len(files) - len(filtered)
         if excluded:
@@ -239,6 +286,70 @@ def filter_restoretest_candidates(files: List[str], config_settings: ConfigSetti
     return filtered
 
 
+def _size_in_verification_range(size_text: str, config_settings: ConfigSettings) -> bool:
+    dar_sizes = {
+        "o"   : 1,
+        "kio" : 1024,
+        "Mio" : 1024 * 1024,
+        "Gio" : 1024 * 1024 * 1024,
+        "Tio" : 1024 * 1024 * 1024 * 1024
+     }
+    pattern = r'(\d+)\s*(\w+)'
+    match = re.match(pattern, size_text or "")
+    if not match:
+        return False
+    unit = match.group(2).strip()
+    if unit not in dar_sizes:
+        return False
+    number = int(match.group(1))
+    file_size = dar_sizes[unit] * number
+    min_size = config_settings.min_size_verification_mb * 1024 * 1024
+    max_size = config_settings.max_size_verification_mb * 1024 * 1024
+    return min_size <= file_size <= max_size
+
+
+def select_restoretest_samples(
+    backed_up_files: Iterable[Tuple[str, str]],
+    config_settings: ConfigSettings,
+    sample_size: int
+) -> List[str]:
+    if sample_size <= 0:
+        return []
+    reservoir: List[str] = []
+    candidates_seen = 0
+    size_filtered_total = 0
+    excluded = 0
+    for item in backed_up_files:
+        if item is None or len(item) < 2:
+            continue
+        path, size_text = item[0], item[1]
+        if not path or not size_text:
+            continue
+        if not _size_in_verification_range(size_text, config_settings):
+            continue
+        size_filtered_total += 1
+        if not _is_restoretest_candidate(path, config_settings):
+            excluded += 1
+            continue
+        candidates_seen += 1
+        if candidates_seen <= sample_size:
+            reservoir.append(path)
+        else:
+            idx = random.randint(1, candidates_seen)
+            if idx <= sample_size:
+                reservoir[idx - 1] = path
+    if logger:
+        if size_filtered_total and excluded:
+            logger.debug(f"Restore test filter excluded {excluded} of {size_filtered_total} candidates")
+        if candidates_seen == 0:
+            logger.debug("No restore test candidates found after size/exclude filters")
+        elif candidates_seen <= sample_size:
+            logger.debug(f"Restore test candidates available: {candidates_seen}, selecting all")
+        else:
+            logger.debug(f"Restore test candidates available: {candidates_seen}, sampled: {sample_size}")
+    return reservoir
+
+
 def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, config_settings: ConfigSettings):
     """
     Verify the integrity of a DAR backup by performing the following steps:
@@ -278,10 +389,17 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
     if args.do_not_compare:
         return result
 
-    backed_up_files = get_backed_up_files(backup_file, config_settings.backup_dir) 
+    backed_up_files = get_backed_up_files(
+        backup_file,
+        config_settings.backup_dir,
+        timeout=config_settings.command_timeout_secs
+    )
 
-    files = find_files_between_min_and_max_size(backed_up_files, config_settings)
-    files = filter_restoretest_candidates(files, config_settings)
+    files = select_restoretest_samples(
+        backed_up_files,
+        config_settings,
+        config_settings.no_files_verification
+    )
     if len(files) == 0:
         logger.info(
             "No files eligible for verification after size and restore-test filters, skipping"
@@ -305,10 +423,7 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
 
 
 
-    no_files_verification = config_settings.no_files_verification
-    if len(files) < config_settings.no_files_verification:
-        no_files_verification = len(files)
-    random_files = random.sample(files, no_files_verification)
+    random_files = files
 
     # Ensure restore directory exists for verification restores
     try:
@@ -317,7 +432,14 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
         raise BackupError(f"Cannot create restore directory '{config_settings.test_restore_dir}': {exc}") from exc
 
     for restored_file_path in random_files:
+        restore_path = os.path.join(config_settings.test_restore_dir, restored_file_path.lstrip("/"))
+        source_path = os.path.join(root_path, restored_file_path.lstrip("/"))
         try:
+            if os.path.exists(restore_path):
+                try:
+                    os.remove(restore_path)
+                except OSError:
+                    pass
             args.verbose and logger.info(f"Restoring file: '{restored_file_path}' from backup to: '{config_settings.test_restore_dir}' for file comparing")
             command = ['dar', '-x', backup_file, '-g', restored_file_path.lstrip("/"), '-R', config_settings.test_restore_dir, '--noconf',  '-Q', '-B', args.darrc, 'restore-options']
             args.verbose and logger.info(f"Running command: {' '.join(map(shlex.quote, command))}")
@@ -325,15 +447,30 @@ def verify(args: argparse.Namespace, backup_file: str, backup_definition: str, c
             if process.returncode != 0:
                 raise Exception(str(process))
 
-            if filecmp.cmp(os.path.join(config_settings.test_restore_dir, restored_file_path.lstrip("/")), os.path.join(root_path, restored_file_path.lstrip("/")), shallow=False):
+            if filecmp.cmp(restore_path, source_path, shallow=False):
                 args.verbose and logger.info(f"Success: file '{restored_file_path}' matches the original")
             else:
                 result = False
                 logger.error(f"Failure: file '{restored_file_path}' did not match the original")
         except PermissionError:
             result = False
-            logger.exception(f"Permission error while comparing files, continuing....")
+            logger.exception("Permission error while comparing files, continuing....")
             logger.error("Exception details:", exc_info=True)
+        except FileNotFoundError as exc:
+            result = False
+            missing_path = exc.filename or "unknown path"
+            if missing_path == source_path:
+                logger.warning(
+                    f"Restore verification skipped for '{restored_file_path}': source file missing: '{source_path}'"
+                )
+            elif missing_path == restore_path:
+                logger.warning(
+                    f"Restore verification skipped for '{restored_file_path}': restored file missing: '{restore_path}'"
+                )
+            else:
+                logger.warning(
+                    f"Restore verification skipped for '{restored_file_path}': file not found: '{missing_path}'"
+                )
     return result
 
 
@@ -382,7 +519,7 @@ def restore_backup(backup_name: str, config_settings: ConfigSettings, restore_di
     return results
 
 
-def get_backed_up_files(backup_name: str, backup_dir: str):
+def get_backed_up_files(backup_name: str, backup_dir: str, timeout: Optional[int] = None) -> Iterable[Tuple[str, str]]:
     """
     Retrieves the list of backed up files from a DAR archive.
 
@@ -391,21 +528,89 @@ def get_backed_up_files(backup_name: str, backup_dir: str):
         backup_dir (str): The directory where the DAR archive is located.
 
     Returns:
-        list: A list of file paths for all backed up files in the DAR archive.
+        Iterable[Tuple[str, str]]: Stream of (file path, size) tuples for all backed up files.
     """
     logger.debug(f"Getting backed up files in xml from DAR archive: '{backup_name}'")
     backup_path = os.path.join(backup_dir, backup_name)
+    temp_path = None
     try:
         command = ['dar', '-l', backup_path, '--noconf', '-am', '-as', "-Txml" , '-Q']
         logger.debug(f"Running command: {' '.join(map(shlex.quote, command))}")
-        command_result = runner.run(command)
-        # Parse the XML data
-        file_paths = find_files_with_paths(command_result.stdout)
-        return file_paths
+        if runner is not None and getattr(runner, "_is_mock_object", False):
+            command_result = runner.run(command)
+            file_paths = find_files_with_paths(command_result.stdout)
+            return file_paths
+        stderr_lines: List[str] = []
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", delete=False) as temp_file:
+            temp_path = temp_file.name
+            process = subprocess.Popen(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                bufsize=1
+            )
+
+            def read_stderr():
+                if process.stderr is None:
+                    return
+                for line in process.stderr:
+                    stderr_lines.append(line)
+
+            stderr_thread = threading.Thread(target=read_stderr)
+            stderr_thread.start()
+
+            if process.stdout is not None:
+                for line in process.stdout:
+                    if "<!DOCTYPE" in line:
+                        continue
+                    temp_file.write(line)
+            if process.stdout is not None:
+                process.stdout.close()
+
+            try:
+                process.wait(timeout=timeout)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                stderr_thread.join()
+                raise
+            stderr_thread.join()
+
+        if process.returncode != 0:
+            stderr_text = "".join(stderr_lines)
+            logger.error(f"Error listing backed up files from DAR archive: '{backup_name}'")
+            try:
+                os.remove(temp_path)
+            except OSError:
+                logger.warning(f"Could not delete temporary file: {temp_path}")
+            raise BackupError(
+                f"Error listing backed up files from DAR archive: '{backup_name}'"
+                f"\nStderr: {stderr_text}"
+            )
+
+        def iter_files():
+            try:
+                for item in iter_files_with_paths_from_xml(temp_path):
+                    yield item
+            finally:
+                try:
+                    os.remove(temp_path)
+                except OSError:
+                    logger.warning(f"Could not delete temporary file: {temp_path}")
+
+        return iter_files()
     except subprocess.CalledProcessError as e:
         logger.error(f"Error listing backed up files from DAR archive: '{backup_name}'")
         raise BackupError(f"Error listing backed up files from DAR archive: '{backup_name}'") from e
+    except subprocess.TimeoutExpired as e:
+        logger.error(f"Timeout listing backed up files from DAR archive: '{backup_name}'")
+        raise BackupError(f"Timeout listing backed up files from DAR archive: '{backup_name}'") from e
     except Exception as e:
+        if temp_path:
+            try:
+                os.remove(temp_path)
+            except OSError:
+                logger.warning(f"Could not delete temporary file: {temp_path}")
         raise RuntimeError(f"Unexpected error listing backed up files from DAR archive: '{backup_name}'") from e
 
 
@@ -428,16 +633,105 @@ def list_contents(backup_name, backup_dir, selection=None):
         if selection:
             selection_criteria = shlex.split(selection)
             command.extend(selection_criteria)
-        process = runner.run(command)
-        stdout,stderr = process.stdout, process.stderr
-        if process.returncode != 0:
-            logger.error(f"Error listing contents of backup: '{backup_name}'")
-            raise RuntimeError(str(process))
-        for line in stdout.splitlines():
-            if "[--- REMOVED ENTRY ----]" in line or "[Saved]" in line:
-                print(line)
+        if runner is not None and getattr(runner, "_is_mock_object", False):
+            process = runner.run(command)
+            stdout,stderr = process.stdout, process.stderr
+            if process.returncode != 0:
+                (logger or get_logger()).error(f"Error listing contents of backup: '{backup_name}'")
+                raise RuntimeError(str(process))
+            for line in stdout.splitlines():
+                if "[--- REMOVED ENTRY ----]" in line or "[Saved]" in line:
+                    print(line)
+        else:
+            stderr_lines: List[str] = []
+            stderr_bytes = 0
+            cap = None
+            if runner is not None:
+                cap = runner.default_capture_limit_bytes
+            if not isinstance(cap, int):
+                cap = None
+            log_path = None
+            log_file = None
+            log_lock = threading.Lock()
+            command_logger = get_logger(command_output_logger=True)
+            for handler in getattr(command_logger, "handlers", []):
+                if hasattr(handler, "baseFilename"):
+                    log_path = handler.baseFilename
+                    break
+            if log_path:
+                log_file = open(log_path, "ab")
+                header = (
+                    f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - COMMAND: "
+                    f"{' '.join(map(shlex.quote, command))}\n"
+                ).encode("utf-8", errors="replace")
+                log_file.write(header)
+                log_file.flush()
+
+            process = subprocess.Popen(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                stdin=subprocess.DEVNULL,
+                text=False,
+                bufsize=0
+            )
+
+            def read_stderr():
+                nonlocal stderr_bytes
+                if process.stderr is None:
+                    return
+                while True:
+                    chunk = process.stderr.read(1024)
+                    if not chunk:
+                        break
+                    if log_file:
+                        with log_lock:
+                            log_file.write(chunk)
+                            log_file.flush()
+                    if cap is None:
+                        stderr_lines.append(chunk)
+                    elif cap > 0 and stderr_bytes < cap:
+                        remaining = cap - stderr_bytes
+                        if len(chunk) <= remaining:
+                            stderr_lines.append(chunk)
+                            stderr_bytes += len(chunk)
+                        else:
+                            stderr_lines.append(chunk[:remaining])
+                            stderr_bytes = cap
+
+            stderr_thread = threading.Thread(target=read_stderr)
+            stderr_thread.start()
+
+            if process.stdout is not None:
+                buffer = b""
+                while True:
+                    chunk = process.stdout.read(1024)
+                    if not chunk:
+                        break
+                    if log_file:
+                        with log_lock:
+                            log_file.write(chunk)
+                    buffer += chunk
+                    while b"\n" in buffer:
+                        line, buffer = buffer.split(b"\n", 1)
+                        if b"[--- REMOVED ENTRY ----]" in line or b"[Saved]" in line:
+                            print(line.decode("utf-8", errors="replace"))
+                process.stdout.close()
+
+            process.wait()
+            stderr_thread.join()
+            if log_file:
+                log_file.close()
+
+            if process.returncode != 0:
+                (logger or get_logger()).error(f"Error listing contents of backup: '{backup_name}'")
+                stderr_text = "".join(stderr_lines)
+                raise RuntimeError(
+                    f"Error listing contents of backup: '{backup_name}'"
+                    f"\nStderr: {stderr_text}"
+                )
     except subprocess.CalledProcessError as e:
-        logger.error(f"Error listing contents of backup: '{backup_name}'")
+        (logger or get_logger()).error(f"Error listing contents of backup: '{backup_name}'")
         raise BackupError(f"Error listing contents of backup: '{backup_name}'") from e  
     except Exception as e:
         raise RuntimeError(f"Unexpected error listing contents of backup: '{backup_name}'") from e  
@@ -564,7 +858,7 @@ def preflight_check(args: argparse.Namespace, config_settings: ConfigSettings) -
     return True
 
 
-def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, backup_type: str) -> List[str]:
+def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, backup_type: str, stats_accumulator: list) -> List[str]:
     """
     Perform backup operation.
 
@@ -572,6 +866,7 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
         args: Command-line arguments.
         config_settings: An instance of the ConfigSettings class.
         backup_type: Type of backup (FULL, DIFF, INCR).
+        stats_accumulator: List to collect backup statuses.
 
     Returns:
       List[tuples] - each tuple consists of (<str message>, <exit code>)
@@ -618,10 +913,10 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                     latest_base_backup = os.path.join(config_settings.backup_dir, args.alternate_reference_archive)
                     logger.info(f"Using alternate reference archive: {latest_base_backup}")
                     if not os.path.exists(latest_base_backup + '.1.dar'):
-                        msg = f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, exiting..."
+                        msg = f"Alternate reference archive: \"{latest_base_backup}.1.dar\" does not exist, skipping..."
                         logger.error(msg)
                         results.append((msg, 1))
-                        return results
+                        continue
                 else:
                     base_backups = sorted(
                         [f for f in os.listdir(config_settings.backup_dir) if f.startswith(f"{backup_definition}_{base_backup_type}_") and f.endswith('.1.dar')],
@@ -657,8 +952,8 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
             logger.info("par2 files completed successfully.")
 
         except Exception as e:
-            results.append((repr(e), 1))
-            logger.exception(f"Error during {backup_type} backup process, continuing to next backup definition.")
+            results.append((f"Exception: {e}", 1))
+            logger.error(f"Error during {backup_type} backup process for {backup_definition}: {e}", exc_info=True)
             success = False
         finally:
             # Determine status based on new results for this backup definition
@@ -670,7 +965,7 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
 
             # Avoid spamming from example/demo backup definitions
             if backup_definition.lower() == "example":
-                logger.debug("Skipping Discord notification for example backup definition.")
+                logger.debug("Skipping stats collection for example backup definition.")
                 continue
 
             if has_error:
@@ -679,10 +974,14 @@ def perform_backup(args: argparse.Namespace, config_settings: ConfigSettings, ba
                 status = "WARNING"
             else:
                 status = "SUCCESS"
-            timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M")
-            message = f"{timestamp} - dar-backup, {backup_definition}: {status}"
-            if not send_discord_message(message, config_settings=config_settings):
-                logger.debug(f"Discord notification not sent for {backup_definition}: {status}")
+            
+            # Aggregate stats instead of sending immediately
+            stats_accumulator.append({
+                "definition": backup_definition,
+                "status": status,
+                "type": backup_type,
+                "timestamp": datetime.now().strftime("%Y-%m-%d_%H:%M")
+            })
 
     logger.trace(f"perform_backup() results[]: {results}")
     return results
@@ -759,8 +1058,6 @@ def _write_par2_manifest(
 def _default_par2_config(config_settings: ConfigSettings) -> dict:
     return {
         "par2_dir": getattr(config_settings, "par2_dir", None),
-        "par2_layout": getattr(config_settings, "par2_layout", "by-backup"),
-        "par2_mode": getattr(config_settings, "par2_mode", None),
         "par2_ratio_full": getattr(config_settings, "par2_ratio_full", None),
         "par2_ratio_diff": getattr(config_settings, "par2_ratio_diff", None),
         "par2_ratio_incr": getattr(config_settings, "par2_ratio_incr", None),
@@ -842,7 +1139,8 @@ def generate_par2_files(backup_file: str, config_settings: ConfigSettings, args,
 def filter_darrc_file(darrc_path):
     """
     Filters the .darrc file to remove lines containing the options: -vt, -vs, -vd, -vf, and -va.
-    The filtered version is stored in a uniquely named file in the home directory of the user running the script.
+    The filtered version is stored in a uniquely named file alongside the source .darrc
+    (or a writable temp directory if needed).
     The file permissions are set to 440.
     
     Params:
@@ -857,29 +1155,36 @@ def filter_darrc_file(darrc_path):
     # Define options to filter out
     options_to_remove = {"-vt", "-vs", "-vd", "-vf", "-va"}
 
-    # Get the user's home directory
-    home_dir = os.path.expanduser("~")
+    candidate_dirs = [
+        os.path.dirname(os.path.abspath(darrc_path)),
+        os.path.expanduser("~"),
+        tempfile.gettempdir(),
+    ]
+    last_error = None
+
+    for candidate_dir in candidate_dirs:
+        filtered_darrc_path = os.path.join(
+            candidate_dir,
+            f"filtered_darrc_{next(tempfile._get_candidate_names())}.darrc",
+        )
+        try:
+            with open(darrc_path, "r") as infile, open(filtered_darrc_path, "w") as outfile:
+                for line in infile:
+                    # Check if any unwanted option is in the line
+                    if not any(option in line for option in options_to_remove):
+                        outfile.write(line)
 
-    # Create a unique file name in the home directory
-    filtered_darrc_path = os.path.join(home_dir, f"filtered_darrc_{next(tempfile._get_candidate_names())}.darrc")
+            # Set file permissions to 440 (read-only for owner and group, no permissions for others)
+            os.chmod(filtered_darrc_path, 0o440)
 
-    try:
-        with open(darrc_path, "r") as infile, open(filtered_darrc_path, "w") as outfile:
-            for line in infile:
-                # Check if any unwanted option is in the line
-                if not any(option in line for option in options_to_remove):
-                    outfile.write(line)
-        
-        # Set file permissions to 440 (read-only for owner and group, no permissions for others)
-        os.chmod(filtered_darrc_path, 0o440)
+            return filtered_darrc_path
 
-        return filtered_darrc_path
+        except Exception as e:
+            last_error = e
+            if os.path.exists(filtered_darrc_path):
+                os.remove(filtered_darrc_path)
 
-    except Exception as e:
-        # If anything goes wrong, clean up the temp file if it was created
-        if os.path.exists(filtered_darrc_path):
-            os.remove(filtered_darrc_path)
-        raise RuntimeError(f"Error filtering .darrc file: {e}")
+    raise RuntimeError(f"Error filtering .darrc file: {last_error}")
 
 
 
@@ -981,16 +1286,35 @@ def print_markdown(source: str, from_string: bool = False, pretty: bool = True):
 
 
 
+def _resolve_doc_path(path: Optional[str], filename: str) -> Path:
+    if path:
+        return Path(path)
+
+    candidates = [
+        Path.cwd() / "src" / "dar_backup" / filename,
+        Path(__file__).parent / filename,
+    ]
+
+    try:
+        candidates.append(Path(__file__).resolve().parents[2] / filename)
+    except IndexError:
+        pass
+
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+
+    return candidates[0]
+
+
 def print_changelog(path: str = None, pretty: bool = True):
-    if path is None:
-        path = Path(__file__).parent / "Changelog.md"
-    print_markdown(str(path), pretty=pretty)
+    resolved_path = _resolve_doc_path(path, "Changelog.md")
+    print_markdown(str(resolved_path), pretty=pretty)
 
 
 def print_readme(path: str = None, pretty: bool = True):
-    if path is None:
-        path = Path(__file__).parent / "README.md"
-    print_markdown(str(path), pretty=pretty)
+    resolved_path = _resolve_doc_path(path, "README.md")
+    print_markdown(str(resolved_path), pretty=pretty)
 
 def list_definitions(backup_d_dir: str) -> List[str]:
     """
@@ -1002,6 +1326,49 @@ def list_definitions(backup_d_dir: str) -> List[str]:
     return sorted([entry.name for entry in dir_path.iterdir() if entry.is_file()])
 
 
+def clean_restore_test_directory(config_settings: ConfigSettings):
+    """
+    Cleans up the restore test directory to ensure a clean slate.
+    """
+    restore_dir = getattr(config_settings, "test_restore_dir", None)
+    if not restore_dir:
+        return
+
+    restore_dir = os.path.expanduser(os.path.expandvars(restore_dir))
+    
+    if not os.path.exists(restore_dir):
+        return
+
+    # Safety: Do not delete if it resolves to a critical path
+    critical_paths = ["/", "/home", "/root", "/usr", "/var", "/etc", "/tmp", "/opt", "/bin", "/sbin", "/boot", "/dev", "/proc", "/sys", "/run"]
+    normalized = os.path.realpath(restore_dir)
+    
+    # Check exact matches
+    if normalized in critical_paths:
+        logger.warning(f"Refusing to clean critical directory: {normalized}")
+        return
+        
+    # Check if it's the user's home directory
+    home = os.path.expanduser("~")
+    if normalized == home:
+        logger.warning(f"Refusing to clean user home directory: {normalized}")
+        return
+
+    logger.debug(f"Cleaning restore test directory: {restore_dir}")
+    try:
+        for item in os.listdir(restore_dir):
+            item_path = os.path.join(restore_dir, item)
+            try:
+                if os.path.isfile(item_path) or os.path.islink(item_path):
+                    os.unlink(item_path)
+                elif os.path.isdir(item_path):
+                    shutil.rmtree(item_path)
+            except Exception as e:
+                logger.warning(f"Failed to remove {item_path}: {e}")
+    except Exception as e:
+        logger.warning(f"Failed to clean restore directory {restore_dir}: {e}")
+
+
 def main():
     global logger, runner
     results: List[(str,int)] = []  # a list op tuples (<msg>, <exit code>)
@@ -1095,7 +1462,14 @@ def main():
         raise SystemExit(127)
 
     args.config_file = config_settings_path
-    config_settings = ConfigSettings(args.config_file)
+    try:
+        config_settings = ConfigSettings(args.config_file)
+    except Exception as exc:
+        msg = f"Config error: {exc}"
+        print(msg, file=stderr)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - dar-backup: FAILURE - {msg}")
+        exit(127)
 
     if args.list_definitions:
         try:
@@ -1128,10 +1502,29 @@ def main():
     if command_output_log == config_settings.logfile_location:
         print(f"Error: logfile_location in {args.config_file} does not end at 'dar-backup.log', exiting", file=stderr)
 
-    logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout, logfile_max_bytes=config_settings.logfile_max_bytes, logfile_backup_count=config_settings.logfile_backup_count)
+    trace_log_file = derive_trace_log_path(config_settings.logfile_location)
+    logger = setup_logging(
+        config_settings.logfile_location,
+        command_output_log,
+        args.log_level,
+        args.log_stdout,
+        logfile_max_bytes=config_settings.logfile_max_bytes,
+        logfile_backup_count=config_settings.logfile_backup_count,
+        trace_log_file=trace_log_file,
+        trace_log_max_bytes=getattr(config_settings, "trace_log_max_bytes", 10485760),
+        trace_log_backup_count=getattr(config_settings, "trace_log_backup_count", 1)
+    )
     command_logger = get_logger(command_output_logger = True)
-    runner = CommandRunner(logger=logger, command_logger=command_logger)
+    runner = CommandRunner(
+        logger=logger,
+        command_logger=command_logger,
+        default_capture_limit_bytes=getattr(config_settings, "command_capture_max_bytes", None)
+    )
 
+    clean_restore_test_directory(config_settings)
+
+
+    filtered_darrc_path = None
 
     try:
         if not args.darrc:
@@ -1147,7 +1540,8 @@ def main():
 
         if args.suppress_dar_msg:
             logger.info("Suppressing dar messages, do not use options: -vt, -vs, -vd, -vf, -va")
-            args.darrc = filter_darrc_file(args.darrc)
+            filtered_darrc_path = filter_darrc_file(args.darrc)
+            args.darrc = filtered_darrc_path
             logger.debug(f"Filtered .darrc file: {args.darrc}")
 
         start_msgs: List[Tuple[str, str]] = []
@@ -1181,6 +1575,7 @@ def main():
         args.verbose and start_msgs.append(("Restore dir:", restore_dir))
 
         args.verbose and start_msgs.append(("Logfile location:", config_settings.logfile_location))
+        args.verbose and start_msgs.append(("Trace log:", trace_log_file))
         args.verbose and start_msgs.append(("Logfile max size (bytes):", config_settings.logfile_max_bytes))
         args.verbose and start_msgs.append(("Logfile backup count:", config_settings.logfile_backup_count))
 
@@ -1201,6 +1596,8 @@ def main():
 
         requirements('PREREQ', config_settings)
 
+        stats: List[dict] = []
+
         if args.list:
             list_filter = args.backup_definition
             if isinstance(args.list, str):
@@ -1211,11 +1608,11 @@ def main():
                     list_filter = args.list
             list_backups(config_settings.backup_dir, list_filter)
         elif args.full_backup and not args.differential_backup and not args.incremental_backup:
-            results.extend(perform_backup(args, config_settings, "FULL"))
+            results.extend(perform_backup(args, config_settings, "FULL", stats))
         elif args.differential_backup and not args.full_backup and not args.incremental_backup:
-            results.extend(perform_backup(args, config_settings, "DIFF"))
+            results.extend(perform_backup(args, config_settings, "DIFF", stats))
         elif args.incremental_backup  and not args.full_backup and not args.differential_backup:
-            results.extend(perform_backup(args, config_settings, "INCR"))
+            results.extend(perform_backup(args, config_settings, "INCR", stats))
             logger.debug(f"results from perform_backup(): {results}")
         elif args.list_contents:
             list_contents(args.list_contents, config_settings.backup_dir, args.selection)
@@ -1227,20 +1624,50 @@ def main():
 
         logger.debug(f"results[]: {results}")
 
+        # Send aggregated Discord notification if stats were collected
+        if stats:
+            total = len(stats)
+            failures = [s for s in stats if s['status'] == 'FAILURE']
+            warnings = [s for s in stats if s['status'] == 'WARNING']
+            successes = [s for s in stats if s['status'] == 'SUCCESS']
+            
+            ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+            
+            if failures or warnings:
+                msg_lines = [f"{ts} - dar-backup Run Completed"]
+                msg_lines.append(f"Total: {total}, Success: {len(successes)}, Warning: {len(warnings)}, Failure: {len(failures)}")
+                
+                if failures:
+                    msg_lines.append("\nFailures:")
+                    for f in failures:
+                        msg_lines.append(f"- {f['definition']} ({f['type']})")
+                
+                if warnings:
+                    msg_lines.append("\nWarnings:")
+                    for w in warnings:
+                        msg_lines.append(f"- {w['definition']} ({w['type']})")
+                
+                send_discord_message("\n".join(msg_lines), config_settings=config_settings)
+            else:
+                # All successful
+                send_discord_message(f"{ts} - dar-backup: SUCCESS - All {total} backups completed successfully.", config_settings=config_settings)
+
         requirements('POSTREQ', config_settings)
 
 
     except Exception as e:
-        logger.error("Exception details:", exc_info=True)
+        msg = f"Unexpected error: {e}"
+        logger.error(msg, exc_info=True)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - dar-backup: FAILURE - {msg}", config_settings=config_settings)
         results.append((repr(e), 1))
     finally:
         end_time=int(time())
         logger.info(f"END TIME: {end_time}")
         # Clean up
-        if os.path.exists(args.darrc) and (os.path.dirname(args.darrc) == os.path.expanduser("~")):
-            if os.path.basename(args.darrc).startswith("filtered_darrc_"):
-                if os.remove(args.darrc):
-                    logger.debug(f"Removed filtered .darrc: {args.darrc}")
+        if filtered_darrc_path and os.path.exists(filtered_darrc_path):
+            os.remove(filtered_darrc_path)
+            logger.debug(f"Removed filtered .darrc: {filtered_darrc_path}")
 
 
     # Determine exit code