dar-backup 1.0.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

dar_backup/demo.py

@@ -37,17 +37,22 @@ DAR_BACKUP_DIR = util.normalize_dir(util.expand_path("~/dar-backup"))
 
 def check_directories(args, vars_map: Dict[str,str]) -> bool:
     """
-    Check if the directories exist and create them if they don't.
+    Check if target paths already exist and are directories.
 
     Returns:
-        bool: True if the directories were created successfully, False otherwise.
+        bool: True if it is safe to proceed, False otherwise.
     """
     result = True
     for key in ("DAR_BACKUP_DIR","BACKUP_DIR","TEST_RESTORE_DIR","CONFIG_DIR","BACKUP_D_DIR"):
         path = Path(vars_map[key])
-        if path.exists() and not args.override:
-            print(f"Directory '{path}' already exists")
-            result = False
+        if path.exists():
+            if not path.is_dir():
+                print(f"Error: '{path}' exists and is not a directory")
+                result = False
+                continue
+            if not args.override:
+                print(f"Directory '{path}' already exists")
+                result = False
     return result
 
 
@@ -72,12 +77,17 @@ def generate_file(args, template: str, file_path: Path, vars_map: Dict[str, str]
     if rendered is None:
         print(f"Error: Template '{template}' could not be rendered.")
         return False    
-    if os.path.exists(file_path) and not args.override:
-        print(f"Error: File '{file_path}' already exists. Use --override to overwrite.")
-        return False
+    if file_path.exists():
+        if file_path.is_dir():
+            print(f"Error: '{file_path}' is a directory, expected a file.")
+            return False
+        if not args.override:
+            print(f"Error: File '{file_path}' already exists. Use --override to overwrite.")
+            return False
     file_path.parent.mkdir(parents=True, exist_ok=True)
     file_path.write_text(rendered)
     print(f"File generated at '{file_path}'")
+    return True
 
 
 
@@ -152,7 +162,6 @@ def main():
         parser.error(
             "Options --root-dir, --dir-to-backup must all be specified together."
         )
-        exit(1)
 
     args.root_dir = util.normalize_dir(util.expand_path(args.root_dir)) if args.root_dir else None
     args.backup_dir = util.normalize_dir(util.expand_path(args.backup_dir)) if args.backup_dir else None

dar_backup/installer.py

@@ -43,6 +43,9 @@ def install_autocompletion():
 
     # ensure RC file and parent directory exist
     rc_file.parent.mkdir(parents=True, exist_ok=True)
+    if rc_file.exists() and rc_file.is_dir():
+        print(f"Error: RC path is a directory: {rc_file}")
+        return
     if not rc_file.exists():
         rc_file.touch()
 
@@ -76,11 +79,17 @@ def uninstall_autocompletion() -> str:
     if not rc_file.exists():
         print(f"❌ RC file not found: {rc_file}")
         return
+    if rc_file.is_dir():
+        print(f"Error: RC path is a directory: {rc_file}")
+        return
 
     content = rc_file.read_text()
     if marker not in content:
         print(f"No autocompletion block found in {rc_file}")
         return f"No autocompletion block found in {rc_file}"  # for unit test
+    if end_marker not in content:
+        print(f"Error: Autocompletion end marker not found in {rc_file}")
+        return f"Autocompletion end marker not found in {rc_file}"
 
     lines = content.splitlines(keepends=True)
     new_lines = []
@@ -128,7 +137,11 @@ def run_installer(config_file: str, create_db_flag: bool):
         log_to_stdout=True,
     )
     command_logger = get_logger(command_output_logger=True)
-    runner = CommandRunner(logger=logger, command_logger=command_logger)
+    runner = CommandRunner(
+        logger=logger,
+        command_logger=command_logger,
+        default_capture_limit_bytes=getattr(config_settings, "command_capture_max_bytes", None)
+    )
 
 
     # Create required directories
@@ -151,6 +164,10 @@ def run_installer(config_file: str, create_db_flag: bool):
     # Optionally create databases for all backup definitions
     if create_db_flag:
         for file in os.listdir(config_settings.backup_d_dir):
+            file_path = os.path.join(config_settings.backup_d_dir, file)
+            if not os.path.isfile(file_path):
+                logger.info(f"Skipping non-file backup definition: {file}")
+                continue
             backup_def = os.path.basename(file)
             print(f"Creating catalog for: {backup_def}")
             result = create_db(backup_def, config_settings, logger, runner)

dar_backup/manager.py

@@ -27,6 +27,8 @@ import os
 import re
 import sys
 import subprocess
+import threading
+import shlex
 
 from inputimeout import inputimeout, TimeoutOccurred
 
@@ -35,6 +37,8 @@ from . import __about__ as about
 from dar_backup.config_settings import ConfigSettings
 from dar_backup.util import setup_logging
 from dar_backup.util import CommandResult
+from dar_backup.util import get_config_file
+from dar_backup.util import send_discord_message
 from dar_backup.util import get_logger
 from dar_backup.util import get_binary_info
 from dar_backup.util import show_version
@@ -47,6 +51,7 @@ from dar_backup.command_runner import CommandResult
 from dar_backup.util import backup_definition_completer, list_archive_completer, archive_content_completer, add_specific_archive_completer
 
 from datetime import datetime
+from sys import stderr
 from time import time
 from typing import Dict, List, NamedTuple, Tuple
 
@@ -60,6 +65,25 @@ logger = None
 runner = None
 
 
+def _open_command_log(command: List[str]):
+    command_logger = get_logger(command_output_logger=True)
+    log_path = None
+    for handler in getattr(command_logger, "handlers", []):
+        if hasattr(handler, "baseFilename"):
+            log_path = handler.baseFilename
+            break
+    if not log_path:
+        return None, None
+    log_file = open(log_path, "ab")
+    header = (
+        f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - COMMAND: "
+        f"{' '.join(map(shlex.quote, command))}\n"
+    ).encode("utf-8", errors="replace")
+    log_file.write(header)
+    log_file.flush()
+    return log_file, threading.Lock()
+
+
 def get_db_dir(config_settings: ConfigSettings) -> str:
     """
     Return the correct directory for storing catalog databases.
@@ -129,24 +153,106 @@ def list_catalogs(backup_def: str, config_settings: ConfigSettings, suppress_out
         return CommandResult(1, '', error_msg)
 
     command = ['dar_manager', '--base', database_path, '--list']
-    process = runner.run(command)
-    stdout, stderr = process.stdout, process.stderr
+    if runner is not None and not hasattr(runner, "default_capture_limit_bytes"):
+        process = runner.run(command, capture_output_limit_bytes=-1)
+        stdout, stderr = process.stdout, process.stderr
 
-    if process.returncode != 0:
-        logger.error(f'Error listing catalogs for: "{database_path}"')
-        logger.error(f"stderr: {stderr}")
-        logger.error(f"stdout: {stdout}")
-        return process
-
-    # Extract only archive basenames from stdout
-    archive_names = []
-    for line in stdout.splitlines():
-        line = line.strip()
-        if not line or "archive #" in line or "dar path" in line or "compression" in line:
-            continue
-        parts = line.split("\t")
-        if len(parts) >= 3:
-            archive_names.append(parts[2].strip())
+        if process.returncode != 0:
+            logger.error(f'Error listing catalogs for: "{database_path}"')
+            logger.error(f"stderr: {stderr}")
+            logger.error(f"stdout: {stdout}")
+            return process
+
+        # Extract only archive basenames from stdout
+        archive_names = []
+        archive_lines = []
+        for line in stdout.splitlines():
+            line = line.strip()
+            if not line or "archive #" in line or "dar path" in line or "compression" in line:
+                continue
+            parts = line.split("\t")
+            if len(parts) >= 3:
+                archive_names.append(parts[2].strip())
+                archive_lines.append(line)
+    else:
+        stderr_lines: List[str] = []
+        stderr_bytes = 0
+        cap = getattr(config_settings, "command_capture_max_bytes", None)
+        if not isinstance(cap, int):
+            cap = None
+        log_file, log_lock = _open_command_log(command)
+
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=False,
+            bufsize=0
+        )
+
+        def read_stderr():
+            nonlocal stderr_bytes
+            if process.stderr is None:
+                return
+            while True:
+                chunk = process.stderr.read(1024)
+                if not chunk:
+                    break
+                if log_file:
+                    with log_lock:
+                        log_file.write(chunk)
+                        log_file.flush()
+                if cap is None:
+                    stderr_lines.append(chunk)
+                elif cap > 0 and stderr_bytes < cap:
+                    remaining = cap - stderr_bytes
+                    if len(chunk) <= remaining:
+                        stderr_lines.append(chunk)
+                        stderr_bytes += len(chunk)
+                    else:
+                        stderr_lines.append(chunk[:remaining])
+                        stderr_bytes = cap
+
+        stderr_thread = threading.Thread(target=read_stderr)
+        stderr_thread.start()
+
+        archive_names = []
+        archive_lines = []
+        if process.stdout is not None:
+            buffer = b""
+            while True:
+                chunk = process.stdout.read(1024)
+                if not chunk:
+                    break
+                if log_file:
+                    with log_lock:
+                        log_file.write(chunk)
+                buffer += chunk
+                while b"\n" in buffer:
+                    line, buffer = buffer.split(b"\n", 1)
+                    stripped = line.strip()
+                    if not stripped:
+                        continue
+                    decoded = stripped.decode("utf-8", errors="replace")
+                    if "archive #" in decoded or "dar path" in decoded or "compression" in decoded:
+                        continue
+                    parts = decoded.split("\t")
+                    if len(parts) >= 3:
+                        archive_names.append(parts[2].strip())
+                        archive_lines.append(decoded)
+            process.stdout.close()
+
+        process.wait()
+        stderr_thread.join()
+        if log_file:
+            log_file.close()
+
+        if process.returncode != 0:
+            logger.error(f'Error listing catalogs for: "{database_path}"')
+            stderr_text = "".join(stderr_lines)
+            if stderr_text:
+                logger.error(f"stderr: {stderr_text}")
+            return CommandResult(process.returncode, "", stderr_text)
 
     # Sort by prefix and date
     def extract_date(arch_name):
@@ -165,7 +271,7 @@ def list_catalogs(backup_def: str, config_settings: ConfigSettings, suppress_out
         for name in archive_names:
             print(name)
 
-    return process
+    return CommandResult(0, "\n".join(archive_lines), "")
 
 
 def cat_no_for_name(archive: str, config_settings: ConfigSettings) -> int:
@@ -182,9 +288,7 @@ def cat_no_for_name(archive: str, config_settings: ConfigSettings) -> int:
     if process.returncode != 0:
         logger.error(f"Error listing catalogs for backup def: '{backup_def}'")
         return -1
-    line_no = 1
     for line in process.stdout.splitlines():
-        line_no += 1
         search = re.search(rf".*?(\d+)\s+.*?({archive}).*", line)
         if search:
             logger.info(f"Found archive: '{archive}', catalog #: '{search.group(1)}'")
@@ -213,26 +317,104 @@ def list_archive_contents(archive: str, config_settings: ConfigSettings) -> int:
 
 
     command = ['dar_manager', '--base', database_path, '-u', f"{cat_no}"]
-    process = runner.run(command, timeout = 10)
+    if runner is not None and not hasattr(runner, "default_capture_limit_bytes"):
+        process = runner.run(command, timeout=10)
+        stdout = process.stdout or ""
+        stderr = process.stderr or ""
+        if process.returncode != 0:
+            logger.error(f'Error listing catalogs for: "{database_path}"')
+            logger.error(f"stderr: {stderr}")
+            logger.error(f"stdout: {stdout}")
+            return process.returncode
 
+        combined_lines = (stdout + "\n" + stderr).splitlines()
+        file_lines = [line for line in combined_lines if line.strip().startswith("[ Saved ]")]
 
-    stdout = process.stdout or ""
-    stderr = process.stderr or ""
+        if file_lines:
+            for line in file_lines:
+                print(line)
+        else:
+            print(f"[info] Archive '{archive}' is empty.")
+
+        return process.returncode
+
+    stderr_lines: List[str] = []
+    stderr_bytes = 0
+    cap = getattr(config_settings, "command_capture_max_bytes", None)
+    log_file, log_lock = _open_command_log(command)
+
+    process = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=False,
+        bufsize=0
+    )
+
+    def read_stderr():
+        nonlocal stderr_bytes
+        if process.stderr is None:
+            return
+        while True:
+            chunk = process.stderr.read(1024)
+            if not chunk:
+                break
+            if log_file:
+                with log_lock:
+                    log_file.write(chunk)
+                    log_file.flush()
+            if cap is None:
+                stderr_lines.append(chunk)
+            elif cap > 0 and stderr_bytes < cap:
+                remaining = cap - stderr_bytes
+                if len(chunk) <= remaining:
+                    stderr_lines.append(chunk)
+                    stderr_bytes += len(chunk)
+                else:
+                    stderr_lines.append(chunk[:remaining])
+                    stderr_bytes = cap
+
+    stderr_thread = threading.Thread(target=read_stderr)
+    stderr_thread.start()
+
+    found = False
+    if process.stdout is not None:
+        buffer = b""
+        while True:
+            chunk = process.stdout.read(1024)
+            if not chunk:
+                break
+            if log_file:
+                with log_lock:
+                    log_file.write(chunk)
+            buffer += chunk
+            while b"\n" in buffer:
+                line, buffer = buffer.split(b"\n", 1)
+                if line.strip().startswith(b"[ Saved ]"):
+                    print(line.decode("utf-8", errors="replace"))
+                    found = True
+        process.stdout.close()
 
+    try:
+        process.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        stderr_thread.join()
+        logger.error(f"Timeout listing contents of archive: '{archive}'")
+        return 1
+
+    stderr_thread.join()
+    if log_file:
+        log_file.close()
 
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
-        logger.error(f"stderr: {stderr}")
-        logger.error(f"stdout: {stdout}")
-
-
-    combined_lines = (stdout + "\n" + stderr).splitlines()
-    file_lines = [line for line in combined_lines if line.strip().startswith("[ Saved ]")]
+        stderr_text = "".join(stderr_lines)
+        if stderr_text:
+            logger.error(f"stderr: {stderr_text}")
+        return process.returncode
 
-    if file_lines:
-        for line in file_lines:
-            print(line)
-    else:
+    if not found:
         print(f"[info] Archive '{archive}' is empty.")
 
     return process.returncode
@@ -250,7 +432,7 @@ def list_catalog_contents(catalog_number: int, backup_def: str, config_settings:
         logger.error(f'Catalog database not found: "{database_path}"')
         return 1
     command = ['dar_manager', '--base', database_path, '-u', f"{catalog_number}"]
-    process = runner.run(command)
+    process = runner.run(command, capture_output_limit_bytes=-1)
     stdout, stderr = process.stdout, process.stderr 
     if process.returncode != 0:
         logger.error(f'Error listing catalogs for: "{database_path}"')
@@ -271,7 +453,7 @@ def find_file(file, backup_def, config_settings):
         logger.error(f'Database not found: "{database_path}"')
         return 1
     command = ['dar_manager', '--base', database_path, '-f', f"{file}"]
-    process = runner.run(command)
+    process = runner.run(command, capture_output_limit_bytes=-1)
     stdout, stderr = process.stdout, process.stderr 
     if process.returncode != 0:
         logger.error(f'Error finding file: {file} in: "{database_path}"')
@@ -492,7 +674,7 @@ def remove_specific_archive(archive: str, config_settings: ConfigSettings) -> in
 
 def build_arg_parser():
     parser = argparse.ArgumentParser(description="Creates/maintains `dar` database catalogs")
-    parser.add_argument('-c', '--config-file', type=str, help="Path to 'dar-backup.conf'", default='~/.config/dar-backup/dar-backup.conf')
+    parser.add_argument('-c', '--config-file', type=str, help="Path to 'dar-backup.conf'", default=None)
     parser.add_argument('--create-db', action='store_true', help='Create missing databases for all backup definitions')
     parser.add_argument('--alternate-archive-dir', type=str, help='Use this directory instead of BACKUP_DIR in config file')
     parser.add_argument('--add-dir', type=str, help='Add all archive catalogs in this directory to databases')
@@ -536,8 +718,20 @@ def main():
         show_version()
         sys.exit(0)
 
-    args.config_file = os.path.expanduser(os.path.expandvars(args.config_file))
-    config_settings = ConfigSettings(args.config_file)
+    config_settings_path = get_config_file(args)
+    if not (os.path.isfile(config_settings_path) and os.access(config_settings_path, os.R_OK)):
+        print(f"Config file {config_settings_path} must exist and be readable.", file=stderr)
+        raise SystemExit(127)
+    args.config_file = config_settings_path
+
+    try:
+        config_settings = ConfigSettings(args.config_file)
+    except Exception as exc:
+        msg = f"Config error: {exc}"
+        print(msg, file=stderr)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - manager: FAILURE - {msg}")
+        sys.exit(127)
 
     if not os.path.dirname(config_settings.logfile_location):
         print(f"Directory for log file '{config_settings.logfile_location}' does not exist, exiting")
@@ -545,16 +739,28 @@ def main():
         return
 
     command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
-    logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout, logfile_max_bytes=config_settings.logfile_max_bytes, logfile_backup_count=config_settings.logfile_backup_count)
+    logger = setup_logging(
+        config_settings.logfile_location,
+        command_output_log,
+        args.log_level,
+        args.log_stdout,
+        logfile_max_bytes=config_settings.logfile_max_bytes,
+        logfile_backup_count=config_settings.logfile_backup_count,
+        trace_log_max_bytes=getattr(config_settings, "trace_log_max_bytes", 10485760),
+        trace_log_backup_count=getattr(config_settings, "trace_log_backup_count", 1)
+    )
     command_logger = get_logger(command_output_logger=True)
-    runner = CommandRunner(logger=logger, command_logger=command_logger)
+    runner = CommandRunner(
+        logger=logger,
+        command_logger=command_logger,
+        default_capture_limit_bytes=getattr(config_settings, "command_capture_max_bytes", None)
+    )
 
     start_msgs: List[Tuple[str, str]] = []
 
     start_time = int(time())
     
     start_msgs.append((f"{show_scriptname()}:", about.__version__))
-    logger.info(f"START TIME: {start_time}")
     logger.debug(f"Command line: {get_invocation_command_line()}")
     logger.debug(f"`args`:\n{args}")
     logger.debug(f"`config_settings`:\n{config_settings}")
@@ -620,63 +826,70 @@ def main():
         return
 
     # --- Modify settings ---
-    if args.alternate_archive_dir:
-        if not os.path.exists(args.alternate_archive_dir):
-            logger.error(f"Alternate archive dir '{args.alternate_archive_dir}' does not exist, exiting")
-            sys.exit(1)
+    try:
+        if args.alternate_archive_dir:
+            if not os.path.exists(args.alternate_archive_dir):
+                logger.error(f"Alternate archive dir '{args.alternate_archive_dir}' does not exist, exiting")
+                sys.exit(1)
+                return
+            config_settings.backup_dir = args.alternate_archive_dir
+
+        # --- Functional logic ---
+        if args.create_db:
+            if args.backup_def:
+                sys.exit(create_db(args.backup_def, config_settings, logger, runner))
+                return
+            else:
+                for root, dirs, files in os.walk(config_settings.backup_d_dir):
+                    for file in files:
+                        current_backupdef = os.path.basename(file)
+                        logger.debug(f"Create catalog db for backup definition: '{current_backupdef}'")
+                        result = create_db(current_backupdef, config_settings, logger, runner)
+                        if result != 0:
+                            sys.exit(result)
+                            return
+
+        if args.add_specific_archive:
+            sys.exit(add_specific_archive(args.add_specific_archive, config_settings))
             return
-        config_settings.backup_dir = args.alternate_archive_dir
 
-    # --- Functional logic ---
-    if args.create_db:
-        if args.backup_def:
-            sys.exit(create_db(args.backup_def, config_settings, logger, runner))
+        if args.add_dir:
+            sys.exit(add_directory(args, config_settings))
             return
-        else:
-            for root, dirs, files in os.walk(config_settings.backup_d_dir):
-                for file in files:
-                    current_backupdef = os.path.basename(file)
-                    logger.debug(f"Create catalog db for backup definition: '{current_backupdef}'")
-                    result = create_db(current_backupdef, config_settings, logger, runner)
-                    if result != 0:
-                        sys.exit(result)
-                        return
-
-    if args.add_specific_archive:
-        sys.exit(add_specific_archive(args.add_specific_archive, config_settings))
-        return
 
-    if args.add_dir:
-        sys.exit(add_directory(args, config_settings))
-        return
-
-    if args.remove_specific_archive:
-        return remove_specific_archive(args.remove_specific_archive, config_settings)
-
-    if args.list_catalogs:
-        if args.backup_def:
-            process = list_catalogs(args.backup_def, config_settings)
-            result = process.returncode
-        else:
-            result = 0
-            for root, dirs, files in os.walk(config_settings.backup_d_dir):
-                for file in files:
-                    current_backupdef = os.path.basename(file)
-                    if list_catalogs(current_backupdef, config_settings).returncode != 0:
-                        result = 1
-        sys.exit(result)
-        return
+        if args.remove_specific_archive:
+            return remove_specific_archive(args.remove_specific_archive, config_settings)
+
+        if args.list_catalogs:
+            if args.backup_def:
+                process = list_catalogs(args.backup_def, config_settings)
+                result = process.returncode
+            else:
+                result = 0
+                for root, dirs, files in os.walk(config_settings.backup_d_dir):
+                    for file in files:
+                        current_backupdef = os.path.basename(file)
+                        if list_catalogs(current_backupdef, config_settings).returncode != 0:
+                            result = 1
+            sys.exit(result)
+            return
 
-    if args.list_archive_contents:
-        result = list_archive_contents(args.list_archive_contents, config_settings)
-        sys.exit(result)
-        return
+        if args.list_archive_contents:
+            result = list_archive_contents(args.list_archive_contents, config_settings)
+            sys.exit(result)
+            return
 
 
-    if args.find_file:
-        result = find_file(args.find_file, args.backup_def, config_settings)
-        sys.exit(result)
-        return
+        if args.find_file:
+            result = find_file(args.find_file, args.backup_def, config_settings)
+            sys.exit(result)
+            return
+    except Exception as e:
+        msg = f"Unexpected error during manager operation: {e}"
+        logger.error(msg, exc_info=True)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - manager: FAILURE - {msg}", config_settings=config_settings)
+        sys.exit(1)
 
 
 if __name__ == "__main__":