dar-backup 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

dar_backup/__about__.py

@@ -1,8 +1,7 @@
-__version__ = "1.0.1"
+__version__ = "1.1.0"
 
 __author__ = "Per Jensen"
 
 __license__ = '''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
 THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 See section 15 and section 16 in the supplied "LICENSE" file.'''
-

dar_backup/clean_log.py

@@ -23,13 +23,54 @@ import re
 import os
 import sys 
 
+from datetime import datetime
+
 from dar_backup import __about__ as about
 from dar_backup.config_settings import ConfigSettings
+from dar_backup.util import send_discord_message, get_logger
 
 LICENSE = '''Licensed under GNU GENERAL PUBLIC LICENSE v3, see the supplied file "LICENSE" for details.
 THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW, not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 See section 15 and section 16 in the supplied "LICENSE" file.'''
 
+TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}\b")
+CLEAN_MESSAGE_PREFIXES = (
+    "Inspecting directory",
+    "Finished Inspecting",
+    "<File",
+    "</File",
+    "<Attributes",
+    "</Attributes",
+    "<Directory",
+    "</Directory",
+    "<Catalog",
+    "</Catalog",
+    "<Symlink",
+    "</Symlink",
+)
+
+def _split_level_and_message(line):
+    line = line.rstrip("\n")
+    if " - " not in line:
+        return None, None
+
+    parts = line.split(" - ")
+    if len(parts) >= 3 and TIMESTAMP_RE.match(parts[0].strip()):
+        level = parts[1]
+        message = " - ".join(parts[2:])
+    else:
+        level = parts[0]
+        message = " - ".join(parts[1:])
+
+    return level.strip(), message
+
+def _should_remove_line(line):
+    level, message = _split_level_and_message(line)
+    if level != "INFO" or message is None:
+        return False
+    message = message.lstrip()
+    return any(message.startswith(prefix) for prefix in CLEAN_MESSAGE_PREFIXES)
+
 def clean_log_file(log_file_path, dry_run=False):
     """Removes specific log lines from the given file using a memory-efficient streaming approach."""
 
@@ -42,7 +83,7 @@ def clean_log_file(log_file_path, dry_run=False):
         print(f"No read permission for '{log_file_path}'")
         sys.exit(1)
 
-    if not os.access(log_file_path, os.W_OK):
+    if not dry_run and not os.access(log_file_path, os.W_OK):
         print(f"Error: No write permission for '{log_file_path}'")
         sys.exit(1)
 
@@ -51,47 +92,20 @@ def clean_log_file(log_file_path, dry_run=False):
         print(f"Performing a dry run on: {log_file_path}")
 
     temp_file_path = log_file_path + ".tmp"
-    
-    patterns = [
-        r"INFO\s*-\s*Inspecting\s*directory",
-        r"INFO\s*-\s*Finished\s*Inspecting",
-        r"INFO\s*-\s*<File",
-        r"INFO\s*-\s*</File",
-        r"INFO\s*-\s*<Attributes",
-        r"INFO\s*-\s*</Attributes",
-        r"INFO\s*-\s*</Directory",
-        r"INFO\s*-\s*<Directory",
-        r"INFO\s*-\s*<Catalog",
-        r"INFO\s*-\s*</Catalog",
-        r"INFO\s*-\s*<Symlink",
-        r"INFO\s*-\s*</Symlink",
-    ]
 
     try:
-        with open(log_file_path, "r", errors="ignore") as infile, open(temp_file_path, "w") as outfile:
+        if dry_run:
+            with open(log_file_path, "r", errors="ignore") as infile:
+                for line in infile:
+                    if _should_remove_line(line):
+                        print(f"Would remove: {line.strip()}")
+            return
 
+        with open(log_file_path, "r", errors="ignore") as infile, open(temp_file_path, "w") as outfile:
             for line in infile:
-                original_line = line  # Store the original line before modifying it
-                matched = False  # Track if a pattern is matched
-
-                for pattern in patterns:
-                    if re.search(pattern, line):  # Check if the pattern matches
-                        if dry_run:
-                            print(f"Would remove: {original_line.strip()}")  # Print full line for dry-run
-                        matched = True  # Mark that a pattern matched
-                        break  # No need to check other patterns if one matches
-
-                if not dry_run and not matched:  # In normal mode, only write non-empty lines
+                if not _should_remove_line(line):
                     outfile.write(line.rstrip() + "\n")
 
-                if dry_run and matched:
-                    continue  # In dry-run mode, skip writing (since we’re just showing)
-
-        
-        # Ensure the temp file exists before renaming
-        if not os.path.exists(temp_file_path):
-            open(temp_file_path, "w").close()  # Create an empty file if nothing was written
-
         os.replace(temp_file_path, log_file_path)
         print(f"Successfully cleaned log file: {log_file_path}")
 
@@ -131,43 +145,68 @@ def main():
 
     args = parser.parse_args()
 
-    config_settings = ConfigSettings(os.path.expanduser(os.path.expandvars(args.config_file)))
+    try:
+        config_settings = ConfigSettings(os.path.expanduser(os.path.expandvars(args.config_file)))
+    except Exception as exc:
+        msg = f"Config error: {exc}"
+        print(msg, file=sys.stderr)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - clean-log: FAILURE - {msg}")
+        sys.exit(127)
 
-    if not args.file:
-        args.file = [config_settings.logfile_location]
+    try:
+        files_to_clean = args.file if args.file else [config_settings.logfile_location]
+        logfile_dir = os.path.dirname(os.path.realpath(config_settings.logfile_location))
+        validated_files = []
 
-    for file_path in args.file:
+        for file_path in files_to_clean:
+            if not isinstance(file_path, (str, bytes, os.PathLike)):
+                print(f"Error: Invalid file path type: {file_path}")
+                sys.exit(1)
 
-        if ".." in os.path.normpath(file_path).split(os.sep):
-            print(f"Error: Path traversal is not allowed: '{file_path}'")
-            sys.exit(1)
+            file_path = os.fspath(file_path)
+            if isinstance(file_path, bytes):
+                file_path = os.fsdecode(file_path)
 
-        logfile_dir = os.path.dirname(os.path.realpath(config_settings.logfile_location))
-        resolved_path = os.path.realpath(file_path)
+            if file_path.strip() == "":
+                print(f"Error: Invalid empty filename '{file_path}'.")
+                sys.exit(1)
 
-        if not resolved_path.startswith(logfile_dir + os.sep):
-            print(f"Error: File is outside allowed directory: '{file_path}'")
-            sys.exit(1)
+            if ".." in os.path.normpath(file_path).split(os.sep):
+                print(f"Error: Path traversal is not allowed: '{file_path}'")
+                sys.exit(1)
 
-        # Validate the file path type and existence        
-        if not isinstance(file_path, (str, bytes, os.PathLike)):
-            print(f"Error: Invalid file path type: {file_path}")
-            sys.exit(1)
+            resolved_path = os.path.realpath(file_path)
 
-        if not os.path.exists(file_path):
-            print(f"Error: Log file '{file_path}' does not exist.")
-            sys.exit(1)
+            if not resolved_path.startswith(logfile_dir + os.sep):
+                print(f"Error: File is outside allowed directory: '{file_path}'")
+                sys.exit(1)
 
-        if file_path.strip() == "":
-            print(f"Error: Invalid empty filename '{file_path}'.")
-            sys.exit(1)
+            if not os.path.exists(file_path):
+                print(f"Error: Log file '{file_path}' does not exist.")
+                sys.exit(1)
 
-
-    # Run the log file cleaning function
-    for log_file in args.file:
-        clean_log_file(log_file, dry_run=args.dry_run)
-    print(f"Log file '{args.file}' has been cleaned successfully.")
+            validated_files.append(file_path)
 
 
+        # Run the log file cleaning function
+        for log_file in validated_files:
+            clean_log_file(log_file, dry_run=args.dry_run)
+        file_list = ", ".join(validated_files)
+        if args.dry_run:
+            print(f"Dry run complete for: {file_list}")
+        else:
+            print(f"Log file '{file_list}' has been cleaned successfully.")
+    except Exception as e:
+        msg = f"Unexpected error during clean-log: {e}"
+        logger = get_logger()
+        if logger:
+            logger.error(msg, exc_info=True)
+        else:
+            print(msg, file=sys.stderr)
+        
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - clean-log: FAILURE - {msg}", config_settings=config_settings)
+        sys.exit(1)
 if __name__ == "__main__":
     main()

dar_backup/cleanup.py

@@ -16,10 +16,8 @@ This script removes old DIFF and INCR archives + accompanying .par2 files accord
 
 import argcomplete
 import argparse
-import logging
 import os
 import re
-import subprocess
 import sys
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
 
@@ -29,7 +27,7 @@ from inputimeout import inputimeout, TimeoutOccurred
 from pathlib import Path
 from sys import stderr
 from time import time
-from typing import Dict, List, NamedTuple, Tuple
+from typing import List, NamedTuple, Tuple
 import glob
 
 
@@ -48,6 +46,7 @@ from dar_backup.util import is_archive_name_allowed
 from dar_backup.util import is_safe_filename
 from dar_backup.util import safe_remove_file
 from dar_backup.util import show_scriptname
+from dar_backup.util import send_discord_message
 
 from dar_backup.command_runner import CommandRunner   
 from dar_backup.command_runner import CommandResult
@@ -67,7 +66,6 @@ def _delete_par2_files(
     else:
         par2_config = {
             "par2_dir": None,
-            "par2_mode": None,
         }
 
     par2_dir = par2_config.get("par2_dir") or backup_dir
@@ -76,49 +74,33 @@ def _delete_par2_files(
         logger.warning(f"PAR2 directory not found, skipping cleanup: {par2_dir}")
         return
 
-    par2_mode = (par2_config.get("par2_mode") or "per-slice").lower()
-
-    if par2_mode == "per-archive":
-        par2_glob = os.path.join(par2_dir, f"{archive_name}*.par2")
-        targets = glob.glob(par2_glob)
-        manifest_path = os.path.join(par2_dir, f"{archive_name}.par2.manifest.ini")
-        if os.path.exists(manifest_path):
-            targets.append(manifest_path)
-        if not targets:
-            logger.info("No par2 files matched the per-archive cleanup pattern.")
-            return
-        for file_path in sorted(set(targets)):
-            try:
-                if dry_run:
-                    logger.info(f"Dry run: would delete PAR2 file: {file_path}")
-                else:
-                    safe_remove_file(file_path, base_dir=Path(par2_dir))
-                    logger.info(f"Deleted PAR2 file: {file_path}")
-            except Exception as e:
-                logger.error(f"Error deleting PAR2 file {file_path}: {e}")
-        return
-
-    if par2_mode != "per-slice":
-        logger.error(f"Unsupported PAR2_MODE during cleanup: {par2_mode}")
-        return
+    par2_glob = os.path.join(par2_dir, f"{archive_name}*.par2")
+    targets = set(glob.glob(par2_glob))
+    manifest_path = os.path.join(par2_dir, f"{archive_name}.par2.manifest.ini")
+    if os.path.exists(manifest_path):
+        targets.add(manifest_path)
 
     par2_regex = re.compile(rf"^{re.escape(archive_name)}\.[0-9]+\.dar.*\.par2$")
-    files_deleted = False
-    for filename in sorted(os.listdir(par2_dir)):
+    for entry in os.scandir(par2_dir):
+        if not entry.is_file():
+            continue
+        filename = entry.name
         if par2_regex.match(filename):
-            file_path = os.path.join(par2_dir, filename)
-            try:
-                if dry_run:
-                    logger.info(f"Dry run: would delete PAR2 file: {file_path}")
-                else:
-                    safe_remove_file(file_path, base_dir=Path(par2_dir))
-                    logger.info(f"Deleted PAR2 file: {file_path}")
-                    files_deleted = True
-            except Exception as e:
-                logger.error(f"Error deleting PAR2 file {file_path}: {e}")
+            targets.add(entry.path)
 
-    if not files_deleted:
-        logger.info("No .par2 matched the regex for deletion.")
+    if not targets:
+        logger.info("No par2 files matched the cleanup patterns.")
+        return
+
+    for file_path in sorted(targets):
+        try:
+            if dry_run:
+                logger.info(f"Dry run: would delete PAR2 file: {file_path}")
+            else:
+                safe_remove_file(file_path, base_dir=Path(par2_dir))
+                logger.info(f"Deleted PAR2 file: {file_path}")
+        except Exception as e:
+            logger.error(f"Error deleting PAR2 file {file_path}: {e}")
 
 
 def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=None, config_settings: ConfigSettings = None):
@@ -138,7 +120,10 @@ def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=Non
     archives_deleted = {}
 
     dry_run = getattr(args, "dry_run", False) is True
-    for filename in sorted(os.listdir(backup_dir)):
+    for entry in os.scandir(backup_dir):
+        if not entry.is_file():
+            continue
+        filename = entry.name
         if not filename.endswith('.dar'):
             continue
         if backup_definition and not filename.startswith(backup_definition):
@@ -152,7 +137,7 @@ def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=Non
                 raise
 
             if file_date < cutoff_date:
-                file_path = os.path.join(backup_dir, filename)
+                file_path = entry.path
                 try:
                     if dry_run:
                         logger.info(f"Dry run: would delete {backup_type} backup: {file_path}")
@@ -160,7 +145,7 @@ def delete_old_backups(backup_dir, age, backup_type, args, backup_definition=Non
                         safe_remove_file(file_path, base_dir=Path(backup_dir))
                         logger.info(f"Deleted {backup_type} backup: {file_path}")
                     archive_name = filename.split('.')[0]
-                    if not archive_name in archives_deleted:
+                    if archive_name not in archives_deleted:
                         logger.debug(f"Archive name: '{archive_name}' added to catalog deletion list")
                     archives_deleted[archive_name] = True
                 except Exception as e:
@@ -190,9 +175,12 @@ def delete_archive(backup_dir, archive_name, args, config_settings: ConfigSettin
     # Delete the specified .dar files according to the naming convention
     files_deleted = False
     dry_run = getattr(args, "dry_run", False) is True
-    for filename in sorted(os.listdir(backup_dir)):
+    for entry in os.scandir(backup_dir):
+        if not entry.is_file():
+            continue
+        filename = entry.name
         if archive_regex.match(filename):
-            file_path = os.path.join(backup_dir, filename)
+            file_path = entry.path
             try:
                 if dry_run:
                     logger.info(f"Dry run: would delete archive slice: {file_path}")
@@ -219,7 +207,7 @@ def delete_catalog(catalog_name: str, args: NamedTuple) -> bool:
     """
     Call `manager.py` to delete the specified catalog in it's database
     """
-    command = [f"manager", "--remove-specific-archive", catalog_name, "--config-file", args.config_file, '--log-level', 'debug', '--log-stdout']
+    command = ["manager", "--remove-specific-archive", catalog_name, "--config-file", args.config_file, '--log-level', 'debug', '--log-stdout']
     logger.info(f"Deleting catalog '{catalog_name}' using config file: '{args.config_file}'")
     try:
         result:CommandResult = runner.run(command)
@@ -308,15 +296,35 @@ def main():
             raise SystemExit(127)
     args.config_file = config_settings_path
 
-    config_settings = ConfigSettings(args.config_file)
+    try:
+        config_settings = ConfigSettings(args.config_file)
+    except Exception as exc:
+        msg = f"Config error: {exc}"
+        print(msg, file=stderr)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - cleanup: FAILURE - {msg}")
+        sys.exit(127)
 
     start_time=int(time())
 
 #    command_output_log = os.path.join(config_settings.logfile_location.removesuffix("dar-backup.log"), "dar-backup-commands.log")
     command_output_log = config_settings.logfile_location.replace("dar-backup.log", "dar-backup-commands.log")
-    logger = setup_logging(config_settings.logfile_location, command_output_log, args.log_level, args.log_stdout, logfile_max_bytes=config_settings.logfile_max_bytes, logfile_backup_count=config_settings.logfile_backup_count)
+    logger = setup_logging(
+        config_settings.logfile_location,
+        command_output_log,
+        args.log_level,
+        args.log_stdout,
+        logfile_max_bytes=config_settings.logfile_max_bytes,
+        logfile_backup_count=config_settings.logfile_backup_count,
+        trace_log_max_bytes=getattr(config_settings, "trace_log_max_bytes", 10485760),
+        trace_log_backup_count=getattr(config_settings, "trace_log_backup_count", 1)
+    )
     command_logger = get_logger(command_output_logger = True)
-    runner = CommandRunner(logger=logger, command_logger=command_logger)
+    runner = CommandRunner(
+        logger=logger,
+        command_logger=command_logger,
+        default_capture_limit_bytes=getattr(config_settings, "command_capture_max_bytes", None)
+    )
 
     start_msgs: List[Tuple[str, str]] = []
 
@@ -341,65 +349,89 @@ def main():
     print_aligned_settings(start_msgs, highlight_keywords=dangerous_keywords, quiet=not args.verbose)
 
     # run PREREQ scripts
-    requirements('PREREQ', config_settings)
-
-    if args.alternate_archive_dir:
-        if not os.path.exists(args.alternate_archive_dir):
-            logger.error(f"Alternate archive directory does not exist: {args.alternate_archive_dir}, exiting")
-            sys.exit(1) 
-        if  not os.path.isdir(args.alternate_archive_dir):
-            logger.error(f"Alternate archive directory is not a directory, exiting")
-            sys.exit(1) 
-        config_settings.backup_dir = args.alternate_archive_dir
-
-    if args.cleanup_specific_archives is None and args.test_mode:
-        logger.info("No --cleanup-specific-archives provided; skipping specific archive deletion in test mode.")
-
-    if args.cleanup_specific_archives or args.cleanup_specific_archives_list:
-        combined = []
-        if args.cleanup_specific_archives:
-            combined.extend(args.cleanup_specific_archives.split(','))
-        combined.extend(args.cleanup_specific_archives_list or [])
-        archive_names = [name.strip() for name in combined if name.strip()]
-        logger.info(f"Cleaning up specific archives: {', '.join(archive_names)}")
-        for archive_name in archive_names:
-            if "_FULL_" in archive_name:
-                if not confirm_full_archive_deletion(archive_name, args.test_mode):
+    try:
+        requirements('PREREQ', config_settings)
+    except Exception as exc:
+        msg = f"PREREQ failed: {exc}"
+        logger.error(msg)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - cleanup: FAILURE - {msg}", config_settings=config_settings)
+        sys.exit(1)
+
+    try:
+        if args.alternate_archive_dir:
+            if not os.path.exists(args.alternate_archive_dir):
+                logger.error(f"Alternate archive directory does not exist: {args.alternate_archive_dir}, exiting")
+                sys.exit(1) 
+            if  not os.path.isdir(args.alternate_archive_dir):
+                logger.error("Alternate archive directory is not a directory, exiting")
+                sys.exit(1) 
+            config_settings.backup_dir = args.alternate_archive_dir
+
+        if args.cleanup_specific_archives is None and args.test_mode:
+            logger.info("No --cleanup-specific-archives provided; skipping specific archive deletion in test mode.")
+
+        if args.cleanup_specific_archives or args.cleanup_specific_archives_list:
+            combined = []
+            if args.cleanup_specific_archives:
+                combined.extend(args.cleanup_specific_archives.split(','))
+            combined.extend(args.cleanup_specific_archives_list or [])
+            archive_names = [name.strip() for name in combined if name.strip()]
+            logger.info(f"Cleaning up specific archives: {', '.join(archive_names)}")
+            for archive_name in archive_names:
+                if not is_archive_name_allowed(archive_name):
+                    logger.error(f"Refusing unsafe archive name: {archive_name}")
                     continue
-            archive_path = os.path.join(config_settings.backup_dir, archive_name.strip())
-            logger.info(f"Deleting archive: {archive_path}")
-            delete_archive(config_settings.backup_dir, archive_name.strip(), args, config_settings)
-    elif args.list:
-        list_backups(config_settings.backup_dir, args.backup_definition)
-    else:
-        backup_definitions = []
-        if args.backup_definition:
-            backup_definitions.append(args.backup_definition)
+                if "_FULL_" in archive_name:
+                    if not confirm_full_archive_deletion(archive_name, args.test_mode):
+                        continue
+                archive_path = os.path.join(config_settings.backup_dir, archive_name.strip())
+                logger.info(f"Deleting archive: {archive_path}")
+                delete_archive(config_settings.backup_dir, archive_name.strip(), args, config_settings)
+        elif args.list:
+            list_backups(config_settings.backup_dir, args.backup_definition)
         else:
-            for root, _, files in os.walk(config_settings.backup_d_dir):
-                for file in files:
-                    backup_definitions.append(file.split('.')[0])
-
-        for definition in backup_definitions:
-            delete_old_backups(
-                config_settings.backup_dir,
-                config_settings.diff_age,
-                'DIFF',
-                args,
-                backup_definition=definition,
-                config_settings=config_settings
-            )
-            delete_old_backups(
-                config_settings.backup_dir,
-                config_settings.incr_age,
-                'INCR',
-                args,
-                backup_definition=definition,
-                config_settings=config_settings
-            )
+            backup_definitions = []
+            if args.backup_definition:
+                backup_definitions.append(args.backup_definition)
+            else:
+                for root, _, files in os.walk(config_settings.backup_d_dir):
+                    for file in files:
+                        backup_definitions.append(file.split('.')[0])
+
+            for definition in backup_definitions:
+                delete_old_backups(
+                    config_settings.backup_dir,
+                    config_settings.diff_age,
+                    'DIFF',
+                    args,
+                    backup_definition=definition,
+                    config_settings=config_settings
+                )
+                delete_old_backups(
+                    config_settings.backup_dir,
+                    config_settings.incr_age,
+                    'INCR',
+                    args,
+                    backup_definition=definition,
+                    config_settings=config_settings
+                )
+    except Exception as e:
+        msg = f"Unexpected error during cleanup: {e}"
+        logger.error(msg, exc_info=True)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - cleanup: FAILURE - {msg}", config_settings=config_settings)
+        sys.exit(1)
 
     # run POST scripts
-    requirements('POSTREQ', config_settings)
+    try:
+        requirements('POSTREQ', config_settings)
+    except Exception as exc:
+        msg = f"POSTREQ failed: {exc}"
+        logger.error(msg)
+        ts = datetime.now().strftime("%Y-%m-%d_%H:%M")
+        send_discord_message(f"{ts} - cleanup: FAILURE - {msg}", config_settings=config_settings)
+        sys.exit(1)
 
 
     end_time=int(time())