dar-backup 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

dar_backup/util.py

@@ -11,7 +11,6 @@ See section 15 and section 16 in the supplied "LICENSE" file
 """
 import typing
 import locale
-import configparser
 import inspect
 import logging
 import json
@@ -31,9 +30,7 @@ import dar_backup.__about__ as about
 
 
 
-from argcomplete.completers import ChoicesCompleter
 from datetime import datetime, date
-from datetime import date 
 from dar_backup.config_settings import ConfigSettings
 from logging.handlers import RotatingFileHandler
 from pathlib import Path
@@ -47,6 +44,26 @@ from typing import Tuple
 logger=None
 secondary_logger=None   
 
+class CleanFormatter(logging.Formatter):
+    """
+    Formatter that ignores exception tracebacks.
+    """
+    def format(self, record):
+        # Save original exception info
+        orig_exc_info = record.exc_info
+        orig_exc_text = record.exc_text
+        
+        # Temporarily hide it
+        record.exc_info = None
+        record.exc_text = None
+        
+        try:
+            return super().format(record)
+        finally:
+            # Restore it so other handlers (like the trace handler) can use it
+            record.exc_info = orig_exc_info
+            record.exc_text = orig_exc_text
+
 #def setup_logging(log_file: str, command_output_log_file: str, log_level: str = "info", log_to_stdout: bool = False) -> logging.Logger:
 def setup_logging(
     log_file: str,
@@ -55,10 +72,15 @@ def setup_logging(
     log_to_stdout: bool = False,
     logfile_max_bytes: int = 26214400,
     logfile_backup_count: int = 5,
+    trace_log_file: str = None,
+    trace_log_max_bytes: int = 10485760,
+    trace_log_backup_count: int = 1
 ) -> logging.Logger:
 
     """
     Sets up logging for the main program and a separate secondary logfile for command outputs.
+    
+    Also sets up a trace log file that captures all logs at DEBUG level including stack traces.
 
     Args:
         log_file (str): The path to the main log file.
@@ -67,6 +89,9 @@ def setup_logging(
         log_to_stdout (bool): If True, log messages will be printed to the console. Defaults to False.
         logfile_max_bytes: max file size of a log file, defailt = 26214400.
         logfile_backup_count: max numbers of logs files, default = 5.
+        trace_log_file (str): Optional path for the trace log file. Defaults to log_file with ".trace.log" suffix.
+        trace_log_max_bytes: max file size of the trace log file, default = 10485760 (10MB).
+        trace_log_backup_count: max numbers of trace log files, default = 1.
 
     Returns:
         a RotatingFileHandler logger instance.
@@ -85,6 +110,7 @@ def setup_logging(
 
         logging.Logger.trace = trace
 
+        # Main log file handler (clean logs)
         file_handler = RotatingFileHandler(
             log_file,
             maxBytes=logfile_max_bytes,
@@ -92,6 +118,23 @@ def setup_logging(
             encoding="utf-8",
         )
 
+        # Trace log file handler (full details)
+        if not trace_log_file:
+            if log_file == "/dev/null":
+                trace_log_file = "/dev/null"
+            else:
+                base, ext = os.path.splitext(log_file)
+                trace_log_file = f"{base}.trace{ext}"
+            
+        trace_handler = RotatingFileHandler(
+            trace_log_file,
+            maxBytes=trace_log_max_bytes,
+            backupCount=trace_log_backup_count,
+            encoding="utf-8",
+        )
+        # Trace handler gets everything (DEBUG level) and keeps tracebacks
+        trace_handler.setLevel(logging.DEBUG)
+
         command_handler = RotatingFileHandler(
             command_output_log_file,
             maxBytes=logfile_max_bytes,
@@ -99,32 +142,50 @@ def setup_logging(
             encoding="utf-8",
         )
 
-        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        file_handler.setFormatter(formatter)
-        command_handler.setFormatter(formatter)
+        standard_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+        clean_formatter = CleanFormatter('%(asctime)s - %(levelname)s - %(message)s')
+        
+        file_handler.setFormatter(clean_formatter)
+        trace_handler.setFormatter(standard_formatter)
+        command_handler.setFormatter(standard_formatter)
 
 
         # Setup main logger
         logger = logging.getLogger("main_logger")
-        logger.setLevel(logging.DEBUG if log_level == "debug" else TRACE_LEVEL_NUM if log_level == "trace" else logging.INFO)
+        # Ensure logger captures everything so trace_handler can see DEBUG messages even if main log_level is INFO
+        logger.setLevel(logging.DEBUG) 
+        
+        # Configure file_handler level based on user preference
+        file_handler.setLevel(logging.DEBUG if log_level == "debug" else TRACE_LEVEL_NUM if log_level == "trace" else logging.INFO)
+        
         logger.addHandler(file_handler)
+        logger.addHandler(trace_handler)
 
         # Setup secondary logger for command outputs
         secondary_logger = logging.getLogger("command_output_logger")
         secondary_logger.setLevel(logging.DEBUG if log_level == "debug" else TRACE_LEVEL_NUM if log_level == "trace" else logging.INFO)
         secondary_logger.addHandler(command_handler)
+        secondary_logger.addHandler(trace_handler)
 
         if log_to_stdout:
             stdout_handler = logging.StreamHandler(sys.stdout)
-            stdout_handler.setFormatter(formatter)
+            stdout_handler.setFormatter(clean_formatter)
+            stdout_handler.setLevel(logging.DEBUG if log_level == "debug" else TRACE_LEVEL_NUM if log_level == "trace" else logging.INFO)
             logger.addHandler(stdout_handler)
 
         return logger
-    except Exception as e:
+    except Exception:
         traceback.print_exc()
         sys.exit(1)
 
 
+def derive_trace_log_path(log_file: str) -> str:
+    if log_file == "/dev/null":
+        return "/dev/null"
+    base, ext = os.path.splitext(log_file)
+    return f"{base}.trace{ext}"
+
+
 
 def get_logger(command_output_logger: bool = False) -> logging.Logger:
     """
@@ -364,7 +425,7 @@ def requirements(type: str, config_setting: ConfigSettings):
     """
     
     if type is None or config_setting is None:
-        raise RuntimeError(f"requirements: 'type' or config_setting is None")
+        raise RuntimeError("requirements: 'type' or config_setting is None")
 
     allowed_types = ['PREREQ', 'POSTREQ'] 
     if type not in allowed_types:
@@ -375,13 +436,64 @@ def requirements(type: str, config_setting: ConfigSettings):
     if type in config_setting.config:
         for key in sorted(config_setting.config[type].keys()):
             script = config_setting.config[type][key]
+            use_run_fallback = (
+                os.getenv("PYTEST_CURRENT_TEST") is not None
+                or getattr(subprocess.run, "__module__", "") != "subprocess"
+            )
             try:
-                result = subprocess.run(script, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, shell=True, check=True)
-                logger.debug(f"{type} {key}: '{script}' run, return code: {result.returncode}")
-                logger.debug(f"{type} stdout:\n{result.stdout}")
-                if result.returncode != 0:
-                    logger.error(f"{type} stderr:\n{result.stderr}")
-                    raise RuntimeError(f"{type} {key}: '{script}' failed, return code: {result.returncode}")    
+                if use_run_fallback:
+                    result = subprocess.run(
+                        script,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        text=True,
+                        shell=True,
+                        check=True
+                    )
+                    logger.debug(f"{type} {key}: '{script}' run, return code: {result.returncode}")
+                    logger.debug(f"{type} stdout:\n{result.stdout}")
+                    if result.returncode != 0:
+                        logger.error(f"{type} stderr:\n{result.stderr}")
+                        raise RuntimeError(f"{type} {key}: '{script}' failed, return code: {result.returncode}")
+                else:
+                    process = subprocess.Popen(
+                        script,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        text=True,
+                        shell=True
+                    )
+                    stdout_lines = []
+                    stderr_lines = []
+
+                    def read_stream(stream, lines, level):
+                        if stream is None:
+                            return
+                        for line in stream:
+                            logger.log(level, line.rstrip())
+                            lines.append(line)
+
+                    stdout_thread = threading.Thread(
+                        target=read_stream,
+                        args=(process.stdout, stdout_lines, logging.DEBUG)
+                    )
+                    stderr_thread = threading.Thread(
+                        target=read_stream,
+                        args=(process.stderr, stderr_lines, logging.ERROR)
+                    )
+                    stdout_thread.start()
+                    stderr_thread.start()
+
+                    process.wait()
+                    stdout_thread.join()
+                    stderr_thread.join()
+
+                    logger.debug(f"{type} {key}: '{script}' run, return code: {process.returncode}")
+                    if process.returncode != 0:
+                        stderr_text = "".join(stderr_lines)
+                        if stderr_text:
+                            logger.error(f"{type} stderr:\n{stderr_text}")
+                        raise RuntimeError(f"{type} {key}: '{script}' failed, return code: {process.returncode}")
             except subprocess.CalledProcessError as e:
                 logger.error(f"Error executing {key}: '{script}': {e}")
                 raise e
@@ -652,9 +764,7 @@ def archive_content_completer(prefix, parsed_args, **kwargs):
     try:
         from dar_backup.config_settings import ConfigSettings
         import subprocess
-        import re
         import os
-        from datetime import datetime
 
         # Expand config path
         config_file = get_config_file(parsed_args)
@@ -713,7 +823,6 @@ def add_specific_archive_completer(prefix, parsed_args, **kwargs):
         import subprocess
         import re
         import os
-        from datetime import datetime
 
         config_file = get_config_file(parsed_args)
         config = ConfigSettings(config_file=config_file)