dar-backup 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

dar_backup/command_runner.py

@@ -13,6 +13,7 @@ try:
 except ImportError:
     termios = None
 import tempfile
+import time
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src")))
 from typing import List, Optional, Union
 from dar_backup.util import get_logger
@@ -84,15 +85,26 @@ class CommandRunner:
         self,
         logger: Optional[logging.Logger] = None,
         command_logger: Optional[logging.Logger] = None,
-        default_timeout: int = 30
+        default_timeout: int = 30,
+        default_capture_limit_bytes: Optional[int] = None
     ):
         self.logger = logger or get_logger()
         self.command_logger = command_logger or get_logger(command_output_logger=True)
+        if default_timeout is not None:
+            try:
+                default_timeout = int(default_timeout)
+            except (TypeError, ValueError):
+                default_timeout = 30
+            if not isinstance(default_timeout, int):
+                default_timeout = 30
         self.default_timeout = default_timeout
+        self.default_capture_limit_bytes = default_capture_limit_bytes
 
         if not self.logger or not self.command_logger:
             self.logger_fallback()
 
+        if self.default_timeout is not None and self.default_timeout <= 0:
+            self.default_timeout = None
 
     def logger_fallback(self):
         """
@@ -129,12 +141,21 @@ class CommandRunner:
         timeout: Optional[int] = None,
         check: bool = False,
         capture_output: bool = True,
+        capture_output_limit_bytes: Optional[int] = None,
+        log_output: bool = True,
         text: bool = True,
         cwd: Optional[str] = None,
         stdin: Optional[int] = subprocess.DEVNULL
     ) -> CommandResult:
         self._text_mode = text
-        timeout = timeout or self.default_timeout
+        if timeout is None:
+            timeout = self.default_timeout
+        if timeout is not None and timeout <= 0:
+            timeout = None
+        if capture_output_limit_bytes is None:
+            capture_output_limit_bytes = self.default_capture_limit_bytes
+        if capture_output_limit_bytes is not None and capture_output_limit_bytes < 0:
+            capture_output_limit_bytes = None
 
         tty_fd = None
         tty_file = None
@@ -163,9 +184,13 @@ class CommandRunner:
             except ValueError as e:
                 stack = traceback.format_exc()
                 self.logger.error(f"Command sanitation failed: {e}")
+                if isinstance(cmd, list):
+                    cmd_text = " ".join(map(str, cmd))
+                else:
+                    cmd_text = str(cmd)
                 return CommandResult(
                     returncode=-1,
-                    note=f"Sanitizing failed: command: {' '.join(cmd)}",
+                    note=f"Sanitizing failed: command: {cmd_text}",
                     stdout='',
                     stderr=str(e),
                     stack=stack,
@@ -183,17 +208,33 @@ class CommandRunner:
 
             stdout_lines = []
             stderr_lines = []
+            truncated_stdout = {"value": False}
+            truncated_stderr = {"value": False}
 
             try:
+                start_time = time.monotonic()
+                use_pipes = capture_output or log_output
                 process = subprocess.Popen(
                     cmd,
-                    stdout=subprocess.PIPE if capture_output else None,
-                    stderr=subprocess.PIPE if capture_output else None,
+                    stdout=subprocess.PIPE if use_pipes else None,
+                    stderr=subprocess.PIPE if use_pipes else None,
                     stdin=stdin,
                     text=False,
                     bufsize=-1,
                     cwd=cwd
                 )
+                pid = getattr(process, "pid", None)
+                if log_output:
+                    self.command_logger.debug(
+                        "Process started pid=%s cwd=%s",
+                        pid if pid is not None else "unknown",
+                        cwd or os.getcwd(),
+                    )
+                self.logger.debug(
+                    "Process started pid=%s cwd=%s",
+                    pid if pid is not None else "unknown",
+                    cwd or os.getcwd(),
+                )
             except Exception as e:
                 stack = traceback.format_exc()
                 return CommandResult(
@@ -203,7 +244,8 @@ class CommandRunner:
                     stack=stack
                 )
 
-            def stream_output(stream, lines, level):
+            def stream_output(stream, lines, level, truncated_flag):
+                captured_bytes = 0
                 try:
                     while True:
                         chunk = stream.read(1024)
@@ -211,10 +253,40 @@ class CommandRunner:
                             break
                         if self._text_mode:
                             decoded = chunk.decode('utf-8', errors='replace')
-                            lines.append(decoded)
-                            self.command_logger.log(level, decoded.strip())
+                            if log_output:
+                                self.command_logger.log(level, decoded.strip())
+                            if capture_output:
+                                if capture_output_limit_bytes is None:
+                                    lines.append(decoded)
+                                else:
+                                    remaining = capture_output_limit_bytes - captured_bytes
+                                    if remaining > 0:
+                                        if len(chunk) <= remaining:
+                                            lines.append(decoded)
+                                            captured_bytes += len(chunk)
+                                        else:
+                                            piece = chunk[:remaining]
+                                            lines.append(piece.decode('utf-8', errors='replace'))
+                                            captured_bytes = capture_output_limit_bytes
+                                            truncated_flag["value"] = True
+                                    else:
+                                        truncated_flag["value"] = True
                         else:
-                            lines.append(chunk)
+                            if capture_output:
+                                if capture_output_limit_bytes is None:
+                                    lines.append(chunk)
+                                else:
+                                    remaining = capture_output_limit_bytes - captured_bytes
+                                    if remaining > 0:
+                                        if len(chunk) <= remaining:
+                                            lines.append(chunk)
+                                            captured_bytes += len(chunk)
+                                        else:
+                                            lines.append(chunk[:remaining])
+                                            captured_bytes = capture_output_limit_bytes
+                                            truncated_flag["value"] = True
+                                    else:
+                                        truncated_flag["value"] = True
                             # Avoid logging raw binary data to prevent garbled logs
                 except Exception as e:
                     self.logger.warning(f"stream_output decode error: {e}")
@@ -222,12 +294,18 @@ class CommandRunner:
                     stream.close()
 
             threads = []
-            if capture_output and process.stdout:
-                t_out = threading.Thread(target=stream_output, args=(process.stdout, stdout_lines, logging.INFO))
+            if (capture_output or log_output) and process.stdout:
+                t_out = threading.Thread(
+                    target=stream_output,
+                    args=(process.stdout, stdout_lines, logging.INFO, truncated_stdout)
+                )
                 t_out.start()
                 threads.append(t_out)
-            if capture_output and process.stderr:
-                t_err = threading.Thread(target=stream_output, args=(process.stderr, stderr_lines, logging.ERROR))
+            if (capture_output or log_output) and process.stderr:
+                t_err = threading.Thread(
+                    target=stream_output,
+                    args=(process.stderr, stderr_lines, logging.ERROR, truncated_stderr)
+                )
                 t_err.start()
                 threads.append(t_err)
 
@@ -235,7 +313,12 @@ class CommandRunner:
                 process.wait(timeout=timeout)
             except subprocess.TimeoutExpired:
                 process.kill()
-                log_msg = f"Command timed out after {timeout} seconds: {' '.join(cmd)}:\n"
+                duration = time.monotonic() - start_time
+                pid = getattr(process, "pid", None)
+                log_msg = (
+                    f"Command timed out after {timeout} seconds: {' '.join(cmd)} "
+                    f"(pid={pid if pid is not None else 'unknown'}, elapsed={duration:.2f}s):\n"
+                )
                 self.logger.error(log_msg)
                 return CommandResult(-1, ''.join(stdout_lines), log_msg.join(stderr_lines))
             except Exception as e:
@@ -246,6 +329,21 @@ class CommandRunner:
 
             for t in threads:
                 t.join()
+            duration = time.monotonic() - start_time
+            pid = getattr(process, "pid", None)
+            if log_output:
+                self.command_logger.debug(
+                    "Process finished pid=%s returncode=%s elapsed=%.2fs",
+                    pid if pid is not None else "unknown",
+                    process.returncode,
+                    duration,
+                )
+            self.logger.debug(
+                "Process finished pid=%s returncode=%s elapsed=%.2fs",
+                pid if pid is not None else "unknown",
+                process.returncode,
+                duration,
+            )
 
             if self._text_mode:
                 stdout_combined = ''.join(stdout_lines)
@@ -254,6 +352,15 @@ class CommandRunner:
                 stdout_combined = b''.join(stdout_lines)
                 stderr_combined = b''.join(stderr_lines)
 
+            note = None
+            if truncated_stdout["value"] or truncated_stderr["value"]:
+                parts = []
+                if truncated_stdout["value"]:
+                    parts.append("stdout truncated")
+                if truncated_stderr["value"]:
+                    parts.append("stderr truncated")
+                note = ", ".join(parts)
+
             if check and process.returncode != 0:
                 self.logger.error(f"Command failed with exit code {process.returncode}")
                 return CommandResult(
@@ -266,7 +373,8 @@ class CommandRunner:
             return CommandResult(
                 process.returncode,
                 stdout_combined,
-                stderr_combined
+                stderr_combined,
+                note=note
             )
         finally:
             if termios is not None and saved_tty_attrs is not None and tty_fd is not None:

dar_backup/config_settings.py

@@ -4,7 +4,6 @@ import configparser
 import re
 from dataclasses import dataclass, field, fields
 from os.path import expandvars, expanduser
-from pathlib import Path
 from typing import Optional, Pattern
 
 from dar_backup.exceptions import ConfigSettingsError
@@ -39,6 +38,7 @@ class ConfigSettings:
     min_size_verification_mb: int = field(init=False)
     no_files_verification: int = field(init=False)
     command_timeout_secs: int = field(init=False)
+    command_capture_max_bytes: Optional[int] = field(init=False, default=None)
     backup_dir: str = field(init=False)
     test_restore_dir: str = field(init=False)
     backup_d_dir: str = field(init=False)
@@ -47,14 +47,14 @@ class ConfigSettings:
     error_correction_percent: int = field(init=False)
     par2_enabled: bool = field(init=False)
     par2_dir: Optional[str] = field(init=False, default=None)
-    par2_layout: Optional[str] = field(init=False, default=None)
-    par2_mode: Optional[str] = field(init=False, default=None)
     par2_ratio_full: Optional[int] = field(init=False, default=None)
     par2_ratio_diff: Optional[int] = field(init=False, default=None)
     par2_ratio_incr: Optional[int] = field(init=False, default=None)
     par2_run_verify: Optional[bool] = field(init=False, default=None)
     logfile_max_bytes: int = field(init=False)
     logfile_no_count: int = field(init=False)    
+    trace_log_max_bytes: int = field(init=False)
+    trace_log_backup_count: int = field(init=False)
     dar_backup_discord_webhook_url: Optional[str] = field(init=False, default=None)
     restoretest_exclude_prefixes: list[str] = field(init=False, default_factory=list)
     restoretest_exclude_suffixes: list[str] = field(init=False, default_factory=list)
@@ -83,6 +83,20 @@ class ConfigSettings:
             "type": int,
             "default": 5,
         },
+        {
+            "section": "MISC",
+            "key": "TRACE_LOG_MAX_BYTES",
+            "attr": "trace_log_max_bytes",
+            "type": int,
+            "default": 10485760, # 10 MB
+        },
+        {
+            "section": "MISC",
+            "key": "TRACE_LOG_BACKUP_COUNT",
+            "attr": "trace_log_backup_count",
+            "type": int,
+            "default": 1,
+        },
         {
             "section": "MISC",
             "key": "DAR_BACKUP_DISCORD_WEBHOOK_URL",
@@ -90,6 +104,13 @@ class ConfigSettings:
             "type": str,
             "default": None,
         },
+        {
+            "section": "MISC",
+            "key": "COMMAND_CAPTURE_MAX_BYTES",
+            "attr": "command_capture_max_bytes",
+            "type": int,
+            "default": 102400,
+        },
         # Add more optional fields here
     ]
 
@@ -98,7 +119,7 @@ class ConfigSettings:
             raise ConfigSettingsError("`config_file` must be specified.")
 
         try:
-            self.config = configparser.ConfigParser()
+            self.config = configparser.ConfigParser(inline_comment_prefixes=['#'])
             loaded_files = self.config.read(self.config_file)
             if not loaded_files:
                 raise RuntimeError(f"Configuration file not found or unreadable: '{self.config_file}'")
@@ -124,8 +145,6 @@ class ConfigSettings:
                 raise ConfigSettingsError(f"Invalid boolean value for 'ENABLED' in [PAR2]: '{val}'")
 
             self.par2_dir = self._get_optional_str("PAR2", "PAR2_DIR", default=None)
-            self.par2_layout = self._get_optional_str("PAR2", "PAR2_LAYOUT", default="by-backup")
-            self.par2_mode = self._get_optional_str("PAR2", "PAR2_MODE", default=None)
             self.par2_ratio_full = self._get_optional_int("PAR2", "PAR2_RATIO_FULL", default=None)
             self.par2_ratio_diff = self._get_optional_int("PAR2", "PAR2_RATIO_DIFF", default=None)
             self.par2_ratio_incr = self._get_optional_int("PAR2", "PAR2_RATIO_INCR", default=None)
@@ -240,8 +259,6 @@ class ConfigSettings:
         """
         par2_config = {
             "par2_dir": self.par2_dir,
-            "par2_layout": self.par2_layout,
-            "par2_mode": self.par2_mode,
             "par2_ratio_full": self.par2_ratio_full,
             "par2_ratio_diff": self.par2_ratio_diff,
             "par2_ratio_incr": self.par2_ratio_incr,
@@ -260,10 +277,6 @@ class ConfigSettings:
                 continue
             if key == "PAR2_DIR":
                 par2_config["par2_dir"] = value
-            elif key == "PAR2_LAYOUT":
-                par2_config["par2_layout"] = value
-            elif key == "PAR2_MODE":
-                par2_config["par2_mode"] = value
             elif key == "PAR2_RATIO_FULL":
                 par2_config["par2_ratio_full"] = int(value)
             elif key == "PAR2_RATIO_DIFF":

dar_backup/dar-backup.conf

@@ -17,7 +17,14 @@ NO_FILES_VERIFICATION = 5
 # The author has such `dar` tasks running for 10-15 hours on the yearly backups, so a value of 24 hours is used.
 # If a timeout is not specified when using the CommandRunner, a default timeout of 30 secs is used.
 COMMAND_TIMEOUT_SECS = 86400
+# Optional limit on captured command output (in bytes). Output beyond this
+# size is still logged but not kept in memory. Use 0 to avoid buffering entirely.
+# Default is 102400.
+# COMMAND_CAPTURE_MAX_BYTES = 102400
 #DAR_BACKUP_DISCORD_WEBHOOK_URL = https://discord.com/api/webhooks/<id>/<token>
+# Optional Trace log configuration (debug level logs with stack traces)
+# TRACE_LOG_MAX_BYTES = 10485760  # 10 MB default
+# TRACE_LOG_BACKUP_COUNT = 1      # 1 backup file default
 
 [DIRECTORIES]
 BACKUP_DIR = @@BACKUP_DIR@@

dar_backup/dar-backup.conf.j2

@@ -31,6 +31,9 @@ LOGFILE_LOCATION = {{ vars_map.DAR_BACKUP_DIR -}}/dar-backup.log
 # LOGFILE_BACKUP_COUNT = 5      # default, change as needed
 # DAR_BACKUP_DISCORD_WEBHOOK_URL **should really** be given as an environment variable for security reasons
 # DAR_BACKUP_DISCORD_WEBHOOK_URL = https://discord.com/api/webhooks/<id>/<token>
+# Optional Trace log configuration (debug level logs with stack traces)
+# TRACE_LOG_MAX_BYTES = 10485760  # 10 MB default
+# TRACE_LOG_BACKUP_COUNT = 1      # 1 backup file default
 
 MAX_SIZE_VERIFICATION_MB = 2
 MIN_SIZE_VERIFICATION_MB = 0
@@ -64,7 +67,6 @@ ERROR_CORRECTION_PERCENT = 5
 ENABLED = True
 # Optional PAR2 configuration
 # PAR2_DIR = /path/to/par2-store
-# PAR2_LAYOUT = by-backup
 # PAR2_RATIOs are meuasured as percentages. Same function as ERROR_CORRECTION_PERCENT
 # PAR2_RATIO_FULL = 10
 # PAR2_RATIO_DIFF = 5