utils_devops 0.1.129__py3-none-any.whl → 0.1.130__py3-none-any.whl

utils_devops/core/systems.py

@@ -11,6 +11,7 @@ import shutil
 import time
 import socket
 import getpass
+import threading
 import ctypes  # For Windows admin check
 from typing import Optional, List, Dict, Union, Any, Callable, Tuple
 from rich.console import Console
@@ -293,157 +294,301 @@ def run(
     elevated: bool = False,
     capture: bool = True,
     logger: Optional[logger] = None,
-    stream: bool = False  # NEW: Add streaming option
+    stream: bool = False
 ) -> subprocess.CompletedProcess:
     """
-    Core run() – smart about shell and elevation.
-    If elevated=True on Unix, the cached sudo password is used (prompted once per process).
-    If sudo authentication fails we clear the cached password and raise a CalledProcessError
-    with a helpful message.
-    
-    NEW: If stream=True, output is streamed in real-time to the logger.
-    Returns subprocess.CompletedProcess
+    Robust run() with cross-platform threaded streaming + smart carriage-return handling.
+
+    - stream=True : streams output in real-time using threads, understands '\r' updates.
+    - capture controls whether stdout/stderr are returned in CompletedProcess (when stream=True,
+      capture=True will also collect into strings).
+    - If streaming fails for any reason we fallback to communicate() to collect remaining output.
+    - Works on Linux and Windows.
     """
-    logger = logger or DEFAULT_LOGGER 
+    logger = logger or DEFAULT_LOGGER
+
     if shell is None:
         shell = isinstance(cmd, str)
-    # Normalize cmd into either str or list form for subprocess
+
+    # Normalize command forms
     if isinstance(cmd, (list, tuple)):
         cmd_str = subprocess.list2cmdline(cmd)
         cmd_list: Union[List[str], str] = list(cmd)
     else:
         cmd_str = str(cmd)
         cmd_list = cmd_str if shell else [cmd_str]
+
     if dry_run:
         logger.info(f"[DRY-RUN] {cmd_str}")
         return subprocess.CompletedProcess(cmd_list if not shell else cmd_str, 0, stdout="", stderr="")
+
     stdin_input: Optional[str] = None
     use_list: Union[List[str], str] = cmd_list
+
+    # Determine platform
+    try:
+        is_win = (is_windows())
+    except NameError:
+        is_win = (os.name == "nt") or sys.platform.startswith("win")
+
+    # Handle elevated
     if elevated:
-        if is_windows():
-            # For Windows, use Start-Process -Verb RunAs via powershell
+        if is_win:
+            # Windows: use Start-Process via powershell RunAs
             ps = f"Start-Process -Verb RunAs -FilePath powershell -ArgumentList '-NoProfile','-Command','{cmd_str}' -Wait -PassThru"
             use_list = ["powershell", "-NoProfile", "-Command", ps]
             shell = False
         else:
-            # Unix: prepend sudo -S and provide password via stdin
-            pw = _get_sudo_password()
-            # Build command list safely (avoid shell where possible)
+            # Unix: attempt to get cached sudo password via user-provided helper
+            try:
+                pw = _get_sudo_password()
+            except NameError:
+                raise RuntimeError("elevated=True requested but _get_sudo_password() is not implemented in the environment.")
             if isinstance(cmd, (list, tuple)):
                 base_list = list(cmd)
             else:
-                # if original was a string and shell=True, pass the string to sudo as a single shell invocation
                 base_list = [cmd_str] if shell else [cmd_str]
             use_list = ["sudo", "-S"] + base_list
             stdin_input = (pw + "\n") if pw is not None else None
-            shell = False  # we pass a list to Popen
-    # Log the final command representation (avoid logging password)
+            shell = False
+
+    # Log final command (avoid logging sensitive data)
     try:
         if isinstance(use_list, list):
             logger.debug(f"Executing (list): {' '.join(use_list)}")
         else:
             logger.debug(f"Executing (shell): {use_list}")
-        
+
         proc = subprocess.Popen(
             use_list if not shell else (cmd_str),
             cwd=str(cwd) if cwd else None,
             env=env,
-            stdout=subprocess.PIPE if capture else None,
-            stderr=subprocess.PIPE if capture else None,
+            stdout=subprocess.PIPE if (capture or stream) else None,
+            stderr=subprocess.PIPE if (capture or stream) else None,
             stdin=subprocess.PIPE if stdin_input is not None else None,
             text=True,
             shell=shell,
-            bufsize=1,  # Line buffered
+            bufsize=1,
             universal_newlines=True,
         )
-        
+
+        # Send sudo password if needed
         if stdin_input is not None and proc.stdin:
-            # write password and flush; do not keep password in logs
             try:
                 proc.stdin.write(stdin_input)
                 proc.stdin.flush()
                 proc.stdin.close()
             except Exception:
-                # If writing fails, ensure we close and continue to wait for process
                 try:
                     proc.stdin.close()
                 except Exception:
                     pass
-        
-        stdout_lines = []
-        stderr_lines = []
-        
-        if stream and capture:
-            # Stream output in real-time
+
+        # Output collectors
+        stdout_lines: List[str] = []
+        stderr_lines: List[str] = []
+
+        # --- Smart threaded streaming implementation ---
+        def _smart_reader(pipe, log_func, collector: Optional[List[str]], stop_event: threading.Event):
+            """
+            Read from pipe in chunks, handle '\r' (line replace) and '\n' (new line).
+            Appends to collector (if provided) and logs via log_func.
+            """
+            try:
+                buffer = ""
+                last_rendered = None  # used to avoid repeated identical logs for \r updates
+
+                # We'll read in reasonably-sized chunks. read() will block, but on separate thread that's fine.
+                while not stop_event.is_set():
+                    chunk = pipe.read(1024)
+                    if not chunk:
+                        # EOF reached
+                        break
+                    buffer += chunk
+
+                    # Process as long as there's control chars
+                    while True:
+                        # find next control char indices
+                        idx_n = buffer.find("\n")
+                        idx_r = buffer.find("\r")
+
+                        if idx_n == -1 and idx_r == -1:
+                            break
+
+                        # Which control comes first?
+                        if idx_r != -1 and (idx_n == -1 or idx_r < idx_n):
+                            # Carriage return: replace current line
+                            line = buffer[:idx_r]
+                            buffer = buffer[idx_r + 1:]
+                            # Only log if changed (avoids spamming identical updates)
+                            if line != last_rendered:
+                                # strip trailing CR/LF but preserve internal whitespace
+                                to_log = line.rstrip("\r\n")
+                                try:
+                                    log_func(to_log)
+                                except Exception:
+                                    # logging should not raise to user
+                                    pass
+                                if collector is not None:
+                                    collector.append(line + ("\n" if collector is not None else ""))
+                                last_rendered = line
+                        else:
+                            # Newline: finalize this line
+                            line = buffer[:idx_n]
+                            buffer = buffer[idx_n + 1:]
+                            to_log = line.rstrip("\r\n")
+                            try:
+                                log_func(to_log)
+                            except Exception:
+                                pass
+                            if collector is not None:
+                                collector.append(line + "\n")
+                            last_rendered = None
+
+                # Flush whatever remains in buffer
+                if buffer:
+                    buf_strip = buffer.rstrip("\r\n")
+                    if buf_strip:
+                        try:
+                            log_func(buf_strip)
+                        except Exception:
+                            pass
+                        if collector is not None:
+                            collector.append(buffer)
+                # close pipe
+                try:
+                    pipe.close()
+                except Exception:
+                    pass
+            except Exception as exc:
+                # Ensure we don't crash the thread; bubble up via logging
+                logger.exception(f"stream reader error: {exc}")
+                try:
+                    pipe.close()
+                except Exception:
+                    pass
+                # re-raise to let outer context know (we'll catch in caller via threads status)
+                raise
+
+        rc = None
+        if stream and (proc.stdout is not None or proc.stderr is not None):
             logger.info("Streaming command output...")
-            
-            # Create file descriptors for reading stdout and stderr
-            import select
-            import sys
-            
-            # Read from stdout and stderr simultaneously
-            while True:
-                # Check if process has terminated
-                if proc.poll() is not None:
-                    break
-                
-                # Use select to wait for data
-                read_fds = []
+            stop_event = threading.Event()
+            threads: List[threading.Thread] = []
+            stream_exception = None
+
+            # Start threads
+            try:
                 if proc.stdout:
-                    read_fds.append(proc.stdout)
+                    t_out = threading.Thread(
+                        target=_smart_reader,
+                        args=(proc.stdout, logger.info, stdout_lines if capture else None, stop_event),
+                        daemon=True
+                    )
+                    t_out.start()
+                    threads.append(t_out)
+
                 if proc.stderr:
-                    read_fds.append(proc.stderr)
-                
-                if not read_fds:
-                    break
-                
-                rlist, _, _ = select.select(read_fds, [], [], 0.1)
-                
-                for fd in rlist:
-                    if fd is proc.stdout:
-                        line = proc.stdout.readline()
-                        if line:
-                            logger.info(line.rstrip())
-                            stdout_lines.append(line)
-                    elif fd is proc.stderr:
-                        line = proc.stderr.readline()
-                        if line:
-                            logger.warning(line.rstrip())
-                            stderr_lines.append(line)
-            
-            # Read any remaining output
-            if proc.stdout:
-                for line in proc.stdout:
-                    logger.info(line.rstrip())
-                    stdout_lines.append(line)
-            if proc.stderr:
-                for line in proc.stderr:
-                    logger.warning(line.rstrip())
-                    stderr_lines.append(line)
-            
-            stdout = "".join(stdout_lines)
-            stderr = "".join(stderr_lines)
-            rc = proc.wait()
-            
+                    t_err = threading.Thread(
+                        target=_smart_reader,
+                        args=(proc.stderr, logger.warning, stderr_lines if capture else None, stop_event),
+                        daemon=True
+                    )
+                    t_err.start()
+                    threads.append(t_err)
+
+                # Wait for process while allowing KeyboardInterrupt
+                try:
+                    rc = proc.wait()
+                except KeyboardInterrupt:
+                    logger.debug("KeyboardInterrupt caught: terminating child process")
+                    stop_event.set()
+                    try:
+                        proc.terminate()
+                    except Exception:
+                        pass
+                    # wait a bit then force kill
+                    try:
+                        proc.wait(timeout=2)
+                    except Exception:
+                        try:
+                            proc.kill()
+                        except Exception:
+                            pass
+                    rc = proc.returncode if proc.returncode is not None else -1
+
+            except Exception as exc:
+                # If any exception occurs while starting/monitoring threads -> fallback
+                stream_exception = exc
+                logger.exception(f"Streaming failed, falling back to communicate(): {exc}")
+            finally:
+                # Signal threads to stop and join
+                stop_event.set()
+                for t in threads:
+                    t.join(timeout=1)
+
+            # If streaming had exception, fallback to communicate to collect remaining output safely.
+            if stream_exception is not None:
+                try:
+                    # communicate will return remaining output that reader threads didn't capture
+                    comm_out, comm_err = proc.communicate(timeout=5)
+                except Exception:
+                    try:
+                        # best-effort: kill and read whatever
+                        proc.kill()
+                    except Exception:
+                        pass
+                    try:
+                        comm_out, comm_err = proc.communicate(timeout=5)
+                    except Exception:
+                        comm_out, comm_err = ("", "")
+                # append remaining to collectors if capture True
+                if capture:
+                    if comm_out:
+                        stdout_lines.append(comm_out)
+                    if comm_err:
+                        stderr_lines.append(comm_err)
+                # set rc if not set
+                if rc is None:
+                    rc = proc.returncode if proc.returncode is not None else 0
+
+            # Compose final stdout/stderr
+            stdout = "".join(stdout_lines) if capture else ""
+            stderr = "".join(stderr_lines) if capture else ""
+            if rc is None:
+                rc = proc.returncode if proc.returncode is not None else 0
+
         else:
-            # Original behavior: wait for completion
-            stdout, stderr = proc.communicate()
+            # original behavior: blocking wait & capture (or not)
+            try:
+                stdout, stderr = proc.communicate()
+            except Exception as exc:
+                # if communicate fails, try to kill and fallback
+                logger.exception(f"proc.communicate() failed: {exc}")
+                try:
+                    proc.kill()
+                except Exception:
+                    pass
+                try:
+                    stdout, stderr = proc.communicate(timeout=5)
+                except Exception:
+                    stdout, stderr = ("", "")
             rc = proc.returncode
-        
+
+        # Build CompletedProcess result
         result = subprocess.CompletedProcess(
             use_list if not shell else cmd_str,
             rc,
             stdout=stdout or "",
             stderr=stderr or "",
         )
-        
+
+        # Post-run handling & sudo auth checks (Unix)
         if rc == 0:
             logger.info(f"Command succeeded (rc={rc})")
         else:
-            # If elevated on Unix and sudo-auth failure detected, clear cache and raise helpful error
-            if elevated and not is_windows():
+            if elevated and not is_win:
                 lowerr = (stderr or "").lower()
-                # Common sudo auth failure tokens
                 auth_tokens = [
                     "incorrect password",
                     "authentication failure",
@@ -454,25 +599,26 @@ def run(
                     "pam_authenticate",
                 ]
                 if any(tok in lowerr for tok in auth_tokens):
-                    # Clear cached password so user isn't forced to keep using wrong one
-                    clear_sudo_password()
-                    logger.error("sudo authentication failed. Cached sudo password cleared.")
-                    # Raise with stderr included for context
+                    try:
+                        clear_sudo_password()
+                    except NameError:
+                        logger.error("sudo authentication failed and clear_sudo_password() not implemented.")
+                    else:
+                        logger.error("sudo authentication failed. Cached sudo password cleared.")
                     raise subprocess.CalledProcessError(rc, use_list if not shell else cmd_str, output=stdout, stderr=stderr)
-            # General logging for failure
-            logger.error(f"Command failed (rc={rc}) – {stderr.strip() if stderr else ''}")
-        
+            logger.error(f"Command failed (rc={rc}) – { (stderr or '').strip() }")
+
         if rc != 0 and not no_die:
             raise subprocess.CalledProcessError(rc, use_list if not shell else cmd_str, output=stdout, stderr=stderr)
-        
+
         return result
-    
+
     except subprocess.CalledProcessError:
-        # Re-raise CalledProcessError so callers can handle; do not clear cached password here
         raise
     except Exception as e:
         logger.exception(f"Unexpected error running command: {e}")
         raise
+
 # -------------------------------------------------
 # Exec helper – shows command + output + logs
 # -------------------------------------------------

utils_devops/extras/docker_ops.py

@@ -430,8 +430,7 @@ def read_compose_file(compose_file: str, env_file: Optional[str] = None) -> Dict
             envs.import_env_to_system(get_env_compose(env_file))
         
         # Read compose file content
-        #content = files.read_file(compose_file)
-        content = strings.parse_yaml(compose_file)
+        content = files.read_file(compose_file)
         
         # Expand environment variables in the content
         data = _expand_env_vars_in_compose(content)

{utils_devops-0.1.129.dist-info → utils_devops-0.1.130.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: utils_devops
-Version: 0.1.129
+Version: 0.1.130
 Summary: Lightweight DevOps utilities for automation scripts: config editing (YAML/JSON/INI/.env), templating, diffing, and CLI tools
 License: MIT
 Keywords: devops,automation,nginx,cli,jinja2,yaml,config,diff,templating,logging,docker,compose,file-ops

{utils_devops-0.1.129.dist-info → utils_devops-0.1.130.dist-info}/RECORD

@@ -6,10 +6,10 @@ utils_devops/core/files.py,sha256=WwLkEy83PTyIQT2yhYJhJA6xkrMi2mfKoDT45BswlvQ,50
 utils_devops/core/logs.py,sha256=xHVSPLE1a7kivrBqQWQdOYjYfm69e2C9yQ7vBu3U68k,30829
 utils_devops/core/script_helpers.py,sha256=NcOh1f1X_TYm5uuP5YFCYpNF5tr2NVE8DehLj9DMo3c,31480
 utils_devops/core/strings.py,sha256=8s0GSjcyTKwLjJjsJ_XfOJxPtyb549icDlU9SUxSvHI,42481
-utils_devops/core/systems.py,sha256=D73G_lvd-OoGmByJ3_gMTifkLm6PuU4AxpFm0vfDiJo,41629
+utils_devops/core/systems.py,sha256=wNbEFUAvbMPdqWN-iXvTzvj5iE9xaWfjZYYvD0EZAH0,47577
 utils_devops/extras/__init__.py,sha256=ZXHeVLHO3_qiW9AY-UQ_YA9cQzmkLGv54a2UbyvtlM0,3571
 utils_devops/extras/aws_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-utils_devops/extras/docker_ops.py,sha256=QdUy3rEXI-hpx1kmuyaHzHMq2XlqxqnER-U1Oip0SIM,175081
+utils_devops/extras/docker_ops.py,sha256=7941chrSL1OnbvQCOM9Uz7TRtalKDPs4yNFkPre3YUA,175029
 utils_devops/extras/git_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 utils_devops/extras/interaction_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 utils_devops/extras/metrics_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,7 +19,7 @@ utils_devops/extras/notification_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 utils_devops/extras/performance_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 utils_devops/extras/ssh_ops.py,sha256=tBhydL7z-XzJUy8Cv4QWn_tHFToyRK0Enx3EllnZS4A,68976
 utils_devops/extras/vault_ops.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-utils_devops-0.1.129.dist-info/METADATA,sha256=6u2GdAnSCoMEkpXXnswQFsNFtFaIj9MqiBAhbRgBLTo,1903
-utils_devops-0.1.129.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-utils_devops-0.1.129.dist-info/entry_points.txt,sha256=ei3B6ZL5yu6dOq-U1r8wsBdkXeg63RAyV7m8_ADaE6k,53
-utils_devops-0.1.129.dist-info/RECORD,,
+utils_devops-0.1.130.dist-info/METADATA,sha256=Qz7EmI-m2ZtIJiLLzRj98dtCYvpbJGfk9EMGQu5hfRQ,1903
+utils_devops-0.1.130.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+utils_devops-0.1.130.dist-info/entry_points.txt,sha256=ei3B6ZL5yu6dOq-U1r8wsBdkXeg63RAyV7m8_ADaE6k,53
+utils_devops-0.1.130.dist-info/RECORD,,