utils_devops 0.1.129__tar.gz → 0.1.136__tar.gz

{utils_devops-0.1.129 → utils_devops-0.1.136}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: utils_devops
-Version: 0.1.129
+Version: 0.1.136
 Summary: Lightweight DevOps utilities for automation scripts: config editing (YAML/JSON/INI/.env), templating, diffing, and CLI tools
 License: MIT
 Keywords: devops,automation,nginx,cli,jinja2,yaml,config,diff,templating,logging,docker,compose,file-ops

{utils_devops-0.1.129 → utils_devops-0.1.136}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "utils_devops"
-version = "0.1.129"  # Bumped for new string features + diffing
+version = "0.1.136"  # Bumped for new string features + diffing
 description = "Lightweight DevOps utilities for automation scripts: config editing (YAML/JSON/INI/.env), templating, diffing, and CLI tools"
 authors = ["Hamed Sheikhan <sh.sheikhan.m@gmail.com>"]
 license = "MIT"

{utils_devops-0.1.129 → utils_devops-0.1.136}/src/utils_devops/core/systems.py

@@ -11,6 +11,7 @@ import shutil
 import time
 import socket
 import getpass
+import threading
 import ctypes  # For Windows admin check
 from typing import Optional, List, Dict, Union, Any, Callable, Tuple
 from rich.console import Console
@@ -293,157 +294,301 @@ def run(
     elevated: bool = False,
     capture: bool = True,
     logger: Optional[logger] = None,
-    stream: bool = False  # NEW: Add streaming option
+    stream: bool = False
 ) -> subprocess.CompletedProcess:
     """
-    Core run() – smart about shell and elevation.
-    If elevated=True on Unix, the cached sudo password is used (prompted once per process).
-    If sudo authentication fails we clear the cached password and raise a CalledProcessError
-    with a helpful message.
-    
-    NEW: If stream=True, output is streamed in real-time to the logger.
-    Returns subprocess.CompletedProcess
+    Robust run() with cross-platform threaded streaming + smart carriage-return handling.
+
+    - stream=True : streams output in real-time using threads, understands '\r' updates.
+    - capture controls whether stdout/stderr are returned in CompletedProcess (when stream=True,
+      capture=True will also collect into strings).
+    - If streaming fails for any reason we fallback to communicate() to collect remaining output.
+    - Works on Linux and Windows.
     """
-    logger = logger or DEFAULT_LOGGER 
+    logger = logger or DEFAULT_LOGGER
+
     if shell is None:
         shell = isinstance(cmd, str)
-    # Normalize cmd into either str or list form for subprocess
+
+    # Normalize command forms
     if isinstance(cmd, (list, tuple)):
         cmd_str = subprocess.list2cmdline(cmd)
         cmd_list: Union[List[str], str] = list(cmd)
     else:
         cmd_str = str(cmd)
         cmd_list = cmd_str if shell else [cmd_str]
+
     if dry_run:
         logger.info(f"[DRY-RUN] {cmd_str}")
         return subprocess.CompletedProcess(cmd_list if not shell else cmd_str, 0, stdout="", stderr="")
+
     stdin_input: Optional[str] = None
     use_list: Union[List[str], str] = cmd_list
+
+    # Determine platform
+    try:
+        is_win = (is_windows())
+    except NameError:
+        is_win = (os.name == "nt") or sys.platform.startswith("win")
+
+    # Handle elevated
     if elevated:
-        if is_windows():
-            # For Windows, use Start-Process -Verb RunAs via powershell
+        if is_win:
+            # Windows: use Start-Process via powershell RunAs
             ps = f"Start-Process -Verb RunAs -FilePath powershell -ArgumentList '-NoProfile','-Command','{cmd_str}' -Wait -PassThru"
             use_list = ["powershell", "-NoProfile", "-Command", ps]
             shell = False
         else:
-            # Unix: prepend sudo -S and provide password via stdin
-            pw = _get_sudo_password()
-            # Build command list safely (avoid shell where possible)
+            # Unix: attempt to get cached sudo password via user-provided helper
+            try:
+                pw = _get_sudo_password()
+            except NameError:
+                raise RuntimeError("elevated=True requested but _get_sudo_password() is not implemented in the environment.")
             if isinstance(cmd, (list, tuple)):
                 base_list = list(cmd)
             else:
-                # if original was a string and shell=True, pass the string to sudo as a single shell invocation
                 base_list = [cmd_str] if shell else [cmd_str]
             use_list = ["sudo", "-S"] + base_list
             stdin_input = (pw + "\n") if pw is not None else None
-            shell = False  # we pass a list to Popen
-    # Log the final command representation (avoid logging password)
+            shell = False
+
+    # Log final command (avoid logging sensitive data)
     try:
         if isinstance(use_list, list):
             logger.debug(f"Executing (list): {' '.join(use_list)}")
         else:
             logger.debug(f"Executing (shell): {use_list}")
-        
+
         proc = subprocess.Popen(
             use_list if not shell else (cmd_str),
             cwd=str(cwd) if cwd else None,
             env=env,
-            stdout=subprocess.PIPE if capture else None,
-            stderr=subprocess.PIPE if capture else None,
+            stdout=subprocess.PIPE if (capture or stream) else None,
+            stderr=subprocess.PIPE if (capture or stream) else None,
             stdin=subprocess.PIPE if stdin_input is not None else None,
             text=True,
             shell=shell,
-            bufsize=1,  # Line buffered
+            bufsize=1,
             universal_newlines=True,
         )
-        
+
+        # Send sudo password if needed
         if stdin_input is not None and proc.stdin:
-            # write password and flush; do not keep password in logs
             try:
                 proc.stdin.write(stdin_input)
                 proc.stdin.flush()
                 proc.stdin.close()
             except Exception:
-                # If writing fails, ensure we close and continue to wait for process
                 try:
                     proc.stdin.close()
                 except Exception:
                     pass
-        
-        stdout_lines = []
-        stderr_lines = []
-        
-        if stream and capture:
-            # Stream output in real-time
+
+        # Output collectors
+        stdout_lines: List[str] = []
+        stderr_lines: List[str] = []
+
+        # --- Smart threaded streaming implementation ---
+        def _smart_reader(pipe, log_func, collector: Optional[List[str]], stop_event: threading.Event):
+            """
+            Read from pipe in chunks, handle '\r' (line replace) and '\n' (new line).
+            Appends to collector (if provided) and logs via log_func.
+            """
+            try:
+                buffer = ""
+                last_rendered = None  # used to avoid repeated identical logs for \r updates
+
+                # We'll read in reasonably-sized chunks. read() will block, but on separate thread that's fine.
+                while not stop_event.is_set():
+                    chunk = pipe.read(1024)
+                    if not chunk:
+                        # EOF reached
+                        break
+                    buffer += chunk
+
+                    # Process as long as there's control chars
+                    while True:
+                        # find next control char indices
+                        idx_n = buffer.find("\n")
+                        idx_r = buffer.find("\r")
+
+                        if idx_n == -1 and idx_r == -1:
+                            break
+
+                        # Which control comes first?
+                        if idx_r != -1 and (idx_n == -1 or idx_r < idx_n):
+                            # Carriage return: replace current line
+                            line = buffer[:idx_r]
+                            buffer = buffer[idx_r + 1:]
+                            # Only log if changed (avoids spamming identical updates)
+                            if line != last_rendered:
+                                # strip trailing CR/LF but preserve internal whitespace
+                                to_log = line.rstrip("\r\n")
+                                try:
+                                    log_func(to_log)
+                                except Exception:
+                                    # logging should not raise to user
+                                    pass
+                                if collector is not None:
+                                    collector.append(line + ("\n" if collector is not None else ""))
+                                last_rendered = line
+                        else:
+                            # Newline: finalize this line
+                            line = buffer[:idx_n]
+                            buffer = buffer[idx_n + 1:]
+                            to_log = line.rstrip("\r\n")
+                            try:
+                                log_func(to_log)
+                            except Exception:
+                                pass
+                            if collector is not None:
+                                collector.append(line + "\n")
+                            last_rendered = None
+
+                # Flush whatever remains in buffer
+                if buffer:
+                    buf_strip = buffer.rstrip("\r\n")
+                    if buf_strip:
+                        try:
+                            log_func(buf_strip)
+                        except Exception:
+                            pass
+                        if collector is not None:
+                            collector.append(buffer)
+                # close pipe
+                try:
+                    pipe.close()
+                except Exception:
+                    pass
+            except Exception as exc:
+                # Ensure we don't crash the thread; bubble up via logging
+                logger.exception(f"stream reader error: {exc}")
+                try:
+                    pipe.close()
+                except Exception:
+                    pass
+                # re-raise to let outer context know (we'll catch in caller via threads status)
+                raise
+
+        rc = None
+        if stream and (proc.stdout is not None or proc.stderr is not None):
             logger.info("Streaming command output...")
-            
-            # Create file descriptors for reading stdout and stderr
-            import select
-            import sys
-            
-            # Read from stdout and stderr simultaneously
-            while True:
-                # Check if process has terminated
-                if proc.poll() is not None:
-                    break
-                
-                # Use select to wait for data
-                read_fds = []
+            stop_event = threading.Event()
+            threads: List[threading.Thread] = []
+            stream_exception = None
+
+            # Start threads
+            try:
                 if proc.stdout:
-                    read_fds.append(proc.stdout)
+                    t_out = threading.Thread(
+                        target=_smart_reader,
+                        args=(proc.stdout, logger.info, stdout_lines if capture else None, stop_event),
+                        daemon=True
+                    )
+                    t_out.start()
+                    threads.append(t_out)
+
                 if proc.stderr:
-                    read_fds.append(proc.stderr)
-                
-                if not read_fds:
-                    break
-                
-                rlist, _, _ = select.select(read_fds, [], [], 0.1)
-                
-                for fd in rlist:
-                    if fd is proc.stdout:
-                        line = proc.stdout.readline()
-                        if line:
-                            logger.info(line.rstrip())
-                            stdout_lines.append(line)
-                    elif fd is proc.stderr:
-                        line = proc.stderr.readline()
-                        if line:
-                            logger.warning(line.rstrip())
-                            stderr_lines.append(line)
-            
-            # Read any remaining output
-            if proc.stdout:
-                for line in proc.stdout:
-                    logger.info(line.rstrip())
-                    stdout_lines.append(line)
-            if proc.stderr:
-                for line in proc.stderr:
-                    logger.warning(line.rstrip())
-                    stderr_lines.append(line)
-            
-            stdout = "".join(stdout_lines)
-            stderr = "".join(stderr_lines)
-            rc = proc.wait()
-            
+                    t_err = threading.Thread(
+                        target=_smart_reader,
+                        args=(proc.stderr, logger.warning, stderr_lines if capture else None, stop_event),
+                        daemon=True
+                    )
+                    t_err.start()
+                    threads.append(t_err)
+
+                # Wait for process while allowing KeyboardInterrupt
+                try:
+                    rc = proc.wait()
+                except KeyboardInterrupt:
+                    logger.debug("KeyboardInterrupt caught: terminating child process")
+                    stop_event.set()
+                    try:
+                        proc.terminate()
+                    except Exception:
+                        pass
+                    # wait a bit then force kill
+                    try:
+                        proc.wait(timeout=2)
+                    except Exception:
+                        try:
+                            proc.kill()
+                        except Exception:
+                            pass
+                    rc = proc.returncode if proc.returncode is not None else -1
+
+            except Exception as exc:
+                # If any exception occurs while starting/monitoring threads -> fallback
+                stream_exception = exc
+                logger.exception(f"Streaming failed, falling back to communicate(): {exc}")
+            finally:
+                # Signal threads to stop and join
+                stop_event.set()
+                for t in threads:
+                    t.join(timeout=1)
+
+            # If streaming had exception, fallback to communicate to collect remaining output safely.
+            if stream_exception is not None:
+                try:
+                    # communicate will return remaining output that reader threads didn't capture
+                    comm_out, comm_err = proc.communicate(timeout=5)
+                except Exception:
+                    try:
+                        # best-effort: kill and read whatever
+                        proc.kill()
+                    except Exception:
+                        pass
+                    try:
+                        comm_out, comm_err = proc.communicate(timeout=5)
+                    except Exception:
+                        comm_out, comm_err = ("", "")
+                # append remaining to collectors if capture True
+                if capture:
+                    if comm_out:
+                        stdout_lines.append(comm_out)
+                    if comm_err:
+                        stderr_lines.append(comm_err)
+                # set rc if not set
+                if rc is None:
+                    rc = proc.returncode if proc.returncode is not None else 0
+
+            # Compose final stdout/stderr
+            stdout = "".join(stdout_lines) if capture else ""
+            stderr = "".join(stderr_lines) if capture else ""
+            if rc is None:
+                rc = proc.returncode if proc.returncode is not None else 0
+
         else:
-            # Original behavior: wait for completion
-            stdout, stderr = proc.communicate()
+            # original behavior: blocking wait & capture (or not)
+            try:
+                stdout, stderr = proc.communicate()
+            except Exception as exc:
+                # if communicate fails, try to kill and fallback
+                logger.exception(f"proc.communicate() failed: {exc}")
+                try:
+                    proc.kill()
+                except Exception:
+                    pass
+                try:
+                    stdout, stderr = proc.communicate(timeout=5)
+                except Exception:
+                    stdout, stderr = ("", "")
             rc = proc.returncode
-        
+
+        # Build CompletedProcess result
         result = subprocess.CompletedProcess(
             use_list if not shell else cmd_str,
             rc,
             stdout=stdout or "",
             stderr=stderr or "",
         )
-        
+
+        # Post-run handling & sudo auth checks (Unix)
         if rc == 0:
             logger.info(f"Command succeeded (rc={rc})")
         else:
-            # If elevated on Unix and sudo-auth failure detected, clear cache and raise helpful error
-            if elevated and not is_windows():
+            if elevated and not is_win:
                 lowerr = (stderr or "").lower()
-                # Common sudo auth failure tokens
                 auth_tokens = [
                     "incorrect password",
                     "authentication failure",
@@ -454,25 +599,26 @@ def run(
                     "pam_authenticate",
                 ]
                 if any(tok in lowerr for tok in auth_tokens):
-                    # Clear cached password so user isn't forced to keep using wrong one
-                    clear_sudo_password()
-                    logger.error("sudo authentication failed. Cached sudo password cleared.")
-                    # Raise with stderr included for context
+                    try:
+                        clear_sudo_password()
+                    except NameError:
+                        logger.error("sudo authentication failed and clear_sudo_password() not implemented.")
+                    else:
+                        logger.error("sudo authentication failed. Cached sudo password cleared.")
                     raise subprocess.CalledProcessError(rc, use_list if not shell else cmd_str, output=stdout, stderr=stderr)
-            # General logging for failure
-            logger.error(f"Command failed (rc={rc}) – {stderr.strip() if stderr else ''}")
-        
+            logger.error(f"Command failed (rc={rc}) – { (stderr or '').strip() }")
+
         if rc != 0 and not no_die:
             raise subprocess.CalledProcessError(rc, use_list if not shell else cmd_str, output=stdout, stderr=stderr)
-        
+
         return result
-    
+
     except subprocess.CalledProcessError:
-        # Re-raise CalledProcessError so callers can handle; do not clear cached password here
         raise
     except Exception as e:
         logger.exception(f"Unexpected error running command: {e}")
         raise
+
 # -------------------------------------------------
 # Exec helper – shows command + output + logs
 # -------------------------------------------------

{utils_devops-0.1.129 → utils_devops-0.1.136}/src/utils_devops/extras/docker_ops.py

@@ -430,8 +430,7 @@ def read_compose_file(compose_file: str, env_file: Optional[str] = None) -> Dict
             envs.import_env_to_system(get_env_compose(env_file))
         
         # Read compose file content
-        #content = files.read_file(compose_file)
-        content = strings.parse_yaml(compose_file)
+        content = files.read_file(compose_file)
         
         # Expand environment variables in the content
         data = _expand_env_vars_in_compose(content)

{utils_devops-0.1.129 → utils_devops-0.1.136}/src/utils_devops/extras/ssh_ops.py

@@ -340,18 +340,35 @@ def ssh_connect(
         key_path, key_content = _resolve_ssh_key(key_file)
         
         if key_content:
+            # **FIXED HERE**: Ensure proper line endings in the key content
+            # SSH keys MUST have newlines at specific positions
+            key_content = _normalize_ssh_key_content(key_content)
+            
             # Create temporary file for key content
             import tempfile
             temp_key_file = tempfile.NamedTemporaryFile(
-                mode='w',
+                mode='w',  # Text mode for proper newline handling
                 suffix='_ssh_key',
-                delete=False
+                delete=False,
+                newline='\n'  # Explicitly use LF newlines
             )
+            
+            # **CRITICAL FIX**: Write with proper line endings
             temp_key_file.write(key_content)
+            temp_key_file.flush()  # Ensure data is written
+            os.fsync(temp_key_file.fileno())  # Force sync to disk
             temp_key_file.close()
             
-            # Set permissions
+            # Set strict permissions for SSH keys
             os.chmod(temp_key_file.name, 0o600)
+            
+            # **DEBUG**: Verify what was written
+            with open(temp_key_file.name, 'r') as f:
+                written_content = f.read()
+                _logger.debug(f"Written key length: {len(written_content)}")
+                _logger.debug(f"Key first 100 chars: {written_content[:100]}")
+                _logger.debug(f"Key lines: {written_content.count(chr(10))}")
+            
             connect_kwargs['key_filename'] = temp_key_file.name
             _logger.debug(f"Using temporary SSH key file: {temp_key_file.name}")
             
@@ -390,8 +407,8 @@ def ssh_connect(
             try:
                 os.unlink(temp_key_file.name)
                 _logger.debug(f"Cleaned up temporary key file: {temp_key_file.name}")
-            except:
-                pass
+            except Exception as e:
+                _logger.warning(f"Failed to cleanup temp key file: {e}")
         
         if client:
             try:
@@ -400,6 +417,117 @@ def ssh_connect(
             except:
                 pass
 
+
+def _normalize_ssh_key_content(key_content: str) -> str:
+    """
+    Normalize SSH key content to ensure proper formatting.
+    
+    SSH keys require specific line breaks:
+    - Must start with -----BEGIN ...-----
+    - Base64 content should be on separate lines (64 chars max per line)
+    - Must end with -----END ...-----
+    
+    Args:
+        key_content: Raw key content (may be single line or malformed)
+    
+    Returns:
+        Properly formatted SSH key
+    """
+    # If already looks properly formatted, return as-is
+    if "-----BEGIN" in key_content and "-----END" in key_content and '\n' in key_content:
+        # Verify line endings
+        lines = key_content.splitlines()
+        if len(lines) > 1:
+            return key_content
+    
+    # Parse and reformat
+    lines = []
+    current_line = []
+    
+    # Remove all existing whitespace and split
+    clean_content = key_content.strip()
+    
+    if clean_content.startswith("-----BEGIN"):
+        # Extract header
+        header_end = clean_content.find("-----", 10) + 5
+        header = clean_content[:header_end]
+        lines.append(header)
+        
+        # Get base64 content between headers
+        base64_start = header_end
+        base64_end = clean_content.find("-----END")
+        base64_content = clean_content[base64_start:base64_end].strip()
+        
+        # Remove all whitespace from base64
+        base64_clean = ''.join(base64_content.split())
+        
+        # Split into 64 character lines (standard for SSH keys)
+        for i in range(0, len(base64_clean), 64):
+            lines.append(base64_clean[i:i+64])
+        
+        # Add footer
+        footer_start = base64_end
+        lines.append(clean_content[footer_start:].strip())
+    else:
+        # Assume it's just base64, wrap in OpenSSH header/footer
+        lines.append("-----BEGIN OPENSSH PRIVATE KEY-----")
+        clean_base64 = ''.join(clean_content.split())
+        for i in range(0, len(clean_base64), 64):
+            lines.append(clean_base64[i:i+64])
+        lines.append("-----END OPENSSH PRIVATE KEY-----")
+    
+    return '\n'.join(lines) + '\n'
+
+
+# Also update your _resolve_ssh_key function to preserve newlines:
+def _resolve_ssh_key(key_input: Optional[Union[str, Path]]) -> Tuple[Optional[Path], Optional[str]]:
+    """
+    Resolve SSH key input to either a file path or key content.
+    
+    Args:
+        key_input: Could be:
+            - Path to key file
+            - Environment variable name ($VAR)
+            - Raw key content string
+    
+    Returns:
+        Tuple of (file_path, key_content)
+    """
+    if not key_input:
+        return None, None
+    
+    # If it's a Path object
+    if isinstance(key_input, Path):
+        if key_input.exists():
+            return key_input, None
+        else:
+            raise FileNotFoundError(f"SSH key file not found: {key_input}")
+    
+    key_input_str = str(key_input)
+    
+    # Check if it's an environment variable reference
+    if key_input_str.startswith('$'):
+        env_var = key_input_str[1:]
+        key_content = os.environ.get(env_var)
+        if not key_content:
+            raise ValueError(f"Environment variable not set: {env_var}")
+        
+        # **CRITICAL**: Don't strip newlines from env var content!
+        # SSH keys in env vars often have escaped newlines (\n)
+        # Replace literal \n with actual newlines
+        if '\\n' in key_content:
+            key_content = key_content.replace('\\n', '\n')
+        
+        return None, key_content
+    
+    # Check if it's a file path
+    if os.path.exists(key_input_str):
+        return Path(key_input_str), None
+    
+    # Assume it's raw key content
+    # **IMPORTANT**: Preserve any newlines in the content
+    return None, key_input_str
+
 def ssh_execute_command(
     host: str,
     command: Union[str, List[str]],
@@ -1465,10 +1593,10 @@ def validate_machine_config(machine: MachineConfig) -> List[str]:
             errors.append(f"Invalid deploy_dir format: {e}")
     
     # Validate key file if specified
-    if machine.key_file:
-        key_path = Path(machine.key_file).expanduser()
-        if not key_path.exists():
-            errors.append(f"SSH key file not found: {key_path}")
+    # if machine.key_file:
+    #     key_path = Path(machine.key_file).expanduser()
+    #     if not key_path.exists():
+    #         errors.append(f"SSH key file not found: {key_path}")
     
     return errors