machineconfig 1.95__py3-none-any.whl → 1.97__py3-none-any.whl

machineconfig/cluster/sessions_managers/zellij_utils/process_monitor.py

@@ -0,0 +1,334 @@
+#!/usr/bin/env python3
+"""
+Process monitoring and status checking utilities for remote commands.
+"""
+import json
+import shlex
+import logging
+from typing import Dict, Tuple, Any
+from .remote_executor import RemoteExecutor
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessMonitor:
+    """Handles process status checking and verification on remote machines."""
+    
+    def __init__(self, remote_executor: RemoteExecutor):
+        self.remote_executor = remote_executor
+    
+    def check_command_status(self, tab_name: str, tab_config: Dict[str, Tuple[str, str]], 
+                           use_verification: bool = True) -> Dict[str, Any]:
+        """Check command status with optional process verification."""
+        if tab_name not in tab_config:
+            return {
+                "status": "unknown",
+                "error": f"Tab '{tab_name}' not found in tracked configuration",
+                "running": False,
+                "pid": None,
+                "command": None,
+                "remote": self.remote_executor.remote_name
+            }
+        
+        # Use the verified method by default for more accurate results
+        if use_verification:
+            return self.get_verified_process_status(tab_name, tab_config)
+        
+        return self._basic_process_check(tab_name, tab_config)
+    
+    def _basic_process_check(self, tab_name: str, tab_config: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        """Basic process checking without verification."""
+        _, command = tab_config[tab_name]
+        
+        try:
+            check_script = self._create_process_check_script(command)
+            remote_cmd = f"$HOME/venvs/ve/bin/python -c {shlex.quote(check_script)}"
+            result = self.remote_executor.run_command(remote_cmd, timeout=15)
+            
+            if result.returncode == 0:
+                try:
+                    matching_processes = json.loads(result.stdout.strip())
+                    
+                    if matching_processes:
+                        return {
+                            "status": "running",
+                            "running": True,
+                            "processes": matching_processes,
+                            "command": command,
+                            "tab_name": tab_name,
+                            "remote": self.remote_executor.remote_name
+                        }
+                    else:
+                        return {
+                            "status": "not_running",
+                            "running": False,
+                            "processes": [],
+                            "command": command,
+                            "tab_name": tab_name,
+                            "remote": self.remote_executor.remote_name
+                        }
+                except json.JSONDecodeError as e:
+                    logger.error(f"Failed to parse remote process check output: {e}")
+                    return {
+                        "status": "error",
+                        "error": f"Failed to parse remote output: {e}",
+                        "running": False,
+                        "command": command,
+                        "tab_name": tab_name,
+                        "remote": self.remote_executor.remote_name
+                    }
+            else:
+                return {
+                    "status": "error", 
+                    "error": f"Remote command failed: {result.stderr}",
+                    "running": False,
+                    "command": command,
+                    "tab_name": tab_name,
+                    "remote": self.remote_executor.remote_name
+                }
+                
+        except Exception as e:
+            logger.error(f"Error checking command status for tab '{tab_name}': {e}")
+            return {
+                "status": "error",
+                "error": str(e),
+                "running": False,
+                "command": command,
+                "tab_name": tab_name,
+                "remote": self.remote_executor.remote_name
+            }
+    
+    def _create_process_check_script(self, command: str) -> str:
+        """Create Python script for checking processes on remote machine."""
+        return f"""
+import psutil
+import json
+import os
+
+def check_process():
+    matching_processes = []
+    full_command = '{command}'
+    cmd_parts = [part for part in full_command.split() if len(part) > 2]
+    current_pid = os.getpid()
+    
+    primary_cmd = cmd_parts[0] if cmd_parts else ''
+    
+    for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'status', 'create_time']):
+        try:
+            if proc.info['pid'] == current_pid:
+                continue
+                
+            if proc.info['cmdline'] and len(proc.info['cmdline']) > 0:
+                cmdline_str = ' '.join(proc.info['cmdline'])
+                
+                if 'check_process()' in cmdline_str or 'psutil.process_iter' in cmdline_str:
+                    continue
+                
+                matches_primary = primary_cmd in cmdline_str
+                matches_parts = sum(1 for part in cmd_parts[1:] if part in cmdline_str)
+                
+                if (matches_primary and matches_parts >= 2) or \\
+                   (full_command in cmdline_str and not any(python_indicator in cmdline_str.lower() 
+                                                           for python_indicator in ['python -c', 'import psutil', 'def check_process'])):
+                    matching_processes.append({{
+                        "pid": proc.info['pid'],
+                        "name": proc.info['name'],
+                        "cmdline": proc.info['cmdline'],
+                        "status": proc.info['status'],
+                        "cmdline_str": cmdline_str,
+                        "create_time": proc.info['create_time']
+                    }})
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            continue
+    
+    return matching_processes
+
+if __name__ == "__main__":
+    processes = check_process()
+    print(json.dumps(processes))
+"""
+    
+    def force_fresh_process_check(self, tab_name: str, tab_config: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        """Force a fresh process check with additional validation."""
+        if tab_name not in tab_config:
+            return {
+                "status": "unknown",
+                "error": f"Tab '{tab_name}' not found in tracked configuration",
+                "running": False,
+                "command": None,
+                "remote": self.remote_executor.remote_name
+            }
+        
+        _, command = tab_config[tab_name]
+        
+        try:
+            # Get timestamp for freshness validation
+            timestamp_result = self.remote_executor.run_command("date +%s", timeout=5)
+            check_timestamp = timestamp_result.stdout.strip() if timestamp_result.returncode == 0 else "unknown"
+            
+            check_script = self._create_fresh_check_script(command)
+            remote_cmd = f"$HOME/venvs/ve/bin/python -c {shlex.quote(check_script)}"
+            result = self.remote_executor.run_command(remote_cmd, timeout=15)
+            
+            if result.returncode == 0:
+                try:
+                    check_result = json.loads(result.stdout.strip())
+                    matching_processes = check_result.get("processes", [])
+                    
+                    return {
+                        "status": "running" if matching_processes else "not_running",
+                        "running": bool(matching_processes),
+                        "processes": matching_processes,
+                        "command": command,
+                        "tab_name": tab_name,
+                        "remote": self.remote_executor.remote_name,
+                        "check_timestamp": check_timestamp,
+                        "method": "force_fresh_check"
+                    }
+                except json.JSONDecodeError as e:
+                    logger.error(f"Failed to parse fresh check output: {e}")
+                    return {
+                        "status": "error",
+                        "error": f"Failed to parse output: {e}",
+                        "running": False,
+                        "command": command,
+                        "tab_name": tab_name,
+                        "remote": self.remote_executor.remote_name,
+                        "raw_output": result.stdout
+                    }
+            else:
+                return {
+                    "status": "error", 
+                    "error": f"Remote command failed: {result.stderr}",
+                    "running": False,
+                    "command": command,
+                    "tab_name": tab_name,
+                    "remote": self.remote_executor.remote_name
+                }
+                
+        except Exception as e:
+            logger.error(f"Error in fresh process check for tab '{tab_name}': {e}")
+            return {
+                "status": "error",
+                "error": str(e),
+                "running": False,
+                "command": command,
+                "tab_name": tab_name,
+                "remote": self.remote_executor.remote_name
+            }
+    
+    def _create_fresh_check_script(self, command: str) -> str:
+        """Create enhanced process checking script with freshness validation."""
+        escaped_command = command.replace("'", "\\'").replace('"', '\\"')
+        
+        return f'''
+import psutil
+import json
+import os
+import time
+
+def force_fresh_check():
+    time.sleep(0.1)
+    
+    matching_processes = []
+    full_command = '{escaped_command}'
+    cmd_parts = [part for part in full_command.split() if len(part) > 2]
+    current_pid = os.getpid()
+    primary_cmd = cmd_parts[0] if cmd_parts else ''
+    
+    check_time = time.time()
+    
+    for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'status', 'create_time']):
+        try:
+            if proc.info['pid'] == current_pid:
+                continue
+                
+            if proc.info['cmdline'] and len(proc.info['cmdline']) > 0:
+                cmdline_str = ' '.join(proc.info['cmdline'])
+                
+                if any(indicator in cmdline_str for indicator in [
+                    'check_process()', 'psutil.process_iter', 'force_fresh_check',
+                    'import psutil', 'def check_process'
+                ]):
+                    continue
+                
+                if proc.info['create_time'] and proc.info['create_time'] > check_time - 5:
+                    continue
+                
+                matches_primary = primary_cmd in cmdline_str and primary_cmd != 'python'
+                matches_parts = sum(1 for part in cmd_parts[1:] if part in cmdline_str)
+                
+                if matches_primary and matches_parts >= 2:
+                    script_indicators = ['-c', 'import ', 'def ', 'psutil']
+                    is_direct_command = not any(script_indicator in cmdline_str.lower() 
+                                              for script_indicator in script_indicators)
+                    
+                    if is_direct_command or (full_command in cmdline_str and 'python -c' not in cmdline_str):
+                        matching_processes.append({{
+                            "pid": proc.info['pid'],
+                            "name": proc.info['name'],
+                            "cmdline": proc.info['cmdline'],
+                            "status": proc.info['status'],
+                            "cmdline_str": cmdline_str,
+                            "create_time": proc.info['create_time'],
+                            "is_direct_command": is_direct_command
+                        }})
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            continue
+    
+    return {{
+        "processes": matching_processes,
+        "check_timestamp": check_time,
+        "search_command": full_command,
+        "search_parts": cmd_parts
+    }}
+
+if __name__ == "__main__":
+    result = force_fresh_check()
+    print(json.dumps(result))
+'''
+    
+    def verify_process_alive(self, pid: int) -> bool:
+        """Verify if a process with given PID is actually alive."""
+        try:
+            verify_cmd = f"kill -0 {pid} 2>/dev/null && echo 'alive' || echo 'dead'"
+            result = self.remote_executor.run_command(verify_cmd, timeout=5)
+            
+            if result.returncode == 0:
+                return result.stdout.strip() == 'alive'
+            return False
+        except Exception:
+            return False
+    
+    def get_verified_process_status(self, tab_name: str, tab_config: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        """Get process status with additional verification that processes are actually alive."""
+        status = self.force_fresh_process_check(tab_name, tab_config)
+        
+        if status.get("running") and status.get("processes"):
+            verified_processes = []
+            for proc in status["processes"]:
+                pid = proc.get("pid")
+                if pid and self.verify_process_alive(pid):
+                    proc["verified_alive"] = True
+                    verified_processes.append(proc)
+                else:
+                    proc["verified_alive"] = False
+                    logger.warning(f"Process PID {pid} found in process list but not actually alive")
+            
+            status["processes"] = verified_processes
+            status["running"] = bool(verified_processes)
+            status["status"] = "running" if verified_processes else "not_running"
+            status["verification_method"] = "kill_signal_check"
+        
+        return status
+    
+    def check_all_commands_status(self, tab_config: Dict[str, Tuple[str, str]]) -> Dict[str, Dict[str, Any]]:
+        """Check status of all commands in the tab configuration."""
+        if not tab_config:
+            logger.warning("No tab configuration provided.")
+            return {}
+        
+        status_report = {}
+        for tab_name in tab_config:
+            status_report[tab_name] = self.check_command_status(tab_name, tab_config)
+        return status_report

machineconfig/cluster/sessions_managers/zellij_utils/remote_executor.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""
+Remote command execution utilities for SSH operations.
+"""
+import subprocess
+import logging
+from typing import Dict, Any
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteExecutor:
+    """Handles SSH command execution on remote machines."""
+    
+    def __init__(self, remote_name: str):
+        self.remote_name = remote_name
+    
+    def run_command(self, command: str, timeout: int = 30) -> subprocess.CompletedProcess[str]:
+        """Execute a command on the remote machine via SSH."""
+        ssh_cmd = ["ssh", self.remote_name, command]
+        try:
+            result = subprocess.run(
+                ssh_cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout
+            )
+            return result
+        except subprocess.TimeoutExpired:
+            logger.error(f"SSH command timed out after {timeout}s: {command}")
+            raise
+        except Exception as e:
+            logger.error(f"SSH command failed: {e}")
+            raise
+    
+    def copy_file_to_remote(self, local_file: str, remote_path: str) -> Dict[str, Any]:
+        """Copy a file to the remote machine using SCP."""
+        scp_cmd = ["scp", local_file, f"{self.remote_name}:{remote_path}"]
+        try:
+            result = subprocess.run(scp_cmd, capture_output=True, text=True)
+            if result.returncode == 0:
+                logger.info(f"File copied successfully to {self.remote_name}:{remote_path}")
+                return {"success": True, "remote_path": remote_path}
+            else:
+                logger.error(f"SCP failed: {result.stderr}")
+                return {"success": False, "error": result.stderr}
+        except Exception as e:
+            logger.error(f"SCP operation failed: {e}")
+            return {"success": False, "error": str(e)}
+    
+    def create_remote_directory(self, remote_dir: str) -> bool:
+        """Create a directory on the remote machine."""
+        try:
+            result = self.run_command(f"mkdir -p {remote_dir}")
+            return result.returncode == 0
+        except Exception as e:
+            logger.error(f"Failed to create remote directory {remote_dir}: {e}")
+            return False
+    
+    def attach_to_session_interactive(self, session_name: str) -> None:
+        """Attach to a Zellij session interactively via SSH."""
+        try:
+            attach_cmd = ["ssh", "-t", self.remote_name, f"zellij attach {session_name}"]
+            logger.info(f"Attaching to Zellij session '{session_name}' on {self.remote_name}")
+            subprocess.run(attach_cmd)
+        except Exception as e:
+            logger.error(f"Failed to attach to session: {e}")
+            raise

machineconfig/cluster/sessions_managers/zellij_utils/session_manager.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""
+Zellij session management utilities for remote operations.
+"""
+import logging
+from typing import Dict, Any, Optional
+from pathlib import Path
+
+from rich.console import Console
+
+from .remote_executor import RemoteExecutor
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+class SessionManager:
+    """Handles Zellij session operations on remote machines."""
+    
+    def __init__(self, remote_executor: RemoteExecutor, session_name: str, tmp_layout_dir: Path):
+        self.remote_executor = remote_executor
+        self.session_name = session_name
+        self.tmp_layout_dir = tmp_layout_dir
+    
+    def copy_layout_to_remote(self, local_layout_file: Path, random_suffix: str) -> str:
+        """Copy the layout file to the remote machine and return the remote path."""
+        remote_layout_dir = f"~/{self.tmp_layout_dir.relative_to(Path.home())}"
+        remote_layout_file = f"{remote_layout_dir}/zellij_layout_{self.session_name}_{random_suffix}.kdl"
+        
+        # Create remote directory
+        if not self.remote_executor.create_remote_directory(remote_layout_dir):
+            raise RuntimeError(f"Failed to create remote directory: {remote_layout_dir}")
+        
+        # Copy layout file to remote machine
+        copy_result = self.remote_executor.copy_file_to_remote(str(local_layout_file), remote_layout_file)
+        if not copy_result["success"]:
+            raise RuntimeError(f"Failed to copy layout file to remote: {copy_result['error']}")
+        
+        # Enhanced Rich logging
+        console.print(f"[bold green]📁 Zellij layout file copied to remote:[/bold green] [yellow]{self.remote_executor.remote_name}[/yellow][cyan]:{remote_layout_file}[/cyan]")
+        return remote_layout_file
+    
+    def check_zellij_session_status(self) -> Dict[str, Any]:
+        """Check if the Zellij session exists and is running."""
+        try:
+            result = self.remote_executor.run_command('zellij list-sessions', timeout=10)
+            
+            if result.returncode == 0:
+                sessions = result.stdout.strip().split('\n') if result.stdout.strip() else []
+                session_running = any(self.session_name in session for session in sessions)
+                
+                return {
+                    "zellij_running": True,
+                    "session_exists": session_running,
+                    "session_name": self.session_name,
+                    "all_sessions": sessions,
+                    "remote": self.remote_executor.remote_name
+                }
+            else:
+                return {
+                    "zellij_running": False,
+                    "error": result.stderr,
+                    "session_name": self.session_name,
+                    "remote": self.remote_executor.remote_name
+                }
+                
+        except Exception as e:
+            return {
+                "zellij_running": False,
+                "error": str(e),
+                "session_name": self.session_name,
+                "remote": self.remote_executor.remote_name
+            }
+    
+    def start_zellij_session(self, layout_file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Start a Zellij session on the remote machine with the generated layout."""
+        try:
+            if layout_file_path:
+                layout_filename = Path(layout_file_path).name
+                remote_layout_file = f"~/{self.tmp_layout_dir.relative_to(Path.home())}/{layout_filename}"
+            else:
+                raise ValueError("No layout file path provided.")
+            
+            # Enhanced Rich logging for session start
+            console.print(f"[bold cyan]🚀 Starting Zellij session[/bold cyan] [yellow]'{self.session_name}'[/yellow] [dim]on remote[/dim] [bold yellow]'{self.remote_executor.remote_name}'[/bold yellow] [dim]with layout:[/dim] [blue]{remote_layout_file}[/blue]")
+            
+            # Start Zellij session with layout
+            start_cmd = f"zellij --layout {remote_layout_file} a -b {self.session_name}"
+            console.print(f"[dim]Executing:[/dim] [green]{start_cmd}[/green]")
+            result = self.remote_executor.run_command(start_cmd, timeout=30)
+            
+            if result.returncode == 0:
+                console.print(f"[bold green]✅ Zellij session[/bold green] [yellow]'{self.session_name}'[/yellow] [green]started successfully on[/green] [bold yellow]{self.remote_executor.remote_name}[/bold yellow]")
+                return {
+                    "success": True,
+                    "session_name": self.session_name,
+                    "remote": self.remote_executor.remote_name,
+                    "message": "Session started successfully"
+                }
+            else:
+                return {
+                    "success": False,
+                    "error": result.stderr,
+                    "session_name": self.session_name,
+                    "remote": self.remote_executor.remote_name
+                }
+                
+        except Exception as e:
+            logger.error(f"Failed to start Zellij session on {self.remote_executor.remote_name}: {e}")
+            return {
+                "success": False,
+                "error": str(e),
+                "session_name": self.session_name,
+                "remote": self.remote_executor.remote_name
+            }
+    
+    def attach_to_session(self) -> None:
+        """Attach to the Zellij session on the remote machine via SSH."""
+        self.remote_executor.attach_to_session_interactive(self.session_name)

machineconfig/cluster/sessions_managers/zellij_utils/status_reporter.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+Status reporting utilities for Zellij remote layouts.
+"""
+import logging
+from typing import Dict, Any, Tuple
+from .process_monitor import ProcessMonitor
+from .session_manager import SessionManager
+
+logger = logging.getLogger(__name__)
+
+
+class StatusReporter:
+    """Handles comprehensive status reporting for Zellij remote sessions."""
+    
+    def __init__(self, process_monitor: ProcessMonitor, session_manager: SessionManager):
+        self.process_monitor = process_monitor
+        self.session_manager = session_manager
+    
+    def get_comprehensive_status(self, tab_config: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        """Get comprehensive status including Zellij session and all commands."""
+        zellij_status = self.session_manager.check_zellij_session_status()
+        commands_status = self.process_monitor.check_all_commands_status(tab_config)
+        
+        running_count = sum(1 for status in commands_status.values() if status.get("running", False))
+        total_count = len(commands_status)
+        
+        return {
+            "zellij_session": zellij_status,
+            "commands": commands_status,
+            "summary": {
+                "total_commands": total_count,
+                "running_commands": running_count,
+                "stopped_commands": total_count - running_count,
+                "session_healthy": zellij_status.get("session_exists", False),
+                "remote": self.session_manager.remote_executor.remote_name
+            }
+        }
+    
+    def print_status_report(self, tab_config: Dict[str, Tuple[str, str]]) -> None:
+        """Print a formatted status report to console."""
+        status = self.get_comprehensive_status(tab_config)
+        remote_name = self.session_manager.remote_executor.remote_name
+        session_name = self.session_manager.session_name
+        
+        print("=" * 60)
+        print(f"🔍 ZELLIJ REMOTE LAYOUT STATUS REPORT ({remote_name})")
+        print("=" * 60)
+        
+        # Zellij session status
+        zellij = status["zellij_session"]
+        if zellij.get("zellij_running", False):
+            if zellij.get("session_exists", False):
+                print(f"✅ Zellij session '{session_name}' is running on {remote_name}")
+            else:
+                print(f"⚠️  Zellij is running on {remote_name} but session '{session_name}' not found")
+        else:
+            print(f"❌ Zellij session issue on {remote_name}: {zellij.get('error', 'Unknown error')}")
+        
+        print()
+        
+        # Commands status
+        print("📋 COMMAND STATUS:")
+        print("-" * 40)
+        
+        for tab_name, cmd_status in status["commands"].items():
+            if cmd_status.get("running", False):
+                print(f"✅ {tab_name}: Running on {remote_name}")
+                if cmd_status.get("processes"):
+                    for proc in cmd_status["processes"][:2]:  # Show first 2 processes
+                        print(f"   └─ PID {proc['pid']}: {proc['name']} ({proc['status']})")
+            else:
+                print(f"❌ {tab_name}: Not running on {remote_name}")
+            print(f"   Command: {cmd_status.get('command', 'Unknown')}")
+            print()
+        
+        # Summary
+        summary = status["summary"]
+        print("📊 SUMMARY:")
+        print(f"   Remote machine: {remote_name}")
+        print(f"   Total commands: {summary['total_commands']}")
+        print(f"   Running: {summary['running_commands']}")
+        print(f"   Stopped: {summary['stopped_commands']}")
+        print(f"   Session healthy: {'✅' if summary['session_healthy'] else '❌'}")
+        print("=" * 60)

machineconfig/cluster/templates/cli_click.py

@@ -5,7 +5,6 @@
 # #
 # # from machineconfig.cluster.distribute import Cluster, WorkloadParams
 # from machineconfig.cluster.remote_machine import RemoteMachineConfig, RemoteMachine, CloudManager
-# from crocodile.file_management import Read
 # from machineconfig.cluster.templates.run_cloud import to_email_default, email_config_name_default, default_cloud
 # import click
 # # from trogon import tui