mcp-ticketer 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

mcp_ticketer/queue/health_monitor.py

@@ -1,12 +1,10 @@
 """Queue health monitoring and alerting system."""
 
-import asyncio
 import logging
 import time
 from datetime import datetime, timedelta
 from enum import Enum
-from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 import psutil
 
@@ -18,6 +16,7 @@ logger = logging.getLogger(__name__)
 
 class HealthStatus(str, Enum):
     """Health status levels."""
+
     HEALTHY = "healthy"
     WARNING = "warning"
     CRITICAL = "critical"
@@ -26,282 +25,312 @@ class HealthStatus(str, Enum):
 
 class HealthAlert:
     """Health alert with severity and details."""
-    
+
     def __init__(
         self,
         level: HealthStatus,
         message: str,
-        details: Optional[Dict[str, Any]] = None,
-        timestamp: Optional[datetime] = None
+        details: Optional[dict[str, Any]] = None,
+        timestamp: Optional[datetime] = None,
     ):
         self.level = level
         self.message = message
         self.details = details or {}
         self.timestamp = timestamp or datetime.now()
-    
+
     def __str__(self) -> str:
         return f"[{self.level.upper()}] {self.message}"
 
 
 class QueueHealthMonitor:
     """Monitors queue health and provides immediate alerts."""
-    
+
     # Health check thresholds
     WORKER_TIMEOUT_SECONDS = 30  # Worker should process items within 30s
-    STUCK_ITEM_THRESHOLD = 300   # 5 minutes for stuck items
-    HIGH_FAILURE_RATE = 0.3      # 30% failure rate is concerning
-    QUEUE_BACKLOG_WARNING = 10   # Warn if more than 10 pending items
+    STUCK_ITEM_THRESHOLD = 300  # 5 minutes for stuck items
+    HIGH_FAILURE_RATE = 0.3  # 30% failure rate is concerning
+    QUEUE_BACKLOG_WARNING = 10  # Warn if more than 10 pending items
     QUEUE_BACKLOG_CRITICAL = 50  # Critical if more than 50 pending items
-    
+
     def __init__(self, queue: Optional[Queue] = None):
         """Initialize health monitor.
-        
+
         Args:
             queue: Queue instance to monitor. Creates new if None.
+
         """
         self.queue = queue or Queue()
         self.manager = WorkerManager()
         self.last_check = datetime.now()
-        self.alerts: List[HealthAlert] = []
-        
-    def check_health(self) -> Dict[str, Any]:
+        self.alerts: list[HealthAlert] = []
+
+    def check_health(self) -> dict[str, Any]:
         """Perform comprehensive health check.
-        
+
         Returns:
             Health status with alerts and metrics
+
         """
         self.alerts.clear()
-        
+
         # Check worker status
         worker_health = self._check_worker_health()
-        
+
         # Check queue status
         queue_health = self._check_queue_health()
-        
+
         # Check for stuck items
         stuck_health = self._check_stuck_items()
-        
+
         # Check failure rates
         failure_health = self._check_failure_rates()
-        
+
         # Determine overall health
         overall_status = self._determine_overall_status()
-        
+
         health_report = {
             "status": overall_status,
             "timestamp": datetime.now().isoformat(),
-            "alerts": [{"level": alert.level, "message": alert.message, "details": alert.details} for alert in self.alerts],
+            "alerts": [
+                {
+                    "level": alert.level,
+                    "message": alert.message,
+                    "details": alert.details,
+                }
+                for alert in self.alerts
+            ],
             "metrics": {
                 "worker": worker_health,
                 "queue": queue_health,
                 "stuck_items": stuck_health,
-                "failure_rate": failure_health
-            }
+                "failure_rate": failure_health,
+            },
         }
-        
+
         self.last_check = datetime.now()
         return health_report
-    
-    def _check_worker_health(self) -> Dict[str, Any]:
+
+    def _check_worker_health(self) -> dict[str, Any]:
         """Check worker process health."""
         worker_status = self.manager.get_status()
-        
+
         metrics = {
             "running": worker_status["running"],
             "pid": worker_status.get("pid"),
             "cpu_percent": worker_status.get("cpu_percent", 0),
-            "memory_mb": worker_status.get("memory_mb", 0)
+            "memory_mb": worker_status.get("memory_mb", 0),
         }
-        
+
         if not worker_status["running"]:
             # Check if we have pending items but no worker
             pending_count = self.queue.get_pending_count()
             if pending_count > 0:
-                self.alerts.append(HealthAlert(
-                    HealthStatus.CRITICAL,
-                    f"Worker not running but {pending_count} items pending",
-                    {"pending_count": pending_count, "action": "start_worker"}
-                ))
+                self.alerts.append(
+                    HealthAlert(
+                        HealthStatus.CRITICAL,
+                        f"Worker not running but {pending_count} items pending",
+                        {"pending_count": pending_count, "action": "start_worker"},
+                    )
+                )
             else:
-                self.alerts.append(HealthAlert(
-                    HealthStatus.WARNING,
-                    "Worker not running (no pending items)",
-                    {"action": "worker_idle"}
-                ))
+                self.alerts.append(
+                    HealthAlert(
+                        HealthStatus.WARNING,
+                        "Worker not running (no pending items)",
+                        {"action": "worker_idle"},
+                    )
+                )
         else:
             # Worker is running, check if it's responsive
             pid = worker_status.get("pid")
             if pid:
                 try:
-                    process = psutil.Process(pid)
+                    psutil.Process(pid)
                     # Check if worker has been idle too long with pending items
                     pending_count = self.queue.get_pending_count()
                     if pending_count > 0:
                         # Check for items that have been pending too long
                         old_pending = self._get_old_pending_items()
                         if old_pending:
-                            self.alerts.append(HealthAlert(
-                                HealthStatus.WARNING,
-                                f"Worker running but {len(old_pending)} items pending for >30s",
-                                {"old_pending_count": len(old_pending), "worker_pid": pid}
-                            ))
+                            self.alerts.append(
+                                HealthAlert(
+                                    HealthStatus.WARNING,
+                                    f"Worker running but {len(old_pending)} items pending for >30s",
+                                    {
+                                        "old_pending_count": len(old_pending),
+                                        "worker_pid": pid,
+                                    },
+                                )
+                            )
                 except (psutil.NoSuchProcess, psutil.AccessDenied):
-                    self.alerts.append(HealthAlert(
-                        HealthStatus.CRITICAL,
-                        "Worker PID exists but process not accessible",
-                        {"pid": pid, "action": "restart_worker"}
-                    ))
-        
+                    self.alerts.append(
+                        HealthAlert(
+                            HealthStatus.CRITICAL,
+                            "Worker PID exists but process not accessible",
+                            {"pid": pid, "action": "restart_worker"},
+                        )
+                    )
+
         return metrics
-    
-    def _check_queue_health(self) -> Dict[str, Any]:
+
+    def _check_queue_health(self) -> dict[str, Any]:
         """Check queue status and backlog."""
         stats = self.queue.get_stats()
-        
+
         pending = stats.get("pending", 0)
         processing = stats.get("processing", 0)
         failed = stats.get("failed", 0)
         completed = stats.get("completed", 0)
-        
+
         metrics = {
             "pending": pending,
             "processing": processing,
             "failed": failed,
             "completed": completed,
-            "total": pending + processing + failed + completed
+            "total": pending + processing + failed + completed,
         }
-        
+
         # Check backlog levels
         if pending >= self.QUEUE_BACKLOG_CRITICAL:
-            self.alerts.append(HealthAlert(
-                HealthStatus.CRITICAL,
-                f"Critical queue backlog: {pending} pending items",
-                {"pending_count": pending, "action": "scale_workers"}
-            ))
+            self.alerts.append(
+                HealthAlert(
+                    HealthStatus.CRITICAL,
+                    f"Critical queue backlog: {pending} pending items",
+                    {"pending_count": pending, "action": "scale_workers"},
+                )
+            )
         elif pending >= self.QUEUE_BACKLOG_WARNING:
-            self.alerts.append(HealthAlert(
-                HealthStatus.WARNING,
-                f"High queue backlog: {pending} pending items",
-                {"pending_count": pending}
-            ))
-        
+            self.alerts.append(
+                HealthAlert(
+                    HealthStatus.WARNING,
+                    f"High queue backlog: {pending} pending items",
+                    {"pending_count": pending},
+                )
+            )
+
         # Check for too many processing items (might indicate stuck workers)
         if processing > 5:  # Should rarely have more than a few processing
-            self.alerts.append(HealthAlert(
-                HealthStatus.WARNING,
-                f"Many items in processing state: {processing}",
-                {"processing_count": processing, "action": "check_stuck_items"}
-            ))
-        
+            self.alerts.append(
+                HealthAlert(
+                    HealthStatus.WARNING,
+                    f"Many items in processing state: {processing}",
+                    {"processing_count": processing, "action": "check_stuck_items"},
+                )
+            )
+
         return metrics
-    
-    def _check_stuck_items(self) -> Dict[str, Any]:
+
+    def _check_stuck_items(self) -> dict[str, Any]:
         """Check for items stuck in processing state."""
         # Reset stuck items first
         self.queue.reset_stuck_items(timeout_minutes=5)  # 5 minute timeout
-        
+
         # Get current stuck items
         stuck_items = self._get_stuck_processing_items()
-        
+
         metrics = {
             "stuck_count": len(stuck_items),
-            "stuck_items": [item.id for item in stuck_items]
+            "stuck_items": [item.id for item in stuck_items],
         }
-        
+
         if stuck_items:
-            self.alerts.append(HealthAlert(
-                HealthStatus.WARNING,
-                f"Found {len(stuck_items)} stuck items, auto-reset applied",
-                {"stuck_items": [item.id for item in stuck_items], "action": "items_reset"}
-            ))
-        
+            self.alerts.append(
+                HealthAlert(
+                    HealthStatus.WARNING,
+                    f"Found {len(stuck_items)} stuck items, auto-reset applied",
+                    {
+                        "stuck_items": [item.id for item in stuck_items],
+                        "action": "items_reset",
+                    },
+                )
+            )
+
         return metrics
-    
-    def _check_failure_rates(self) -> Dict[str, Any]:
+
+    def _check_failure_rates(self) -> dict[str, Any]:
         """Check recent failure rates."""
         stats = self.queue.get_stats()
-        
+
         total_items = sum(stats.values())
         failed_items = stats.get("failed", 0)
-        
+
         failure_rate = failed_items / total_items if total_items > 0 else 0
-        
+
         metrics = {
             "failure_rate": failure_rate,
             "failed_count": failed_items,
-            "total_count": total_items
+            "total_count": total_items,
         }
-        
+
         if failure_rate >= self.HIGH_FAILURE_RATE and total_items >= 10:
-            self.alerts.append(HealthAlert(
-                HealthStatus.CRITICAL,
-                f"High failure rate: {failure_rate:.1%} ({failed_items}/{total_items})",
-                {"failure_rate": failure_rate, "action": "investigate_failures"}
-            ))
-        
+            self.alerts.append(
+                HealthAlert(
+                    HealthStatus.CRITICAL,
+                    f"High failure rate: {failure_rate:.1%} ({failed_items}/{total_items})",
+                    {"failure_rate": failure_rate, "action": "investigate_failures"},
+                )
+            )
+
         return metrics
-    
+
     def _determine_overall_status(self) -> HealthStatus:
         """Determine overall health status from alerts."""
         if not self.alerts:
             return HealthStatus.HEALTHY
-        
+
         # Check for critical alerts
         if any(alert.level == HealthStatus.CRITICAL for alert in self.alerts):
             return HealthStatus.CRITICAL
-        
+
         # Check for warnings
         if any(alert.level == HealthStatus.WARNING for alert in self.alerts):
             return HealthStatus.WARNING
-        
+
         return HealthStatus.HEALTHY
-    
-    def _get_old_pending_items(self) -> List:
+
+    def _get_old_pending_items(self) -> list:
         """Get items that have been pending for too long."""
         cutoff_time = datetime.now() - timedelta(seconds=self.WORKER_TIMEOUT_SECONDS)
-        
+
         items = self.queue.list_items(status=QueueStatus.PENDING, limit=100)
-        return [
-            item for item in items 
-            if item.created_at < cutoff_time
-        ]
-    
-    def _get_stuck_processing_items(self) -> List:
+        return [item for item in items if item.created_at < cutoff_time]
+
+    def _get_stuck_processing_items(self) -> list:
         """Get items stuck in processing state."""
         cutoff_time = datetime.now() - timedelta(seconds=self.STUCK_ITEM_THRESHOLD)
-        
+
         items = self.queue.list_items(status=QueueStatus.PROCESSING, limit=100)
-        return [
-            item for item in items 
-            if item.created_at < cutoff_time
-        ]
-    
-    def get_immediate_alerts(self) -> List[HealthAlert]:
+        return [item for item in items if item.created_at < cutoff_time]
+
+    def get_immediate_alerts(self) -> list[HealthAlert]:
         """Get alerts that require immediate attention."""
         return [
-            alert for alert in self.alerts 
+            alert
+            for alert in self.alerts
             if alert.level in [HealthStatus.CRITICAL, HealthStatus.FAILED]
         ]
-    
-    def auto_repair(self) -> Dict[str, Any]:
+
+    def auto_repair(self) -> dict[str, Any]:
         """Attempt automatic repair of detected issues."""
         repair_actions = []
-        
+
         # Check health first
-        health = self.check_health()
-        
+        self.check_health()
+
         for alert in self.alerts:
             action = alert.details.get("action")
-            
+
             if action == "start_worker":
                 try:
                     if self.manager.start():
-                        repair_actions.append(f"Started worker for {alert.details.get('pending_count')} pending items")
+                        repair_actions.append(
+                            f"Started worker for {alert.details.get('pending_count')} pending items"
+                        )
                     else:
                         repair_actions.append("Failed to start worker")
                 except Exception as e:
                     repair_actions.append(f"Error starting worker: {e}")
-            
+
             elif action == "restart_worker":
                 try:
                     self.manager.stop()
@@ -312,11 +341,13 @@ class QueueHealthMonitor:
                         repair_actions.append("Failed to restart worker")
                 except Exception as e:
                     repair_actions.append(f"Error restarting worker: {e}")
-            
+
             elif action == "items_reset":
-                repair_actions.append(f"Reset {alert.details.get('stuck_items', [])} stuck items")
-        
+                repair_actions.append(
+                    f"Reset {alert.details.get('stuck_items', [])} stuck items"
+                )
+
         return {
             "actions_taken": repair_actions,
-            "timestamp": datetime.now().isoformat()
+            "timestamp": datetime.now().isoformat(),
         }

mcp_ticketer/queue/manager.py

@@ -81,12 +81,22 @@ class WorkerManager:
         # Try to start worker
         return self.start()
 
-    def start(self) -> bool:
+    def start(self, timeout: float = 30.0) -> bool:
         """Start the worker process.
 
+        Args:
+            timeout: Maximum seconds to wait for worker to become fully operational (default: 30)
+
         Returns:
             True if started successfully, False otherwise
 
+        Raises:
+            TimeoutError: If worker doesn't start within timeout period
+
+        Note:
+            If timeout occurs, a worker may already be running in another process.
+            Check for stale lock files or processes using `get_status()`.
+
         """
         # Check if already running
         if self.is_running():
@@ -96,7 +106,19 @@ class WorkerManager:
         # Try to acquire lock
         if not self._acquire_lock():
             logger.warning("Could not acquire lock - another worker may be running")
-            return False
+            # Wait briefly to see if worker becomes operational
+            start_time = time.time()
+            while time.time() - start_time < timeout:
+                if self.is_running():
+                    logger.info("Worker started by another process")
+                    return True
+                time.sleep(1)
+
+            raise TimeoutError(
+                f"Worker start timed out after {timeout}s. "
+                f"Lock is held but worker is not running. "
+                f"Check for stale lock files or zombie processes."
+            )
 
         try:
             # Start worker in subprocess using the same Python executable as the CLI
@@ -113,9 +135,12 @@ class WorkerManager:
             if env_file.exists():
                 logger.debug(f"Loading environment from {env_file} for subprocess")
                 from dotenv import dotenv_values
+
                 env_vars = dotenv_values(env_file)
                 subprocess_env.update(env_vars)
-                logger.debug(f"Added {len(env_vars)} environment variables from .env.local")
+                logger.debug(
+                    f"Added {len(env_vars)} environment variables from .env.local"
+                )
 
             # Start as background process
             process = subprocess.Popen(
@@ -130,20 +155,42 @@ class WorkerManager:
             # Save PID
             self.pid_file.write_text(str(process.pid))
 
-            # Give the process a moment to start
-            import time
-
-            time.sleep(0.5)
-
-            # Verify process is running
-            if not psutil.pid_exists(process.pid):
-                logger.error("Worker process died immediately after starting")
-                self._cleanup()
-                return False
-
-            logger.info(f"Started worker process with PID {process.pid}")
-            return True
+            # Wait for worker to become fully operational with timeout
+            start_time = time.time()
+            while time.time() - start_time < timeout:
+                # Check if process exists
+                if not psutil.pid_exists(process.pid):
+                    logger.error("Worker process died during startup")
+                    self._cleanup()
+                    return False
+
+                # Check if worker is fully running (not just process exists)
+                if self.is_running():
+                    elapsed = time.time() - start_time
+                    logger.info(
+                        f"Worker started successfully in {elapsed:.1f}s with PID {process.pid}"
+                    )
+                    return True
+
+                # Log progress for long startups
+                elapsed = time.time() - start_time
+                if elapsed > 5 and int(elapsed) % 5 == 0:
+                    logger.debug(
+                        f"Waiting for worker to start, elapsed: {elapsed:.1f}s"
+                    )
+
+                time.sleep(0.5)
+
+            # Timeout reached
+            raise TimeoutError(
+                f"Worker start timed out after {timeout}s. "
+                f"Process spawned (PID {process.pid}) but worker did not become operational. "
+                f"Check worker logs for startup errors."
+            )
 
+        except TimeoutError:
+            # Re-raise timeout errors with context preserved
+            raise
         except Exception as e:
             logger.error(f"Failed to start worker: {e}")
             self._release_lock()
@@ -281,6 +328,7 @@ class WorkerManager:
 
         Returns:
             Path to Python executable
+
         """
         # First, try to detect if we're running in a pipx environment
         # by checking if the current executable is in a pipx venv
@@ -293,18 +341,21 @@ class WorkerManager:
 
         # Check if we can find the mcp-ticketer executable and extract its Python
         import shutil
+
         mcp_ticketer_path = shutil.which("mcp-ticketer")
         if mcp_ticketer_path:
             try:
                 # Read the shebang line to get the Python executable
-                with open(mcp_ticketer_path, 'r') as f:
+                with open(mcp_ticketer_path) as f:
                     first_line = f.readline().strip()
                     if first_line.startswith("#!") and "python" in first_line:
                         python_path = first_line[2:].strip()
                         if os.path.exists(python_path):
-                            logger.debug(f"Using Python from mcp-ticketer shebang: {python_path}")
+                            logger.debug(
+                                f"Using Python from mcp-ticketer shebang: {python_path}"
+                            )
                             return python_path
-            except (OSError, IOError):
+            except OSError:
                 pass
 
         # Fallback to sys.executable