mcp-code-indexer 2.0.2__py3-none-any.whl → 2.2.0__py3-none-any.whl

mcp_code_indexer/database/connection_health.py

@@ -262,12 +262,15 @@ class ConnectionHealthMonitor:
             }
         )
     
-    def get_health_status(self) -> Dict:
+    def get_health_status(self, include_retry_stats: bool = True) -> Dict:
         """
         Get current health status and metrics.
         
+        Args:
+            include_retry_stats: Whether to include retry executor statistics
+        
         Returns:
-            Dictionary with health status, metrics, and recent history
+            Dictionary with health status, metrics, recent history, and retry stats
         """
         # Get recent health status (last 5 checks)
         recent_checks = self._health_history[-5:] if self._health_history else []
@@ -276,7 +279,7 @@ class ConnectionHealthMonitor:
             if recent_checks else 0
         )
         
-        return {
+        health_status = {
             "is_monitoring": self._is_monitoring,
             "current_status": {
                 "is_healthy": (
@@ -301,6 +304,22 @@ class ConnectionHealthMonitor:
                 "timeout_seconds": self.timeout_seconds
             }
         }
+        
+        # Include retry executor statistics if available
+        if include_retry_stats and hasattr(self.database_manager, '_retry_executor'):
+            retry_executor = self.database_manager._retry_executor
+            if retry_executor:
+                health_status["retry_statistics"] = retry_executor.get_retry_stats()
+        
+        # Include database-level statistics if available
+        if hasattr(self.database_manager, 'get_database_stats'):
+            try:
+                db_stats = self.database_manager.get_database_stats()
+                health_status["database_statistics"] = db_stats
+            except Exception as e:
+                logger.warning(f"Failed to get database statistics: {e}")
+        
+        return health_status
     
     def get_recent_history(self, count: int = 10) -> List[Dict]:
         """
@@ -322,6 +341,171 @@ class ConnectionHealthMonitor:
             }
             for check in recent_checks
         ]
+    
+    def get_comprehensive_diagnostics(self) -> Dict:
+        """
+        Get comprehensive database health diagnostics for monitoring.
+        
+        This method provides detailed diagnostics suitable for the 
+        check_database_health MCP tool.
+        
+        Returns:
+            Comprehensive health diagnostics including retry metrics, 
+            performance data, and resilience statistics
+        """
+        # Get base health status with retry stats
+        base_status = self.get_health_status(include_retry_stats=True)
+        
+        # Add detailed performance analysis
+        diagnostics = {
+            **base_status,
+            "performance_analysis": {
+                "health_check_performance": {
+                    "avg_response_time_ms": self.metrics.avg_response_time_ms,
+                    "response_time_threshold_exceeded": self.metrics.avg_response_time_ms > 100,
+                    "recent_performance_trend": self._get_performance_trend()
+                },
+                "failure_analysis": {
+                    "failure_rate_percent": (
+                        (self.metrics.failed_checks / self.metrics.total_checks * 100)
+                        if self.metrics.total_checks > 0 else 0
+                    ),
+                    "consecutive_failures": self.metrics.consecutive_failures,
+                    "approaching_failure_threshold": (
+                        self.metrics.consecutive_failures >= self.failure_threshold - 1
+                    ),
+                    "pool_refresh_frequency": self.metrics.pool_refreshes
+                }
+            },
+            "resilience_indicators": {
+                "overall_health_score": self._calculate_health_score(),
+                "retry_effectiveness": self._analyze_retry_effectiveness(),
+                "connection_stability": self._assess_connection_stability(),
+                "recommendations": self._generate_health_recommendations()
+            },
+            "recent_history": self.get_recent_history(count=5)
+        }
+        
+        return diagnostics
+    
+    def _get_performance_trend(self) -> str:
+        """Analyze recent performance trend."""
+        if len(self._health_history) < 5:
+            return "insufficient_data"
+        
+        recent_times = [
+            check.response_time_ms for check in self._health_history[-5:]
+            if check.is_healthy
+        ]
+        
+        if len(recent_times) < 2:
+            return "insufficient_healthy_checks"
+        
+        # Simple trend analysis
+        if recent_times[-1] > recent_times[0] * 1.5:
+            return "degrading"
+        elif recent_times[-1] < recent_times[0] * 0.7:
+            return "improving"
+        else:
+            return "stable"
+    
+    def _calculate_health_score(self) -> float:
+        """Calculate overall health score (0-100)."""
+        if self.metrics.total_checks == 0:
+            return 100.0
+        
+        # Base score from success rate
+        success_rate = (self.metrics.successful_checks / self.metrics.total_checks) * 100
+        
+        # Penalize consecutive failures
+        failure_penalty = min(self.metrics.consecutive_failures * 10, 50)
+        
+        # Penalize high response times
+        response_penalty = min(max(0, self.metrics.avg_response_time_ms - 50) / 10, 20)
+        
+        # Calculate final score
+        score = success_rate - failure_penalty - response_penalty
+        return max(0.0, min(100.0, score))
+    
+    def _analyze_retry_effectiveness(self) -> Dict:
+        """Analyze retry mechanism effectiveness."""
+        if not hasattr(self.database_manager, '_retry_executor'):
+            return {"status": "no_retry_executor"}
+        
+        retry_executor = self.database_manager._retry_executor
+        if not retry_executor:
+            return {"status": "retry_executor_not_initialized"}
+        
+        retry_stats = retry_executor.get_retry_stats()
+        
+        return {
+            "status": "active",
+            "effectiveness_score": retry_stats.get("success_rate_percent", 0),
+            "retry_frequency": retry_stats.get("retry_rate_percent", 0),
+            "avg_attempts_per_operation": retry_stats.get("average_attempts_per_operation", 0),
+            "is_effective": retry_stats.get("success_rate_percent", 0) > 85
+        }
+    
+    def _assess_connection_stability(self) -> Dict:
+        """Assess connection stability."""
+        stability_score = 100.0
+        
+        # Penalize pool refreshes
+        if self.metrics.pool_refreshes > 0:
+            stability_score -= min(self.metrics.pool_refreshes * 15, 60)
+        
+        # Penalize consecutive failures
+        if self.metrics.consecutive_failures > 0:
+            stability_score -= min(self.metrics.consecutive_failures * 20, 80)
+        
+        return {
+            "stability_score": max(0.0, stability_score),
+            "pool_refreshes": self.metrics.pool_refreshes,
+            "consecutive_failures": self.metrics.consecutive_failures,
+            "is_stable": stability_score > 70
+        }
+    
+    def _generate_health_recommendations(self) -> List[str]:
+        """Generate health recommendations based on current metrics."""
+        recommendations = []
+        
+        # High failure rate
+        if self.metrics.total_checks > 0:
+            failure_rate = (self.metrics.failed_checks / self.metrics.total_checks) * 100
+            if failure_rate > 20:
+                recommendations.append(
+                    f"High failure rate ({failure_rate:.1f}%) - check database configuration"
+                )
+        
+        # High response times
+        if self.metrics.avg_response_time_ms > 100:
+            recommendations.append(
+                f"High response times ({self.metrics.avg_response_time_ms:.1f}ms) - consider optimizing queries"
+            )
+        
+        # Approaching failure threshold
+        if self.metrics.consecutive_failures >= self.failure_threshold - 1:
+            recommendations.append(
+                "Approaching failure threshold - pool refresh imminent"
+            )
+        
+        # Frequent pool refreshes
+        if self.metrics.pool_refreshes > 3:
+            recommendations.append(
+                "Frequent pool refreshes detected - investigate underlying connection issues"
+            )
+        
+        # No recent successful checks
+        if (self.metrics.last_success_time and 
+            datetime.utcnow() - self.metrics.last_success_time > timedelta(minutes=5)):
+            recommendations.append(
+                "No successful health checks in last 5 minutes - database may be unavailable"
+            )
+        
+        if not recommendations:
+            recommendations.append("Database health is optimal")
+        
+        return recommendations
 
 
 class DatabaseMetricsCollector:

mcp_code_indexer/database/database.py

@@ -21,8 +21,11 @@ from mcp_code_indexer.database.models import (
     Project, FileDescription, MergeConflict, SearchResult,
     CodebaseSizeInfo, ProjectOverview, WordFrequencyResult, WordFrequencyTerm
 )
-from mcp_code_indexer.database.retry_handler import (
-    RetryHandler, ConnectionRecoveryManager, create_retry_handler
+from mcp_code_indexer.database.retry_executor import (
+    RetryExecutor, create_retry_executor
+)
+from mcp_code_indexer.database.exceptions import (
+    DatabaseError, DatabaseLockError, classify_sqlite_error, is_retryable_error
 )
 from mcp_code_indexer.database.connection_health import (
     ConnectionHealthMonitor, DatabaseMetricsCollector
@@ -45,7 +48,10 @@ class DatabaseManager:
                  retry_count: int = 5,
                  timeout: float = 10.0,
                  enable_wal_mode: bool = True,
-                 health_check_interval: float = 30.0):
+                 health_check_interval: float = 30.0,
+                 retry_min_wait: float = 0.1,
+                 retry_max_wait: float = 2.0,
+                 retry_jitter: float = 0.2):
         """Initialize database manager with path to SQLite database."""
         self.db_path = db_path
         self.pool_size = pool_size
@@ -53,13 +59,20 @@ class DatabaseManager:
         self.timeout = timeout
         self.enable_wal_mode = enable_wal_mode
         self.health_check_interval = health_check_interval
+        self.retry_min_wait = retry_min_wait
+        self.retry_max_wait = retry_max_wait
+        self.retry_jitter = retry_jitter
         self._connection_pool: List[aiosqlite.Connection] = []
         self._pool_lock = None  # Will be initialized in async context
         self._write_lock = None  # Write serialization lock, initialized in async context
         
         # Retry and recovery components - configure with provided settings
-        self._retry_handler = create_retry_handler(max_attempts=retry_count)
-        self._recovery_manager = None  # Initialized in async context
+        self._retry_executor = create_retry_executor(
+            max_attempts=retry_count,
+            min_wait_seconds=retry_min_wait,
+            max_wait_seconds=retry_max_wait,
+            jitter_max_seconds=retry_jitter
+        )
         
         # Health monitoring and metrics
         self._health_monitor = None  # Initialized in async context
@@ -73,8 +86,7 @@ class DatabaseManager:
         self._pool_lock = asyncio.Lock()
         self._write_lock = asyncio.Lock()
         
-        # Initialize connection recovery manager
-        self._recovery_manager = ConnectionRecoveryManager(self)
+        # Connection recovery is now handled by the retry executor
         
         # Initialize health monitoring with configured interval
         self._health_monitor = ConnectionHealthMonitor(
@@ -87,6 +99,8 @@ class DatabaseManager:
         # Ensure database directory exists
         self.db_path.parent.mkdir(parents=True, exist_ok=True)
         
+        # Database initialization now uses the modern retry executor directly
+        
         # Apply migrations in order
         migrations_dir = Path(__file__).parent.parent.parent.parent / "migrations"
         migration_files = sorted(migrations_dir.glob("*.sql"))
@@ -217,30 +231,48 @@ class DatabaseManager:
         """
         Get a database connection with write serialization and automatic retry logic.
         
-        This combines write serialization with retry handling for maximum resilience
-        against database locking issues.
+        This uses the new RetryExecutor to properly handle retry logic without
+        the broken yield-in-retry-loop pattern that caused generator errors.
         
         Args:
             operation_name: Name of the operation for logging and monitoring
         """
-        if self._write_lock is None or self._retry_handler is None:
+        if self._write_lock is None:
             raise RuntimeError("DatabaseManager not initialized - call initialize() first")
         
-        async with self._retry_handler.with_retry(operation_name):
+        async def get_write_connection():
+            """Inner function to get connection - will be retried by executor."""
+            async with self._write_lock:
+                async with self.get_connection() as conn:
+                    return conn
+        
+        try:
+            # Use retry executor to handle connection acquisition with retries
+            connection = await self._retry_executor.execute_with_retry(
+                get_write_connection, 
+                operation_name
+            )
+            
             try:
-                async with self._write_lock:
-                    async with self.get_connection() as conn:
-                        yield conn
-                        
-                # Reset failure count on success
-                if self._recovery_manager:
-                    self._recovery_manager.reset_failure_count()
+                yield connection
+                
+                # Success - retry executor handles all failure tracking
                     
             except Exception as e:
-                # Handle persistent failures
-                if self._recovery_manager:
-                    await self._recovery_manager.handle_persistent_failure(operation_name, e)
+                # Error handling is managed by the retry executor
                 raise
+                
+        except DatabaseError:
+            # Re-raise our custom database errors as-is
+            raise
+        except Exception as e:
+            # Classify and wrap other exceptions
+            classified_error = classify_sqlite_error(e, operation_name)
+            logger.error(
+                f"Database operation '{operation_name}' failed: {classified_error.message}",
+                extra={"structured_data": classified_error.to_dict()}
+            )
+            raise classified_error
     
     def get_database_stats(self) -> Dict[str, Any]:
         """
@@ -253,14 +285,11 @@ class DatabaseManager:
             "connection_pool": {
                 "configured_size": self.pool_size,
                 "current_size": len(self._connection_pool)
-            }
+            },
+            "retry_executor": self._retry_executor.get_retry_stats() if self._retry_executor else {},
         }
         
-        if self._retry_handler:
-            stats["retry_stats"] = self._retry_handler.get_retry_stats()
-        
-        if self._recovery_manager:
-            stats["recovery_stats"] = self._recovery_manager.get_recovery_stats()
+        # Legacy retry handler removed - retry executor stats are included above
         
         if self._health_monitor:
             stats["health_status"] = self._health_monitor.get_health_status()
@@ -347,10 +376,13 @@ class DatabaseManager:
         """
         Execute a database operation within a transaction with automatic retry.
         
+        Uses the new RetryExecutor for robust retry handling with proper error
+        classification and exponential backoff.
+        
         Args:
             operation_func: Async function that takes a connection and performs the operation
             operation_name: Name of the operation for logging
-            max_retries: Maximum retry attempts
+            max_retries: Maximum retry attempts (overrides default retry executor config)
             timeout_seconds: Transaction timeout in seconds
             
         Returns:
@@ -363,9 +395,9 @@ class DatabaseManager:
             
             result = await db.execute_transaction_with_retry(my_operation, "insert_data")
         """
-        last_error = None
         
-        for attempt in range(1, max_retries + 1):
+        async def execute_transaction():
+            """Inner function to execute transaction - will be retried by executor."""
             try:
                 async with self.get_immediate_transaction(operation_name, timeout_seconds) as conn:
                     result = await operation_func(conn)
@@ -382,34 +414,15 @@ class DatabaseManager:
                 return result
                 
             except (aiosqlite.OperationalError, asyncio.TimeoutError) as e:
-                last_error = e
-                
                 # Record locking event for metrics
                 if self._metrics_collector and "locked" in str(e).lower():
                     self._metrics_collector.record_locking_event(operation_name, str(e))
                 
-                if attempt < max_retries:
-                    # Exponential backoff with jitter
-                    delay = 0.1 * (2 ** (attempt - 1))
-                    jitter = delay * 0.1 * (2 * random.random() - 1)  # ±10% jitter
-                    wait_time = max(0.05, delay + jitter)
-                    
-                    logger.warning(
-                        f"Transaction attempt {attempt} failed for {operation_name}, retrying in {wait_time:.2f}s: {e}",
-                        extra={
-                            "structured_data": {
-                                "transaction_retry": {
-                                    "operation": operation_name,
-                                    "attempt": attempt,
-                                    "delay_seconds": wait_time,
-                                    "error": str(e)
-                                }
-                            }
-                        }
-                    )
-                    await asyncio.sleep(wait_time)
-                else:
-                    # Record failed operation metrics
+                # Classify the error for better handling
+                classified_error = classify_sqlite_error(e, operation_name)
+                
+                # Record failed operation metrics for non-retryable errors
+                if not is_retryable_error(classified_error):
                     if self._metrics_collector:
                         self._metrics_collector.record_operation(
                             operation_name,
@@ -417,21 +430,34 @@ class DatabaseManager:
                             False,
                             len(self._connection_pool)
                         )
-                    
-                    logger.error(
-                        f"Transaction failed after {max_retries} attempts for {operation_name}: {e}",
-                        extra={
-                            "structured_data": {
-                                "transaction_failure": {
-                                    "operation": operation_name,
-                                    "max_retries": max_retries,
-                                    "final_error": str(e)
-                                }
-                            }
-                        }
-                    )
+                
+                raise classified_error
         
-        raise last_error
+        try:
+            # Create a temporary retry executor with custom max_retries if different from default
+            if max_retries != self._retry_executor.config.max_attempts:
+                from mcp_code_indexer.database.retry_executor import RetryConfig, RetryExecutor
+                temp_config = RetryConfig(
+                    max_attempts=max_retries,
+                    min_wait_seconds=self._retry_executor.config.min_wait_seconds,
+                    max_wait_seconds=self._retry_executor.config.max_wait_seconds,
+                    jitter_max_seconds=self._retry_executor.config.jitter_max_seconds
+                )
+                temp_executor = RetryExecutor(temp_config)
+                return await temp_executor.execute_with_retry(execute_transaction, operation_name)
+            else:
+                return await self._retry_executor.execute_with_retry(execute_transaction, operation_name)
+                
+        except DatabaseError as e:
+            # Record failed operation metrics for final failure
+            if self._metrics_collector:
+                self._metrics_collector.record_operation(
+                    operation_name,
+                    timeout_seconds * 1000,
+                    False,
+                    len(self._connection_pool)
+                )
+            raise
     
     # Project operations