django-cfg 1.5.20__py3-none-any.whl → 1.5.31__py3-none-any.whl

django_cfg/utils/pool_monitor.py

@@ -0,0 +1,320 @@
+"""
+Connection Pool Monitoring for Django-CFG.
+
+Provides utilities to monitor and inspect PostgreSQL connection pool status.
+Works with Django 5.1+ native connection pooling (psycopg3).
+
+Usage:
+    from django_cfg.utils.pool_monitor import PoolMonitor
+
+    # Get pool statistics
+    monitor = PoolMonitor()
+    stats = monitor.get_pool_stats()
+
+    # Check pool health
+    health = monitor.check_pool_health()
+
+    # Log pool status
+    monitor.log_pool_status()
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+from django.conf import settings
+from django.db import connection
+
+from .smart_defaults import _detect_asgi_mode, get_pool_config
+
+logger = logging.getLogger('django_cfg.pool_monitor')
+
+
+class PoolMonitor:
+    """
+    Monitor and inspect database connection pool.
+
+    Provides methods to retrieve pool statistics, check health,
+    and log pool status for operational visibility.
+    """
+
+    def __init__(self, database_alias: str = 'default'):
+        """
+        Initialize pool monitor.
+
+        Args:
+            database_alias: Database alias to monitor (default: 'default')
+        """
+        self.database_alias = database_alias
+
+    def get_pool_stats(self) -> Optional[Dict[str, Any]]:
+        """
+        Get current connection pool statistics.
+
+        Returns:
+            Dict with pool statistics:
+            {
+                'pool_size': int,          # Current pool size
+                'pool_available': int,     # Available connections
+                'pool_min_size': int,      # Configured minimum size
+                'pool_max_size': int,      # Configured maximum size
+                'pool_timeout': int,       # Connection timeout
+                'is_asgi': bool,           # Deployment mode
+                'environment': str,        # Environment name
+                'backend': str,            # Database backend
+                'has_pool': bool,          # Whether pooling is enabled
+            }
+
+            Returns None if connection pooling is not configured.
+
+        Example:
+            >>> monitor = PoolMonitor()
+            >>> stats = monitor.get_pool_stats()
+            >>> print(f"Pool size: {stats['pool_size']}/{stats['pool_max_size']}")
+        """
+        try:
+            # Get database configuration
+            db_config = settings.DATABASES.get(self.database_alias, {})
+            backend = db_config.get('ENGINE', 'unknown')
+
+            # Check if PostgreSQL with pooling
+            if 'postgresql' not in backend:
+                logger.debug(f"Database {self.database_alias} is not PostgreSQL, pooling not available")
+                return None
+
+            pool_options = db_config.get('OPTIONS', {}).get('pool', {})
+            if not pool_options:
+                logger.debug(f"No pool configuration found for database {self.database_alias}")
+                return None
+
+            # Detect environment and mode
+            is_asgi = _detect_asgi_mode()
+            environment = getattr(settings, 'ENVIRONMENT', 'production')
+
+            # Get expected pool config
+            expected_config = get_pool_config(environment, is_asgi=is_asgi)
+
+            # Try to get actual pool statistics from psycopg3
+            pool_size = None
+            pool_available = None
+
+            try:
+                # Access underlying psycopg3 connection pool
+                # This requires psycopg3 with connection pooling
+                db_conn = connection.connection
+                if db_conn and hasattr(db_conn, 'pgconn'):
+                    pool = getattr(db_conn, 'pool', None)
+                    if pool:
+                        # psycopg3 ConnectionPool has these attributes
+                        pool_size = getattr(pool, 'size', None)
+                        pool_available = getattr(pool, 'available', None)
+            except Exception as e:
+                logger.debug(f"Could not retrieve live pool stats: {e}")
+
+            return {
+                'pool_size': pool_size,
+                'pool_available': pool_available,
+                'pool_min_size': pool_options.get('min_size', expected_config['min_size']),
+                'pool_max_size': pool_options.get('max_size', expected_config['max_size']),
+                'pool_timeout': pool_options.get('timeout', expected_config['timeout']),
+                'max_lifetime': pool_options.get('max_lifetime', 3600),
+                'max_idle': pool_options.get('max_idle', 600),
+                'is_asgi': is_asgi,
+                'environment': environment,
+                'backend': backend,
+                'has_pool': True,
+            }
+
+        except Exception as e:
+            logger.error(f"Failed to get pool stats: {e}", exc_info=True)
+            return None
+
+    def check_pool_health(self) -> Dict[str, Any]:
+        """
+        Check connection pool health status.
+
+        Returns:
+            Dict with health information:
+            {
+                'healthy': bool,           # Overall health status
+                'status': str,             # 'healthy', 'warning', 'critical', 'unavailable'
+                'capacity_percent': float, # Pool capacity usage (0-100)
+                'issues': list,            # List of detected issues
+                'recommendations': list,   # Recommended actions
+            }
+
+        Health thresholds:
+            - < 70% capacity: Healthy
+            - 70-90% capacity: Warning
+            - > 90% capacity: Critical
+
+        Example:
+            >>> monitor = PoolMonitor()
+            >>> health = monitor.check_pool_health()
+            >>> if not health['healthy']:
+            ...     print(f"Issues: {', '.join(health['issues'])}")
+        """
+        stats = self.get_pool_stats()
+
+        if not stats or not stats['has_pool']:
+            return {
+                'healthy': True,  # No pool = no problem (using regular connections)
+                'status': 'unavailable',
+                'capacity_percent': 0.0,
+                'issues': ['Connection pooling not configured'],
+                'recommendations': ['Consider enabling connection pooling for production'],
+            }
+
+        issues = []
+        recommendations = []
+        healthy = True
+        status = 'healthy'
+
+        # Calculate capacity if live stats available
+        capacity_percent = 0.0
+        if stats['pool_size'] is not None and stats['pool_max_size']:
+            capacity_percent = (stats['pool_size'] / stats['pool_max_size']) * 100
+
+            # Check capacity thresholds
+            if capacity_percent >= 90:
+                status = 'critical'
+                healthy = False
+                issues.append(f"Pool capacity critical: {capacity_percent:.1f}% used")
+                recommendations.append("Increase DB_POOL_MAX_SIZE or scale database")
+            elif capacity_percent >= 70:
+                status = 'warning'
+                issues.append(f"Pool capacity high: {capacity_percent:.1f}% used")
+                recommendations.append("Monitor pool usage and consider increasing max_size")
+
+        # Check if min_size is reasonable
+        min_size = stats['pool_min_size']
+        max_size = stats['pool_max_size']
+
+        if min_size >= max_size * 0.9:
+            issues.append(f"Min size ({min_size}) too close to max size ({max_size})")
+            recommendations.append("Reduce DB_POOL_MIN_SIZE for better resource management")
+
+        # Check timeout
+        timeout = stats['pool_timeout']
+        if timeout > 30:
+            issues.append(f"Pool timeout high: {timeout}s")
+            recommendations.append("Long timeouts may indicate slow queries or insufficient pool size")
+
+        # Check ASGI vs WSGI pool sizing
+        is_asgi = stats['is_asgi']
+        mode = 'ASGI' if is_asgi else 'WSGI'
+
+        expected_config = get_pool_config(stats['environment'], is_asgi=is_asgi)
+        if max_size < expected_config['max_size'] * 0.5:
+            issues.append(f"Pool size low for {mode} mode")
+            recommendations.append(f"Consider increasing to {expected_config['max_size']} for {mode}")
+
+        return {
+            'healthy': healthy and len(issues) == 0,
+            'status': status,
+            'capacity_percent': capacity_percent,
+            'issues': issues,
+            'recommendations': recommendations,
+        }
+
+    def log_pool_status(self, level: str = 'info') -> None:
+        """
+        Log current pool status to Django logger.
+
+        Args:
+            level: Log level ('debug', 'info', 'warning', 'error')
+
+        Example:
+            >>> monitor = PoolMonitor()
+            >>> monitor.log_pool_status(level='info')
+        """
+        stats = self.get_pool_stats()
+
+        if not stats:
+            logger.debug(f"No pool statistics available for database {self.database_alias}")
+            return
+
+        health = self.check_pool_health()
+
+        log_func = getattr(logger, level, logger.info)
+
+        mode = 'ASGI' if stats['is_asgi'] else 'WSGI'
+        status_emoji = {
+            'healthy': '✅',
+            'warning': '⚠️',
+            'critical': '🔴',
+            'unavailable': '⚪',
+        }.get(health['status'], '❓')
+
+        log_message = (
+            f"[Pool Monitor] {status_emoji} Status: {health['status'].upper()} | "
+            f"Mode: {mode} | Env: {stats['environment']} | "
+            f"Pool: {stats['pool_size'] or '?'}/{stats['pool_max_size']} | "
+            f"Capacity: {health['capacity_percent']:.1f}%"
+        )
+
+        log_func(log_message)
+
+        if health['issues']:
+            for issue in health['issues']:
+                logger.warning(f"[Pool Monitor] Issue: {issue}")
+
+        if health['recommendations']:
+            for rec in health['recommendations']:
+                logger.info(f"[Pool Monitor] Recommendation: {rec}")
+
+    def get_pool_info_dict(self) -> Dict[str, Any]:
+        """
+        Get complete pool information as a dictionary.
+
+        Combines statistics and health check into a single dict.
+        Useful for API responses or structured logging.
+
+        Returns:
+            Dict with complete pool information including stats and health.
+
+        Example:
+            >>> monitor = PoolMonitor()
+            >>> info = monitor.get_pool_info_dict()
+            >>> print(json.dumps(info, indent=2))
+        """
+        stats = self.get_pool_stats()
+        health = self.check_pool_health()
+
+        if not stats:
+            return {
+                'available': False,
+                'reason': 'Connection pooling not configured',
+            }
+
+        return {
+            'available': True,
+            'statistics': stats,
+            'health': health,
+            'timestamp': self._get_timestamp(),
+        }
+
+    @staticmethod
+    def _get_timestamp() -> str:
+        """Get current timestamp in ISO format."""
+        from datetime import datetime
+        return datetime.utcnow().isoformat() + 'Z'
+
+
+# Convenience function for quick pool status check
+def get_pool_status(database_alias: str = 'default') -> Dict[str, Any]:
+    """
+    Convenience function to quickly get pool status.
+
+    Args:
+        database_alias: Database alias to check (default: 'default')
+
+    Returns:
+        Dict with pool information (same as PoolMonitor.get_pool_info_dict())
+
+    Example:
+        >>> from django_cfg.utils.pool_monitor import get_pool_status
+        >>> status = get_pool_status()
+        >>> print(status['health']['status'])
+    """
+    monitor = PoolMonitor(database_alias=database_alias)
+    return monitor.get_pool_info_dict()

django_cfg/utils/smart_defaults.py

@@ -7,6 +7,8 @@ Following KISS principle:
 - Logging handled by django_logger module
 """
 
+import os
+import sys
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -14,9 +16,9 @@ from typing import Any, Dict, List, Optional
 def get_log_filename() -> str:
     """
     Determine the correct log filename based on project type.
-    
+
     Returns:
-        - 'django-cfg.log' for django-cfg projects  
+        - 'django-cfg.log' for django-cfg projects
         - 'django.log' for regular Django projects
     """
     try:
@@ -35,6 +37,188 @@ def get_log_filename() -> str:
         return 'django-cfg.log'
 
 
+def _detect_asgi_mode() -> bool:
+    """
+    Detect if Django is running in ASGI or WSGI mode.
+
+    Detection priority:
+    1. DJANGO_ASGI environment variable (explicit override)
+    2. ASGI_APPLICATION setting (if Django is configured)
+    3. Command-line arguments (uvicorn, daphne, hypercorn)
+    4. Default: False (WSGI mode)
+
+    Returns:
+        True if ASGI mode, False if WSGI mode
+
+    Examples:
+        >>> os.environ['DJANGO_ASGI'] = 'true'
+        >>> _detect_asgi_mode()
+        True
+
+        >>> 'uvicorn' in sys.argv[0]
+        >>> _detect_asgi_mode()
+        True
+    """
+    # 1. Check explicit env var override
+    asgi_env = os.environ.get('DJANGO_ASGI', '').lower()
+    if asgi_env in ('true', '1', 'yes'):
+        return True
+    elif asgi_env in ('false', '0', 'no'):
+        return False
+
+    # 2. Check Django settings for ASGI_APPLICATION
+    try:
+        from django.conf import settings
+        if hasattr(settings, 'ASGI_APPLICATION') and settings.ASGI_APPLICATION:
+            return True
+    except (ImportError, Exception):
+        pass
+
+    # 3. Check command-line arguments for ASGI servers
+    command_line = ' '.join(sys.argv).lower()
+    asgi_servers = ['uvicorn', 'daphne', 'hypercorn']
+    for server in asgi_servers:
+        if server in command_line:
+            return True
+
+    # Default: WSGI mode
+    return False
+
+
+def get_pool_config(environment: str = "development", is_asgi: Optional[bool] = None) -> Dict[str, Any]:
+    """
+    Get connection pool configuration.
+
+    By default, uses simple environment-based configuration. Set AUTO_POOL_SIZE=true
+    to enable automatic ASGI/WSGI detection and optimization.
+
+    Args:
+        environment: Environment name ('development', 'testing', 'staging', 'production')
+        is_asgi: Deployment mode. If None and AUTO_POOL_SIZE=true, auto-detects mode
+
+    Returns:
+        Dict with pool configuration:
+        {
+            'min_size': int,      # Minimum pool size
+            'max_size': int,      # Maximum pool size
+            'timeout': int,       # Connection timeout (seconds)
+            'max_lifetime': int,  # Max connection lifetime (seconds)
+            'max_idle': int,      # Max idle time before closing (seconds)
+        }
+
+    Environment Variables:
+        DB_POOL_MIN_SIZE: Minimum pool size (default: 10)
+        DB_POOL_MAX_SIZE: Maximum pool size (default: 50)
+        DB_POOL_TIMEOUT: Connection timeout in seconds (default: 30)
+        AUTO_POOL_SIZE: Enable automatic ASGI/WSGI detection (default: false)
+
+    Examples:
+        # Simple static config (default):
+        >>> get_pool_config('production')
+        {'min_size': 10, 'max_size': 50, ...}
+
+        # With auto-detection:
+        >>> os.environ['AUTO_POOL_SIZE'] = 'true'
+        >>> get_pool_config('production', is_asgi=True)
+        {'min_size': 5, 'max_size': 20, ...}  # Optimized for ASGI
+    """
+    # Check if auto-detection is enabled
+    auto_detect = os.environ.get('AUTO_POOL_SIZE', 'false').lower() in ('true', '1', 'yes')
+
+    # Simple static configuration (default)
+    if not auto_detect and is_asgi is None:
+        # Use simple env var based config
+        try:
+            min_size = int(os.environ.get('DB_POOL_MIN_SIZE', 10))
+        except ValueError:
+            min_size = 10
+
+        try:
+            max_size = int(os.environ.get('DB_POOL_MAX_SIZE', 50))
+        except ValueError:
+            max_size = 50
+
+        try:
+            timeout = int(os.environ.get('DB_POOL_TIMEOUT', 30))
+        except ValueError:
+            timeout = 30
+
+        # Validate
+        if min_size >= max_size:
+            min_size = max(1, max_size - 1)
+
+        return {
+            'min_size': min_size,
+            'max_size': max_size,
+            'timeout': timeout,
+            'max_lifetime': 3600,  # 1 hour
+            'max_idle': 600,       # 10 minutes
+        }
+
+    # Auto-detect ASGI mode if enabled and not specified
+    if is_asgi is None:
+        is_asgi = _detect_asgi_mode()
+
+    # Pool configuration matrix
+    # Format: (min_size, max_size, timeout)
+    pool_configs = {
+        'development': {
+            'asgi': (2, 10, 10),
+            'wsgi': (3, 15, 20),
+        },
+        'testing': {
+            'asgi': (1, 5, 5),
+            'wsgi': (2, 10, 10),
+        },
+        'staging': {
+            'asgi': (3, 15, 10),
+            'wsgi': (5, 30, 20),
+        },
+        'production': {
+            'asgi': (5, 20, 10),
+            'wsgi': (10, 50, 30),
+        },
+    }
+
+    # Get base configuration
+    env_key = environment.lower()
+    if env_key not in pool_configs:
+        # Fallback to development for unknown environments
+        env_key = 'development'
+
+    mode_key = 'asgi' if is_asgi else 'wsgi'
+    min_size, max_size, timeout = pool_configs[env_key][mode_key]
+
+    # Allow environment variable overrides
+    try:
+        min_size = int(os.environ.get('DB_POOL_MIN_SIZE', min_size))
+    except ValueError:
+        pass  # Keep default if invalid
+
+    try:
+        max_size = int(os.environ.get('DB_POOL_MAX_SIZE', max_size))
+    except ValueError:
+        pass
+
+    try:
+        timeout = int(os.environ.get('DB_POOL_TIMEOUT', timeout))
+    except ValueError:
+        pass
+
+    # Validate: min_size must be < max_size
+    if min_size >= max_size:
+        min_size = max(1, max_size - 1)
+
+    # Build and return pool configuration
+    return {
+        'min_size': min_size,
+        'max_size': max_size,
+        'timeout': timeout,
+        'max_lifetime': 3600,  # 1 hour
+        'max_idle': 600,       # 10 minutes
+    }
+
+
 class SmartDefaults:
     """
     Environment-aware smart defaults for Django configuration.
@@ -45,18 +229,50 @@ class SmartDefaults:
 
     @staticmethod
     def get_database_defaults(environment: str = "development", debug: bool = False, engine: str = "sqlite3") -> Dict[str, Any]:
-        """Get database configuration defaults."""
+        """
+        Get database configuration defaults.
+
+        For PostgreSQL with Django 5.1+:
+        - Uses native connection pooling (recommended for ASGI/async apps)
+        - CONN_MAX_AGE = 0 (required with native pooling)
+        - ATOMIC_REQUESTS = True (default - safe and works with pooling)
+        - Pool sizes: Auto-configured based on environment and ASGI/WSGI mode
+        - Health checks: Handled automatically by psycopg3 pool
+
+        Note on Transaction Safety:
+        ATOMIC_REQUESTS=True is enabled by default, which wraps each request
+        in a database transaction. This adds ~5-10ms overhead but ensures data
+        integrity without manual transaction management.
+
+        This works perfectly fine with connection pooling. If you need to optimize
+        for read-heavy workloads, you can disable ATOMIC_REQUESTS and use selective
+        transactions via Django's @transaction.atomic decorator on write views.
+
+        References:
+        - Django 5.1+ native pooling: https://docs.djangoproject.com/en/5.2/ref/databases/#connection-pooling
+        - ASGI best practices: persistent connections should be disabled with ASGI
+        """
         defaults = {
             'ENGINE': 'django.db.backends.sqlite3',
             'NAME': Path('db') / 'db.sqlite3',
-            'ATOMIC_REQUESTS': True,
-            'CONN_MAX_AGE': 60,
+            'ATOMIC_REQUESTS': True,  # Safe default - ~5-10ms overhead acceptable for data integrity
+            'CONN_MAX_AGE': 0,  # Set to 0 for native pooling (Django 5.1+)
+            'CONN_HEALTH_CHECKS': True,  # Enable health checks to prevent stale connections
             'OPTIONS': {}
         }
 
         # Add engine-specific options
         if engine == "django.db.backends.postgresql":
-            defaults['OPTIONS']['connect_timeout'] = 20
+            # Native connection pooling for Django 5.1+ with psycopg >= 3.1
+            # See: https://docs.djangoproject.com/en/5.2/ref/databases/#postgresql-connection-pooling
+
+            # Get dynamic pool configuration based on environment and deployment mode
+            pool_config = get_pool_config(environment=environment, is_asgi=None)
+
+            defaults['OPTIONS'] = {
+                'connect_timeout': 20,
+                'pool': pool_config,  # Dynamic pool config (ASGI/WSGI aware)
+            }
         elif engine == "django.db.backends.sqlite3":
             defaults['OPTIONS']['timeout'] = 20  # SQLite uses 'timeout'
 
@@ -139,7 +355,16 @@ class SmartDefaults:
 
     @staticmethod
     def get_middleware_defaults() -> List[str]:
-        """Get middleware configuration defaults."""
+        """
+        Get middleware configuration defaults.
+
+        Note:
+            ConnectionPoolCleanupMiddleware is automatically added LAST if
+            enable_pool_cleanup=True in DjangoConfig (default).
+
+            This middleware prevents connection leaks when using native
+            connection pooling with ATOMIC_REQUESTS=False.
+        """
         return [
             'corsheaders.middleware.CorsMiddleware',
             'django.middleware.security.SecurityMiddleware',
@@ -149,6 +374,7 @@ class SmartDefaults:
             'django.contrib.auth.middleware.AuthenticationMiddleware',
             'django.contrib.messages.middleware.MessageMiddleware',
             'django.middleware.clickjacking.XFrameOptionsMiddleware',
+            # ConnectionPoolCleanupMiddleware added in DjangoConfig.get_middleware()
         ]
 
     @staticmethod