setu-trafficmonitor 2.0.0__py3-none-any.whl

trafficmonitor/circuit_breaker.py

@@ -0,0 +1,63 @@
+"""
+Circuit breaker pattern for database resilience
+"""
+import time
+import threading
+from enum import Enum
+from typing import Callable, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class CircuitState(Enum):
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+
+class CircuitBreaker:
+    """Circuit breaker for database operations"""
+    
+    def __init__(self, failure_threshold: int = 5, timeout: int = 60):
+        self.failure_threshold = failure_threshold
+        self.timeout = timeout
+        self.failure_count = 0
+        self.last_failure_time = None
+        self.state = CircuitState.CLOSED
+        self._lock = threading.Lock()
+    
+    def call(self, func: Callable, *args, **kwargs) -> Any:
+        """Execute function with circuit breaker protection"""
+        with self._lock:
+            if self.state == CircuitState.OPEN:
+                if time.time() - self.last_failure_time > self.timeout:
+                    self.state = CircuitState.HALF_OPEN
+                else:
+                    raise Exception("Circuit breaker is OPEN")
+            
+            try:
+                result = func(*args, **kwargs)
+                self._on_success()
+                return result
+            except Exception as e:
+                self._on_failure()
+                raise e
+    
+    def _on_success(self):
+        """Reset circuit breaker on successful operation"""
+        self.failure_count = 0
+        self.state = CircuitState.CLOSED
+    
+    def _on_failure(self):
+        """Handle failure in circuit breaker"""
+        self.failure_count += 1
+        self.last_failure_time = time.time()
+        
+        if self.failure_count >= self.failure_threshold:
+            self.state = CircuitState.OPEN
+            logger.error(f"Circuit breaker opened after {self.failure_count} failures")
+
+
+# Global circuit breaker instance
+db_circuit_breaker = CircuitBreaker()

trafficmonitor/conf.py

@@ -0,0 +1,154 @@
+"""
+TrafficMonitor Configuration
+Enterprise-grade configuration management with environment support.
+"""
+import os
+from django.conf import settings
+from typing import List, Dict, Any
+
+
+class TrafficMonitorConfig:
+    """
+    Centralized configuration for TrafficMonitor.
+    Supports environment variables and Django settings override.
+    """
+    
+    # Environment Detection
+    ENVIRONMENT = os.getenv('DJANGO_ENV', getattr(settings, 'ENVIRONMENT', 'development'))
+    IS_PRODUCTION = ENVIRONMENT.lower() in ['production', 'prod']
+    
+    # Role-based Access Control
+    VIEWER_ROLES = getattr(settings, 'TRAFFICMONITOR_VIEWER_ROLES', 
+                          os.getenv('TRAFFICMONITOR_VIEWER_ROLES', 'analyst,viewer,developer').split(','))
+    ADMIN_ROLES = getattr(settings, 'TRAFFICMONITOR_ADMIN_ROLES', 
+                         os.getenv('TRAFFICMONITOR_ADMIN_ROLES', 'admin,superuser').split(','))
+    
+    # Authentication Headers
+    USER_ID_HEADER = getattr(settings, 'TRAFFICMONITOR_USER_ID_HEADER', 
+                            os.getenv('TRAFFICMONITOR_USER_ID_HEADER', 'X-User-ID'))
+    USER_ROLE_HEADER = getattr(settings, 'TRAFFICMONITOR_USER_ROLE_HEADER', 
+                              os.getenv('TRAFFICMONITOR_USER_ROLE_HEADER', 'X-User-Role'))
+    
+    # Data Retention
+    RETENTION_DAYS = int(getattr(settings, 'TRAFFICMONITOR_RETENTION_DAYS', 
+                                os.getenv('TRAFFICMONITOR_RETENTION_DAYS', '90')))
+    
+    # Performance Settings
+    ASYNC_LOGGING = getattr(settings, 'TRAFFICMONITOR_ASYNC_LOGGING', 
+                           os.getenv('TRAFFICMONITOR_ASYNC_LOGGING', 'false').lower() == 'true')
+    
+    ENABLE_SAMPLING = getattr(settings, 'TRAFFICMONITOR_ENABLE_SAMPLING', 
+                             os.getenv('TRAFFICMONITOR_ENABLE_SAMPLING', 'false').lower() == 'true')
+    
+    SAMPLE_RATE = float(getattr(settings, 'TRAFFICMONITOR_SAMPLE_RATE', 
+                               os.getenv('TRAFFICMONITOR_SAMPLE_RATE', '1.0')))
+    
+    # Database Settings
+    BULK_CREATE_BATCH_SIZE = int(getattr(settings, 'TRAFFICMONITOR_BULK_BATCH_SIZE', 
+                                        os.getenv('TRAFFICMONITOR_BULK_BATCH_SIZE', '1000')))
+    
+    # Circuit Breaker Settings
+    CIRCUIT_BREAKER_ENABLED = getattr(settings, 'TRAFFICMONITOR_CIRCUIT_BREAKER_ENABLED', 
+                                     os.getenv('TRAFFICMONITOR_CIRCUIT_BREAKER_ENABLED', 'true').lower() == 'true')
+    
+    CIRCUIT_BREAKER_FAILURE_THRESHOLD = int(getattr(settings, 'TRAFFICMONITOR_CB_FAILURE_THRESHOLD', 
+                                                   os.getenv('TRAFFICMONITOR_CB_FAILURE_THRESHOLD', '5')))
+    
+    # Security Settings
+    ENABLE_PII_SANITIZATION = getattr(settings, 'TRAFFICMONITOR_ENABLE_PII_SANITIZATION', 
+                                     os.getenv('TRAFFICMONITOR_ENABLE_PII_SANITIZATION', 'true').lower() == 'true')
+    
+    MAX_BODY_LENGTH = int(getattr(settings, 'TRAFFICMONITOR_MAX_BODY_LENGTH', 
+                                 os.getenv('TRAFFICMONITOR_MAX_BODY_LENGTH', '10000')))
+    
+    # Excluded Paths (production optimized)
+    DEFAULT_EXCLUDED_PATHS = [
+        '/admin/jsi18n/',
+        '/static/',
+        '/media/',
+        '/__debug__/',
+        '/favicon.ico',
+        '/health/',
+        '/healthz/',
+        '/metrics/',
+        '/prometheus/',
+    ]
+    
+    EXCLUDED_PATHS = getattr(settings, 'TRAFFICMONITOR_EXCLUDED_PATHS', DEFAULT_EXCLUDED_PATHS)
+    
+    # Monitoring & Alerting
+    ENABLE_METRICS = getattr(settings, 'TRAFFICMONITOR_ENABLE_METRICS', 
+                            os.getenv('TRAFFICMONITOR_ENABLE_METRICS', 'true').lower() == 'true')
+    
+    ALERT_ERROR_THRESHOLD = int(getattr(settings, 'TRAFFICMONITOR_ALERT_ERROR_THRESHOLD', 
+                                       os.getenv('TRAFFICMONITOR_ALERT_ERROR_THRESHOLD', '100')))
+    
+    @classmethod
+    def validate_configuration(cls) -> List[str]:
+        """Validate configuration and return list of issues"""
+        issues = []
+        
+        if cls.SAMPLE_RATE < 0 or cls.SAMPLE_RATE > 1:
+            issues.append("SAMPLE_RATE must be between 0 and 1")
+        
+        if cls.RETENTION_DAYS < 1:
+            issues.append("RETENTION_DAYS must be positive")
+        
+        if cls.BULK_CREATE_BATCH_SIZE < 1:
+            issues.append("BULK_CREATE_BATCH_SIZE must be positive")
+        
+        return issues
+    
+    @classmethod
+    def get_production_settings(cls) -> Dict[str, Any]:
+        """Get recommended production settings"""
+        return {
+            'ASYNC_LOGGING': True,
+            'ENABLE_SAMPLING': True,
+            'SAMPLE_RATE': 0.1,  # 10% sampling in production
+            'CIRCUIT_BREAKER_ENABLED': True,
+            'ENABLE_PII_SANITIZATION': True,
+            'ENABLE_METRICS': True,
+            'BULK_CREATE_BATCH_SIZE': 5000,
+        }
+    
+    @classmethod
+    def is_user_authorized(cls, user_role, required_roles=None):
+        """
+        Check if user role has access.
+        
+        Args:
+            user_role: Role from header (X-User-Role)
+            required_roles: List of required roles (defaults to VIEWER_ROLES + ADMIN_ROLES)
+        
+        Returns:
+            Boolean indicating if user is authorized
+        """
+        if not user_role:
+            return False
+        
+        if required_roles is None:
+            required_roles = cls.VIEWER_ROLES + cls.ADMIN_ROLES
+        
+        return user_role.lower() in [r.lower() for r in required_roles]
+    
+    @classmethod
+    def is_admin(cls, user_role):
+        """Check if user has admin role"""
+        return cls.is_user_authorized(user_role, cls.ADMIN_ROLES)
+    
+    @classmethod
+    def get_user_info_from_request(cls, request):
+        """
+        Extract user ID and role from request headers.
+        
+        Returns:
+            dict: {'user_id': str, 'role': str}
+        """
+        user_id = request.META.get(f'HTTP_{cls.USER_ID_HEADER.replace("-", "_").upper()}')
+        user_role = request.META.get(f'HTTP_{cls.USER_ROLE_HEADER.replace("-", "_").upper()}')
+        
+        return {
+            'user_id': user_id,
+            'role': user_role
+        }

trafficmonitor/dashboard_security.py

@@ -0,0 +1,111 @@
+"""
+Dashboard security middleware and utilities
+"""
+from django.http import HttpResponseForbidden, JsonResponse
+from django.utils.deprecation import MiddlewareMixin
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.clickjacking import xframe_options_deny
+from django.views.decorators.cache import never_cache
+from functools import wraps
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class DashboardSecurityMiddleware(MiddlewareMixin):
+    """Security middleware specifically for dashboard endpoints"""
+    
+    def process_request(self, request):
+        # Only apply to analytics endpoints
+        if not request.path.startswith('/analytics/') and not request.path.startswith('/api/analytics/'):
+            return None
+        
+        # Add security headers
+        return None
+    
+    def process_response(self, request, response):
+        # Only apply to analytics endpoints
+        if not request.path.startswith('/analytics/') and not request.path.startswith('/api/analytics/'):
+            return response
+        
+        # Security headers for dashboard
+        response['X-Content-Type-Options'] = 'nosniff'
+        response['X-Frame-Options'] = 'DENY'
+        response['X-XSS-Protection'] = '1; mode=block'
+        response['Referrer-Policy'] = 'strict-origin-when-cross-origin'
+        
+        # CSP for dashboard
+        csp = (
+            "default-src 'self'; "
+            "script-src 'self' 'unsafe-inline'; "
+            "style-src 'self' 'unsafe-inline'; "
+            "img-src 'self' data:; "
+            "font-src 'self'; "
+            "connect-src 'self'; "
+            "frame-ancestors 'none';"
+        )
+        response['Content-Security-Policy'] = csp
+        
+        return response
+
+
+def dashboard_auth_required(view_func):
+    """
+    Decorator for dashboard views requiring authentication
+    """
+    @wraps(view_func)
+    def wrapper(request, *args, **kwargs):
+        from trafficmonitor.conf import TrafficMonitorConfig
+        
+        # Get user info from headers
+        user_info = TrafficMonitorConfig.get_user_info_from_request(request)
+        
+        if not TrafficMonitorConfig.is_user_authorized(user_info.get('role')):
+            logger.warning(f"Unauthorized dashboard access attempt from {request.META.get('REMOTE_ADDR')}")
+            
+            if request.path.startswith('/api/'):
+                return JsonResponse({'error': 'Unauthorized'}, status=401)
+            else:
+                return HttpResponseForbidden("Access denied. Required role not found in headers.")
+        
+        # Add user info to request for logging
+        request.trafficmonitor_user = user_info
+        
+        return view_func(request, *args, **kwargs)
+    
+    return wrapper
+
+
+def rate_limit_dashboard(max_requests=100, window_seconds=3600):
+    """
+    Simple rate limiting for dashboard endpoints
+    """
+    def decorator(view_func):
+        @wraps(view_func)
+        def wrapper(request, *args, **kwargs):
+            from django.core.cache import cache
+            
+            # Get client identifier
+            client_ip = request.META.get('REMOTE_ADDR', 'unknown')
+            user_id = getattr(request, 'trafficmonitor_user', {}).get('user_id', 'anonymous')
+            
+            cache_key = f"dashboard_rate_limit:{client_ip}:{user_id}"
+            
+            # Get current request count
+            current_requests = cache.get(cache_key, 0)
+            
+            if current_requests >= max_requests:
+                logger.warning(f"Rate limit exceeded for {client_ip}:{user_id}")
+                
+                if request.path.startswith('/api/'):
+                    return JsonResponse({'error': 'Rate limit exceeded'}, status=429)
+                else:
+                    return HttpResponseForbidden("Rate limit exceeded. Please try again later.")
+            
+            # Increment counter
+            cache.set(cache_key, current_requests + 1, timeout=window_seconds)
+            
+            return view_func(request, *args, **kwargs)
+        
+        return wrapper
+    return decorator

trafficmonitor/db_utils.py

@@ -0,0 +1,37 @@
+"""
+Database utilities for enterprise-grade performance
+"""
+from django.db import transaction
+from django.core.cache import cache
+from typing import List, Dict, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseManager:
+    """Handles database operations with connection pooling and bulk operations"""
+    
+    @staticmethod
+    @transaction.atomic
+    def bulk_create_logs(log_data: List[Dict[str, Any]], batch_size: int = 1000):
+        """Bulk create request logs with proper transaction handling"""
+        from trafficmonitor.models import RequestLog
+        
+        logs = []
+        for data in log_data:
+            logs.append(RequestLog(**data))
+            
+        # Use bulk_create with ignore_conflicts for high concurrency
+        RequestLog.objects.bulk_create(logs, batch_size=batch_size, ignore_conflicts=True)
+        
+    @staticmethod
+    def get_cached_analytics(cache_key: str, query_func, ttl: int = 300):
+        """Cache analytics queries with proper invalidation"""
+        cached_result = cache.get(cache_key)
+        if cached_result is not None:
+            return cached_result
+            
+        result = query_func()
+        cache.set(cache_key, result, ttl)
+        return result

trafficmonitor/exceptions.py

@@ -0,0 +1,93 @@
+"""
+Custom exceptions and error handling for TrafficMonitor
+"""
+import time
+import functools
+from typing import Callable, Any, Type, Tuple
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class TrafficMonitorException(Exception):
+    """Base exception for TrafficMonitor"""
+    pass
+
+
+class DatabaseConnectionError(TrafficMonitorException):
+    """Database connection related errors"""
+    pass
+
+
+class ConfigurationError(TrafficMonitorException):
+    """Configuration validation errors"""
+    pass
+
+
+class AuthenticationError(TrafficMonitorException):
+    """Authentication/authorization errors"""
+    pass
+
+
+def retry_on_exception(
+    exceptions: Tuple[Type[Exception], ...] = (Exception,),
+    max_attempts: int = 3,
+    delay: float = 1.0,
+    backoff: float = 2.0
+):
+    """
+    Decorator for retrying functions on specific exceptions
+    
+    Args:
+        exceptions: Tuple of exception types to retry on
+        max_attempts: Maximum number of retry attempts
+        delay: Initial delay between retries (seconds)
+        backoff: Multiplier for delay on each retry
+    """
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> Any:
+            current_delay = delay
+            
+            for attempt in range(max_attempts):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    if attempt == max_attempts - 1:
+                        logger.error(f"Function {func.__name__} failed after {max_attempts} attempts: {e}")
+                        raise
+                    
+                    logger.warning(f"Attempt {attempt + 1} failed for {func.__name__}: {e}. Retrying in {current_delay}s")
+                    time.sleep(current_delay)
+                    current_delay *= backoff
+            
+            return None  # Should never reach here
+        return wrapper
+    return decorator
+
+
+class ErrorHandler:
+    """Centralized error handling"""
+    
+    @staticmethod
+    def handle_database_error(error: Exception, context: dict) -> None:
+        """Handle database-related errors"""
+        logger.error("Database error occurred", extra={
+            "error_type": type(error).__name__,
+            "error_message": str(error),
+            "context": context
+        }, exc_info=True)
+        
+        # Could integrate with alerting system here
+        
+    @staticmethod
+    def handle_middleware_error(error: Exception, request) -> None:
+        """Handle middleware errors gracefully"""
+        logger.error("Middleware error occurred", extra={
+            "error_type": type(error).__name__,
+            "error_message": str(error),
+            "path": getattr(request, 'path', 'unknown'),
+            "method": getattr(request, 'method', 'unknown')
+        }, exc_info=True)
+        
+        # Don't let middleware errors break the request flow

trafficmonitor/health.py

@@ -0,0 +1,66 @@
+"""
+Health checks for TrafficMonitor
+"""
+from django.db import connection
+from django.core.cache import cache
+from django.http import JsonResponse
+from django.views import View
+from typing import Dict, Any
+import time
+
+
+class HealthCheckView(View):
+    """Health check endpoint for load balancers"""
+    
+    def get(self, request):
+        """Perform health checks"""
+        health_status = {
+            'status': 'healthy',
+            'timestamp': time.time(),
+            'checks': {}
+        }
+        
+        # Database check
+        try:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT 1")
+            health_status['checks']['database'] = {'status': 'healthy'}
+        except Exception as e:
+            health_status['checks']['database'] = {'status': 'unhealthy', 'error': str(e)}
+            health_status['status'] = 'unhealthy'
+        
+        # Cache check
+        try:
+            cache.set('health_check', 'ok', 30)
+            cache.get('health_check')
+            health_status['checks']['cache'] = {'status': 'healthy'}
+        except Exception as e:
+            health_status['checks']['cache'] = {'status': 'unhealthy', 'error': str(e)}
+            health_status['status'] = 'unhealthy'
+        
+        # Configuration check
+        from trafficmonitor.conf import TrafficMonitorConfig
+        config_issues = TrafficMonitorConfig.validate_configuration()
+        if config_issues:
+            health_status['checks']['configuration'] = {'status': 'unhealthy', 'issues': config_issues}
+            health_status['status'] = 'unhealthy'
+        else:
+            health_status['checks']['configuration'] = {'status': 'healthy'}
+        
+        status_code = 200 if health_status['status'] == 'healthy' else 503
+        return JsonResponse(health_status, status=status_code)
+
+
+class ReadinessCheckView(View):
+    """Readiness check for Kubernetes"""
+    
+    def get(self, request):
+        """Check if application is ready to serve traffic"""
+        from trafficmonitor.models import RequestLog
+        
+        try:
+            # Check if we can query the database
+            RequestLog.objects.first()
+            return JsonResponse({'status': 'ready'})
+        except Exception as e:
+            return JsonResponse({'status': 'not_ready', 'error': str(e)}, status=503)