setu-trafficmonitor 2.0.0__py3-none-any.whl

trafficmonitor/analytics/views.py

@@ -0,0 +1,694 @@
+from datetime import datetime, timedelta
+from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
+from django.db.models import (
+    Count, Avg, Max, Min, Q, F,
+    CharField, Value, IntegerField
+)
+from django.db.models.functions import (
+    TruncDate, TruncHour, TruncDay,
+    Concat, Cast
+)
+from django.utils import timezone
+from django.views.generic import TemplateView
+from django.core.cache import cache
+from django.http import JsonResponse
+from django.views.decorators.cache import cache_page
+from django.views.decorators.vary import vary_on_headers
+from rest_framework import status
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.permissions import IsAdminUser
+from rest_framework.response import Response
+import logging
+
+from trafficmonitor.models import RequestLog
+from trafficmonitor.permissions import HasTrafficMonitorAccess, HasTrafficMonitorAdminAccess
+from trafficmonitor.conf import TrafficMonitorConfig
+from trafficmonitor.monitoring import performance_timer, metrics
+
+logger = logging.getLogger(__name__)
+
+
+class RoleBasedAccessMixin:
+    """
+    Mixin for role-based access control using headers.
+    If no role header exists, allow access. Only restrict if role header is present.
+    """
+    def dispatch(self, request, *args, **kwargs):
+        role_header = request.META.get('HTTP_X_USER_ROLE')
+        
+        # If no role header, allow access
+        if not role_header:
+            return super().dispatch(request, *args, **kwargs)
+        
+        # If role header exists, check if it's authorized
+        authorized_roles = ['admin', 'staff', 'manager']  # Configure as needed
+        if role_header not in authorized_roles:
+            from django.http import HttpResponseForbidden
+            return HttpResponseForbidden("Access denied. Insufficient role permissions.")
+        
+        return super().dispatch(request, *args, **kwargs)
+
+
+class AnalyticsDashboardView(RoleBasedAccessMixin, TemplateView):
+    """
+    Main analytics dashboard view with role-based access control.
+    Displays enterprise-grade analytics for request logs.
+
+    Access controlled by X-User-Role header (configurable roles).
+    """
+    template_name = 'analytics/dashboard.html'
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+
+        # Get date range from query params or use defaults
+        date_range = self.request.GET.get('range', 'last_7_days')
+        context['selected_range'] = date_range
+
+        # Get filter parameters
+        method_filter = self.request.GET.get('method', '')
+        status_filter = self.request.GET.get('status', '')
+        path_filter = self.request.GET.get('path', '')
+        user_filter = self.request.GET.get('user', '')
+
+        context['filters'] = {
+            'method': method_filter,
+            'status': status_filter,
+            'path': path_filter,
+            'user': user_filter,
+        }
+
+        return context
+
+
+class AnalyticsQueryHelper:
+    """
+    Helper class for analytics queries with optimized database operations.
+    All queries use proper indexing and aggregation.
+    """
+
+    @staticmethod
+    def get_date_range(range_type='last_7_days', start_date=None, end_date=None):
+        """
+        Get start and end datetime for a given range type.
+
+        Args:
+            range_type: 'today', 'yesterday', 'last_7_days', 'last_30_days', 'custom'
+            start_date: For custom range (datetime object)
+            end_date: For custom range (datetime object)
+
+        Returns:
+            Tuple of (start_datetime, end_datetime)
+        """
+        now = timezone.now()
+        today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+
+        if range_type == 'today':
+            return today_start, now
+        elif range_type == 'yesterday':
+            yesterday_start = today_start - timedelta(days=1)
+            return yesterday_start, today_start
+        elif range_type == 'last_7_days':
+            return now - timedelta(days=7), now
+        elif range_type == 'last_30_days':
+            return now - timedelta(days=30), now
+        elif range_type == 'custom' and start_date and end_date:
+            return start_date, end_date
+        else:
+            # Default to last 7 days
+            return now - timedelta(days=7), now
+
+    @staticmethod
+    def get_base_queryset(start_date, end_date, method=None, status_code_range=None,
+                          path_contains=None, user_id=None, operation_type=None):
+        """
+        Get base queryset with filters applied.
+        Optimized for enterprise-level querying.
+
+        Args:
+            start_date: Start datetime
+            end_date: End datetime
+            method: HTTP method (GET, POST, etc.)
+            status_code_range: Tuple of (min, max) status codes
+            path_contains: String to filter paths
+            user_id: User ID to filter (requested_user_id from header)
+            operation_type: Operation type (READ, WRITE, DELETE)
+
+        Returns:
+            Filtered QuerySet
+        """
+        qs = RequestLog.objects.filter(
+            timestamp__gte=start_date,
+            timestamp__lte=end_date
+        )
+
+        if method:
+            qs = qs.filter(method=method)
+
+        if status_code_range:
+            min_code, max_code = status_code_range
+            qs = qs.filter(status_code__gte=min_code, status_code__lte=max_code)
+
+        if path_contains:
+            qs = qs.filter(path__icontains=path_contains)
+
+        if user_id:
+            qs = qs.filter(requested_user_id=user_id)
+        
+        if operation_type:
+            qs = qs.filter(operation_type=operation_type)
+
+        return qs
+
+    @staticmethod
+    def get_total_requests(start_date, end_date, **filters):
+        """
+        Get total request count for a date range.
+        Uses COUNT aggregation - very fast with indexed timestamp.
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+        return qs.count()
+
+    @staticmethod
+    def get_requests_by_status_code(start_date, end_date, **filters):
+        """
+        Get request counts grouped by status code.
+        Optimized with values() + annotate() pattern.
+
+        Returns:
+            List of dicts: [{'status_code': 200, 'count': 150}, ...]
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.values('status_code').annotate(
+            count=Count('id')
+        ).order_by('-count')
+
+        return list(results)
+
+    @staticmethod
+    def get_requests_by_method(start_date, end_date, **filters):
+        """
+        Get request counts grouped by HTTP method.
+
+        Returns:
+            List of dicts: [{'method': 'GET', 'count': 300}, ...]
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.values('method').annotate(
+            count=Count('id')
+        ).order_by('-count')
+
+        return list(results)
+
+    @staticmethod
+    def get_top_endpoints(start_date, end_date, limit=10, **filters):
+        """
+        Get top API endpoints by request count.
+        Includes average response time for each endpoint.
+
+        Returns:
+            List of dicts with path, count, avg_response_time
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.values('path').annotate(
+            count=Count('id'),
+            avg_response_time=Avg('response_time_ms'),
+            min_response_time=Min('response_time_ms'),
+            max_response_time=Max('response_time_ms')
+        ).order_by('-count')[:limit]
+
+        return list(results)
+
+    @staticmethod
+    def get_slowest_endpoints(start_date, end_date, limit=10, **filters):
+        """
+        Get slowest endpoints by average response time.
+        Filters out None response times.
+
+        Returns:
+            List of dicts with path, avg_response_time, count
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.filter(
+            response_time_ms__isnull=False
+        ).values('path').annotate(
+            avg_response_time=Avg('response_time_ms'),
+            count=Count('id'),
+            max_response_time=Max('response_time_ms')
+        ).order_by('-avg_response_time')[:limit]
+
+        return list(results)
+
+    @staticmethod
+    def get_top_ip_addresses(start_date, end_date, limit=10, **filters):
+        """
+        Get top IP addresses by request count.
+
+        Returns:
+            List of dicts: [{'ip_address': '1.2.3.4', 'count': 50}, ...]
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.filter(
+            ip_address__isnull=False
+        ).values('ip_address').annotate(
+            count=Count('id')
+        ).order_by('-count')[:limit]
+
+        return list(results)
+
+    @staticmethod
+    def get_requests_over_time(start_date, end_date, granularity='day', **filters):
+        """
+        Get request counts over time with specified granularity.
+        Uses TruncDate or TruncHour for efficient grouping.
+
+        Args:
+            granularity: 'hour' or 'day'
+
+        Returns:
+            List of dicts: [{'date': datetime, 'count': 100}, ...]
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        if granularity == 'hour':
+            results = qs.annotate(
+                period=TruncHour('timestamp')
+            ).values('period').annotate(
+                count=Count('id')
+            ).order_by('period')
+        else:
+            results = qs.annotate(
+                period=TruncDay('timestamp')
+            ).values('period').annotate(
+                count=Count('id')
+            ).order_by('period')
+
+        return list(results)
+
+    @staticmethod
+    def get_error_trend(start_date, end_date, **filters):
+        """
+        Get error trends over time (4xx and 5xx errors).
+        Groups by day and separates client vs server errors.
+
+        Returns:
+            List of dicts with date, client_errors (4xx), server_errors (5xx)
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.annotate(
+            period=TruncDay('timestamp')
+        ).values('period').annotate(
+            client_errors=Count('id', filter=Q(status_code__gte=400, status_code__lt=500)),
+            server_errors=Count('id', filter=Q(status_code__gte=500)),
+            total_errors=Count('id', filter=Q(status_code__gte=400))
+        ).order_by('period')
+
+        return list(results)
+
+    @staticmethod
+    def get_hourly_heatmap(start_date, end_date, **filters):
+        """
+        Get request volume per hour of day for heatmap visualization.
+        Groups by hour (0-23) across all days in range.
+
+        Returns:
+            List of dicts: [{'hour': 0, 'count': 120}, ...]
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        results = qs.annotate(
+            period=TruncHour('timestamp')
+        ).values('period').annotate(
+            count=Count('id')
+        ).order_by('period')
+
+        # Group by hour of day
+        hourly_data = {}
+        for item in results:
+            hour = item['period'].hour
+            hourly_data[hour] = hourly_data.get(hour, 0) + item['count']
+
+        # Convert to list format
+        return [{'hour': hour, 'count': count} for hour, count in sorted(hourly_data.items())]
+
+    @staticmethod
+    def get_status_code_summary(start_date, end_date, **filters):
+        """
+        Get summary of requests grouped by status code ranges.
+
+        Returns:
+            Dict with counts for success (2xx), redirect (3xx), client_error (4xx), server_error (5xx)
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        summary = qs.aggregate(
+            success=Count('id', filter=Q(status_code__gte=200, status_code__lt=300)),
+            redirect=Count('id', filter=Q(status_code__gte=300, status_code__lt=400)),
+            client_error=Count('id', filter=Q(status_code__gte=400, status_code__lt=500)),
+            server_error=Count('id', filter=Q(status_code__gte=500)),
+            total=Count('id')
+        )
+
+        return summary
+
+    @staticmethod
+    def get_performance_summary(start_date, end_date, **filters):
+        """
+        Get performance metrics summary.
+
+        Returns:
+            Dict with avg_response_time, max_response_time, avg_query_count
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        summary = qs.aggregate(
+            avg_response_time=Avg('response_time_ms', filter=Q(response_time_ms__isnull=False)),
+            max_response_time=Max('response_time_ms'),
+            min_response_time=Min('response_time_ms'),
+            avg_query_count=Avg('query_count', filter=Q(query_count__isnull=False)),
+            max_query_count=Max('query_count')
+        )
+
+        return summary
+
+    @staticmethod
+    def get_user_activity(start_date, end_date, limit=10, **filters):
+        """
+        Get top users by request count using requested_user_id from headers.
+
+        Returns:
+            List of dicts with user info and request count
+        """
+        qs = AnalyticsQueryHelper.get_base_queryset(start_date, end_date, **filters)
+
+        # Users with IDs
+        identified_users = qs.filter(
+            requested_user_id__isnull=False
+        ).values('requested_user_id').annotate(
+            count=Count('id')
+        ).order_by('-count')[:limit]
+
+        # Anonymous requests count
+        anonymous_count = qs.filter(requested_user_id__isnull=True).count()
+
+        return {
+            'identified': list(identified_users),
+            'anonymous_count': anonymous_count
+        }
+
+    @staticmethod
+    def get_comprehensive_analytics(start_date, end_date, **filters):
+        """
+        Get all analytics data in a single optimized set of queries.
+
+        Returns:
+            Dict containing all analytics data
+        """
+        # Get total counts for different periods
+        now = timezone.now()
+        today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+
+        data = {
+            # Total counts by period
+            'totals': {
+                'today': AnalyticsQueryHelper.get_total_requests(
+                    today_start, now, **filters
+                ),
+                'last_7_days': AnalyticsQueryHelper.get_total_requests(
+                    now - timedelta(days=7), now, **filters
+                ),
+                'last_30_days': AnalyticsQueryHelper.get_total_requests(
+                    now - timedelta(days=30), now, **filters
+                ),
+                'selected_range': AnalyticsQueryHelper.get_total_requests(
+                    start_date, end_date, **filters
+                ),
+            },
+
+            # Status code breakdown
+            'status_summary': AnalyticsQueryHelper.get_status_code_summary(
+                start_date, end_date, **filters
+            ),
+
+            # Status codes detail
+            'status_codes': AnalyticsQueryHelper.get_requests_by_status_code(
+                start_date, end_date, **filters
+            ),
+
+            # HTTP methods
+            'methods': AnalyticsQueryHelper.get_requests_by_method(
+                start_date, end_date, **filters
+            ),
+
+            # Top endpoints
+            'top_endpoints': AnalyticsQueryHelper.get_top_endpoints(
+                start_date, end_date, limit=10, **filters
+            ),
+
+            # Slowest endpoints
+            'slowest_endpoints': AnalyticsQueryHelper.get_slowest_endpoints(
+                start_date, end_date, limit=10, **filters
+            ),
+
+            # Top IPs
+            'top_ips': AnalyticsQueryHelper.get_top_ip_addresses(
+                start_date, end_date, limit=10, **filters
+            ),
+
+            # Performance metrics
+            'performance': AnalyticsQueryHelper.get_performance_summary(
+                start_date, end_date, **filters
+            ),
+
+            # User activity
+            'users': AnalyticsQueryHelper.get_user_activity(
+                start_date, end_date, limit=10, **filters
+            ),
+
+            # Time series data
+            'requests_over_time': AnalyticsQueryHelper.get_requests_over_time(
+                start_date, end_date, granularity='day', **filters
+            ),
+
+            # Error trends
+            'error_trend': AnalyticsQueryHelper.get_error_trend(
+                start_date, end_date, **filters
+            ),
+
+            # Hourly heatmap
+            'hourly_heatmap': AnalyticsQueryHelper.get_hourly_heatmap(
+                start_date, end_date, **filters
+            ),
+        }
+
+        return data
+
+
+@api_view(['GET'])
+@permission_classes([HasTrafficMonitorAccess])
+def analytics_overview_api(request):
+    """
+    DRF API endpoint for comprehensive analytics data.
+    Returns enterprise-grade analytics with Read/Write insights.
+
+    Requires X-User-Role header with authorized role.
+
+    Query Parameters:
+        - range: 'today', 'yesterday', 'last_7_days', 'last_30_days', 'custom'
+        - start_date: For custom range (ISO format: 2024-01-01)
+        - end_date: For custom range (ISO format: 2024-01-31)
+        - method: Filter by HTTP method (GET, POST, etc.)
+        - status: Filter by status code (200, 404, etc.)
+        - path: Filter by path contains
+        - user: Filter by requested_user_id
+        - operation_type: Filter by operation (READ, WRITE, DELETE)
+
+    Returns:
+        JSON response with all analytics data including Read/Write metrics
+    """
+    # Parse query parameters
+    range_type = request.GET.get('range', 'last_7_days')
+
+    # Handle custom date range
+    start_date = None
+    end_date = None
+    if range_type == 'custom':
+        start_str = request.GET.get('start_date')
+        end_str = request.GET.get('end_date')
+
+        if start_str and end_str:
+            try:
+                start_date = timezone.make_aware(
+                    datetime.strptime(start_str, '%Y-%m-%d')
+                )
+                end_date = timezone.make_aware(
+                    datetime.strptime(end_str, '%Y-%m-%d').replace(
+                        hour=23, minute=59, second=59
+                    )
+                )
+            except ValueError:
+                return Response(
+                    {'error': 'Invalid date format. Use YYYY-MM-DD'},
+                    status=status.HTTP_400_BAD_REQUEST
+                )
+        else:
+            return Response(
+                {'error': 'start_date and end_date required for custom range'},
+                status=status.HTTP_400_BAD_REQUEST
+            )
+
+    # Get date range
+    start_date, end_date = AnalyticsQueryHelper.get_date_range(
+        range_type, start_date, end_date
+    )
+
+    # Build filters dict
+    filters = {}
+    if request.GET.get('method'):
+        filters['method'] = request.GET.get('method')
+    if request.GET.get('status'):
+        try:
+            filters['status_code_range'] = (
+                int(request.GET.get('status')),
+                int(request.GET.get('status'))
+            )
+        except ValueError:
+            pass
+    if request.GET.get('path'):
+        filters['path_contains'] = request.GET.get('path')
+    if request.GET.get('user'):
+        filters['user_id'] = request.GET.get('user')
+    if request.GET.get('operation_type'):
+        filters['operation_type'] = request.GET.get('operation_type')
+
+    # Get comprehensive analytics
+    analytics_data = AnalyticsQueryHelper.get_comprehensive_analytics(
+        start_date, end_date, **filters
+    )
+    
+    # Add enhanced enterprise metrics
+    from trafficmonitor.analytics.enhanced_queries import EnhancedAnalyticsQueries
+    
+    analytics_data['read_write_summary'] = EnhancedAnalyticsQueries.get_read_write_summary(
+        start_date, end_date, **filters
+    )
+    analytics_data['read_write_over_time'] = EnhancedAnalyticsQueries.get_read_write_over_time(
+        start_date, end_date, **filters
+    )
+    analytics_data['api_health'] = EnhancedAnalyticsQueries.get_api_health_metrics(
+        start_date, end_date, **filters
+    )
+    analytics_data['endpoint_categories'] = EnhancedAnalyticsQueries.get_endpoint_category_breakdown(
+        start_date, end_date, **filters
+    )
+    analytics_data['user_activity_detailed'] = EnhancedAnalyticsQueries.get_user_activity_detailed(
+        start_date, end_date, limit=20, **filters
+    )
+    analytics_data['performance_outliers'] = EnhancedAnalyticsQueries.get_performance_outliers(
+        start_date, end_date, **filters
+    )
+    analytics_data['throughput'] = EnhancedAnalyticsQueries.get_throughput_metrics(
+        start_date, end_date, **filters
+    )
+
+    # Add metadata
+    analytics_data['metadata'] = {
+        'range_type': range_type,
+        'start_date': start_date.isoformat(),
+        'end_date': end_date.isoformat(),
+        'filters': filters,
+        'generated_at': timezone.now().isoformat(),
+    }
+
+    return Response(analytics_data, status=status.HTTP_200_OK)
+
+
+@api_view(['GET'])
+@permission_classes([HasTrafficMonitorAccess])
+def analytics_chart_data_api(request, chart_type):
+    """
+    API endpoint for specific chart data.
+    Allows fetching individual chart data without full analytics.
+
+    Requires X-User-Role header with authorized role.
+
+    Args:
+        chart_type: 'time-series', 'status-codes', 'methods', 'endpoints',
+                   'performance', 'heatmap', 'errors', 'read-write', 'api-health'
+
+    Returns:
+        JSON response with chart-specific data
+    """
+    # Get date range
+    range_type = request.GET.get('range', 'last_7_days')
+    start_date, end_date = AnalyticsQueryHelper.get_date_range(range_type)
+
+    # Build filters
+    filters = {}
+    if request.GET.get('method'):
+        filters['method'] = request.GET.get('method')
+    if request.GET.get('path'):
+        filters['path_contains'] = request.GET.get('path')
+
+    # Import enhanced queries
+    from trafficmonitor.analytics.enhanced_queries import EnhancedAnalyticsQueries
+    
+    # Route to appropriate query based on chart type
+    chart_data_map = {
+        'time-series': lambda: AnalyticsQueryHelper.get_requests_over_time(
+            start_date, end_date, **filters
+        ),
+        'status-codes': lambda: AnalyticsQueryHelper.get_requests_by_status_code(
+            start_date, end_date, **filters
+        ),
+        'methods': lambda: AnalyticsQueryHelper.get_requests_by_method(
+            start_date, end_date, **filters
+        ),
+        'endpoints': lambda: AnalyticsQueryHelper.get_top_endpoints(
+            start_date, end_date, **filters
+        ),
+        'performance': lambda: AnalyticsQueryHelper.get_slowest_endpoints(
+            start_date, end_date, **filters
+        ),
+        'heatmap': lambda: AnalyticsQueryHelper.get_hourly_heatmap(
+            start_date, end_date, **filters
+        ),
+        'errors': lambda: AnalyticsQueryHelper.get_error_trend(
+            start_date, end_date, **filters
+        ),
+        'read-write': lambda: EnhancedAnalyticsQueries.get_read_write_over_time(
+            start_date, end_date, **filters
+        ),
+        'api-health': lambda: EnhancedAnalyticsQueries.get_api_health_metrics(
+            start_date, end_date, **filters
+        ),
+        'endpoint-categories': lambda: EnhancedAnalyticsQueries.get_endpoint_category_breakdown(
+            start_date, end_date, **filters
+        ),
+        'throughput': lambda: EnhancedAnalyticsQueries.get_throughput_metrics(
+            start_date, end_date, **filters
+        ),
+    }
+
+    if chart_type not in chart_data_map:
+        return Response(
+            {'error': f'Invalid chart type: {chart_type}'},
+            status=status.HTTP_400_BAD_REQUEST
+        )
+
+    data = chart_data_map[chart_type]()
+
+    return Response({
+        'chart_type': chart_type,
+        'data': data,
+        'metadata': {
+            'start_date': start_date.isoformat(),
+            'end_date': end_date.isoformat(),
+        }
+    }, status=status.HTTP_200_OK)

trafficmonitor/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class TrafficmonitorConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'trafficmonitor'
+    verbose_name = 'Traffic Monitor'