django-cfg 1.5.1__py3-none-any.whl → 1.5.3__py3-none-any.whl

django_cfg/apps/rq/views/monitoring.py

@@ -0,0 +1,248 @@
+"""
+Django-RQ Monitoring ViewSet.
+
+Provides REST API endpoints for RQ health checks and configuration.
+"""
+
+from datetime import datetime
+
+from django_cfg.mixins import AdminAPIMixin
+from django_cfg.modules.django_logging import get_logger
+from drf_spectacular.utils import extend_schema
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+
+from ..serializers import HealthCheckSerializer, RQConfigSerializer
+
+logger = get_logger("rq.monitoring")
+
+
+class RQMonitorViewSet(AdminAPIMixin, viewsets.ViewSet):
+    """
+    ViewSet for RQ monitoring and health checks.
+
+    Provides endpoints for:
+    - Health check (cluster status)
+    - Configuration view
+    - Prometheus metrics
+
+    Requires admin authentication (JWT, Session, or Basic Auth).
+    """
+
+    @extend_schema(
+        tags=["RQ Monitoring"],
+        summary="Health check",
+        description="Returns RQ cluster health status including worker count and queue status.",
+        responses={
+            200: HealthCheckSerializer,
+        },
+    )
+    @action(detail=False, methods=["get"], url_path="health")
+    def health(self, request):
+        """Health check endpoint."""
+        try:
+            import django_rq
+            from rq import Worker
+
+            # Check Redis connection
+            redis_connected = True
+            worker_count = 0
+            total_jobs = 0
+            queue_count = 0
+
+            try:
+                # Get all queues
+                from django.conf import settings
+
+                queue_names = settings.RQ_QUEUES.keys() if hasattr(settings, 'RQ_QUEUES') else []
+                queue_count = len(queue_names)
+
+                # Count workers per queue (use set to avoid duplicates)
+                worker_names = set()
+                for queue_name in queue_names:
+                    try:
+                        queue = django_rq.get_queue(queue_name)
+                        # Get workers for this queue's connection
+                        queue_workers = Worker.all(connection=queue.connection)
+                        worker_names.update(w.name for w in queue_workers)
+                        # Count jobs
+                        total_jobs += queue.count
+                    except Exception as e:
+                        logger.debug(f"Failed to get queue {queue_name}: {e}")
+
+                worker_count = len(worker_names)
+
+            except Exception as e:
+                logger.error(f"Redis connection error: {e}", exc_info=True)
+                redis_connected = False
+
+            # Determine health status
+            if not redis_connected:
+                health_status = "unhealthy"
+            elif worker_count == 0:
+                health_status = "degraded"
+            else:
+                health_status = "healthy"
+
+            health_data = {
+                "status": health_status,
+                "worker_count": worker_count,
+                "queue_count": queue_count,
+                "total_jobs": total_jobs,
+                "timestamp": datetime.now().isoformat(),
+                "enabled": True,
+                "redis_connected": redis_connected,
+                "wrapper_url": "",  # Empty for now, can be configured later
+                "has_api_key": False,  # No API key required for RQ
+            }
+
+            serializer = HealthCheckSerializer(data=health_data)
+            serializer.is_valid(raise_exception=True)
+            return Response(serializer.data)
+
+        except Exception as e:
+            logger.error(f"Health check error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+    @extend_schema(
+        tags=["RQ Monitoring"],
+        summary="Get RQ configuration",
+        description="Returns current RQ configuration from django-cfg.",
+        responses={
+            200: RQConfigSerializer,
+        },
+    )
+    @action(detail=False, methods=["get"], url_path="config")
+    def config(self, request):
+        """Get RQ configuration."""
+        try:
+            from django.conf import settings
+            from ..services import get_rq_config
+
+            # Get configuration
+            queues = {}
+            if hasattr(settings, 'RQ_QUEUES'):
+                # Sanitize queue config (hide passwords)
+                for name, config in settings.RQ_QUEUES.items():
+                    queues[name] = {
+                        k: v if k not in ('PASSWORD', 'password') else '***'
+                        for k, v in config.items()
+                    }
+
+            async_mode = True
+            show_admin_link = getattr(settings, 'RQ_SHOW_ADMIN_LINK', True)
+            api_token_configured = bool(getattr(settings, 'RQ_API_TOKEN', None))
+
+            # Check prometheus
+            prometheus_enabled = True
+            try:
+                import prometheus_client  # noqa: F401
+            except ImportError:
+                prometheus_enabled = False
+
+            # Get scheduled tasks from django-cfg config
+            schedules = []
+            rq_config = get_rq_config()
+            if rq_config and hasattr(rq_config, 'schedules'):
+                for schedule in rq_config.schedules:
+                    schedules.append({
+                        "func": schedule.func,
+                        "queue": schedule.queue,
+                        "cron": schedule.cron,
+                        "interval": schedule.interval,
+                        "scheduled_time": schedule.scheduled_time,
+                        "description": schedule.description,
+                        "timeout": schedule.timeout,
+                        "result_ttl": schedule.result_ttl,
+                        "repeat": schedule.repeat,
+                    })
+
+            config_data = {
+                "enabled": True,
+                "queues": queues,
+                "async_mode": async_mode,
+                "show_admin_link": show_admin_link,
+                "prometheus_enabled": prometheus_enabled,
+                "api_token_configured": api_token_configured,
+                "schedules": schedules,
+            }
+
+            serializer = RQConfigSerializer(data=config_data)
+            serializer.is_valid(raise_exception=True)
+            return Response(serializer.data)
+
+        except Exception as e:
+            logger.error(f"Config error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+    @extend_schema(
+        tags=["RQ Monitoring"],
+        summary="Prometheus metrics",
+        description="Returns Prometheus metrics for RQ queues and workers.",
+        responses={
+            200: {"description": "Prometheus metrics in text format"},
+        },
+    )
+    @action(detail=False, methods=["get"], url_path="metrics")
+    def metrics(self, request):
+        """
+        Prometheus metrics endpoint.
+
+        Uses django-rq's built-in RQCollector for comprehensive metrics:
+        - rq_workers: Worker count by name, state, and queues
+        - rq_job_successful_total: Successful job count per worker
+        - rq_job_failed_total: Failed job count per worker
+        - rq_working_seconds_total: Total working time per worker
+        - rq_jobs: Job count by queue and status (queued, started, finished, failed, deferred, scheduled)
+        """
+        try:
+            # Try to import prometheus_client and RQCollector
+            try:
+                from prometheus_client import CollectorRegistry
+                from prometheus_client.exposition import choose_encoder
+            except ImportError:
+                return Response(
+                    {"error": "prometheus_client not installed. Install with: pip install django-rq[prometheus]"},
+                    status=status.HTTP_501_NOT_IMPLEMENTED,
+                )
+
+            try:
+                from django_rq.contrib.prometheus import RQCollector
+            except ImportError:
+                return Response(
+                    {"error": "RQCollector not available. Ensure prometheus_client is installed."},
+                    status=status.HTTP_501_NOT_IMPLEMENTED,
+                )
+
+            # Create registry and register RQCollector
+            registry = CollectorRegistry(auto_describe=True)
+            registry.register(RQCollector())
+
+            # Choose encoder based on Accept header
+            encoder, content_type = choose_encoder(request.META.get('HTTP_ACCEPT', ''))
+
+            # Support filtering by metric name (Prometheus query param)
+            if 'name[]' in request.GET:
+                registry = registry.restricted_registry(request.GET.getlist('name[]'))
+
+            # Generate and return metrics
+            metrics_output = encoder(registry)
+
+            return Response(
+                metrics_output,
+                content_type=content_type,
+            )
+
+        except Exception as e:
+            logger.error(f"Metrics error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )

django_cfg/apps/rq/views/queues.py

@@ -0,0 +1,261 @@
+"""
+Django-RQ Queue Management ViewSet.
+
+Provides REST API endpoints for managing RQ queues.
+"""
+
+from django_cfg.mixins import AdminAPIMixin
+from django_cfg.modules.django_logging import get_logger
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiParameter, extend_schema
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+
+from ..serializers import QueueStatsSerializer, QueueDetailSerializer
+from ..services import queue_to_model
+
+logger = get_logger("rq.queues")
+
+
+class QueueViewSet(AdminAPIMixin, viewsets.ViewSet):
+    """
+    ViewSet for RQ queue management.
+
+    Provides endpoints for:
+    - Listing all queues with statistics
+    - Getting detailed queue information
+    - Emptying queues
+    - Getting queue job lists
+
+    Requires admin authentication (JWT, Session, or Basic Auth).
+    """
+
+    @extend_schema(
+        tags=["RQ Queues"],
+        summary="List all queues",
+        description="Returns list of all configured RQ queues with statistics. Supports filtering by queue name.",
+        parameters=[
+            OpenApiParameter(
+                name="name",
+                type=str,
+                location=OpenApiParameter.QUERY,
+                required=False,
+                description="Filter by queue name (exact match or substring)",
+            ),
+        ],
+        responses={
+            200: QueueStatsSerializer(many=True),
+        },
+    )
+    def list(self, request):
+        """List all queues with statistics and optional filtering."""
+        try:
+            import django_rq
+            from django.conf import settings
+
+            if not hasattr(settings, 'RQ_QUEUES'):
+                return Response([])
+
+            # Get query params for filtering
+            name_filter = request.query_params.get('name')
+
+            queues_data = []
+
+            for queue_name in settings.RQ_QUEUES.keys():
+                try:
+                    # Apply name filter
+                    if name_filter and name_filter.lower() not in queue_name.lower():
+                        continue
+
+                    queue = django_rq.get_queue(queue_name)
+
+                    # Convert RQ Queue to Pydantic model
+                    queue_model = queue_to_model(queue, queue_name)
+
+                    # Convert to dict for DRF serializer (only stats fields)
+                    queue_dict = {
+                        "name": queue_model.name,
+                        "count": queue_model.count,
+                        "queued_jobs": queue_model.queued_jobs,
+                        "started_jobs": queue_model.started_jobs,
+                        "finished_jobs": queue_model.finished_jobs,
+                        "failed_jobs": queue_model.failed_jobs,
+                        "deferred_jobs": queue_model.deferred_jobs,
+                        "scheduled_jobs": queue_model.scheduled_jobs,
+                        "workers": queue_model.workers,
+                    }
+
+                    serializer = QueueStatsSerializer(data=queue_dict)
+                    serializer.is_valid(raise_exception=True)
+                    queues_data.append(serializer.data)
+
+                except Exception as e:
+                    logger.error(f"Failed to get queue {queue_name}: {e}", exc_info=True)
+                    continue
+
+            return Response(queues_data)
+
+        except Exception as e:
+            logger.error(f"Queue list error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+    @extend_schema(
+        tags=["RQ Queues"],
+        summary="Get queue details",
+        description="Returns detailed information about a specific queue.",
+        responses={
+            200: QueueDetailSerializer,
+            404: {"description": "Queue not found"},
+        },
+    )
+    def retrieve(self, request, pk=None):
+        """Get detailed queue information."""
+        try:
+            import django_rq
+
+            try:
+                queue = django_rq.get_queue(pk)
+            except Exception:
+                return Response(
+                    {"error": f"Queue {pk} not found"},
+                    status=status.HTTP_404_NOT_FOUND,
+                )
+
+            # Convert RQ Queue to Pydantic model
+            queue_model = queue_to_model(queue, pk)
+
+            # Convert to dict for DRF serializer
+            queue_dict = {
+                "name": queue_model.name,
+                "count": queue_model.count,
+                "queued_jobs": queue_model.queued_jobs,
+                "started_jobs": queue_model.started_jobs,
+                "finished_jobs": queue_model.finished_jobs,
+                "failed_jobs": queue_model.failed_jobs,
+                "deferred_jobs": queue_model.deferred_jobs,
+                "scheduled_jobs": queue_model.scheduled_jobs,
+                "workers": queue_model.workers,
+                "oldest_job_timestamp": queue_model.oldest_job_timestamp,
+                "connection_kwargs": {
+                    'host': queue_model.connection_host,
+                    'port': queue_model.connection_port,
+                    'db': queue_model.connection_db,
+                },
+                "is_async": queue_model.is_async,
+            }
+
+            serializer = QueueDetailSerializer(data=queue_dict)
+            serializer.is_valid(raise_exception=True)
+            return Response(serializer.data)
+
+        except Exception as e:
+            logger.error(f"Queue detail error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+    @extend_schema(
+        tags=["RQ Queues"],
+        summary="Empty queue",
+        description="Removes all jobs from the specified queue.",
+        responses={
+            200: {"description": "Queue emptied successfully"},
+            404: {"description": "Queue not found"},
+        },
+    )
+    @action(detail=True, methods=["post"], url_path="empty")
+    def empty(self, request, pk=None):
+        """Empty queue (remove all jobs)."""
+        try:
+            import django_rq
+
+            try:
+                queue = django_rq.get_queue(pk)
+            except Exception:
+                return Response(
+                    {"error": f"Queue {pk} not found"},
+                    status=status.HTTP_404_NOT_FOUND,
+                )
+
+            job_count = queue.count
+            queue.empty()
+
+            return Response({
+                "success": True,
+                "message": f"Emptied queue '{pk}', removed {job_count} jobs",
+                "queue": pk,
+                "removed_jobs": job_count,
+            })
+
+        except Exception as e:
+            logger.error(f"Queue empty error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+    @extend_schema(
+        tags=["RQ Queues"],
+        summary="Get queue jobs",
+        description="Returns list of job IDs in the queue.",
+        parameters=[
+            OpenApiParameter(
+                name="limit",
+                type=OpenApiTypes.INT,
+                location=OpenApiParameter.QUERY,
+                description="Number of jobs to return (default: 100)",
+                required=False,
+            ),
+            OpenApiParameter(
+                name="offset",
+                type=OpenApiTypes.INT,
+                location=OpenApiParameter.QUERY,
+                description="Offset for pagination (default: 0)",
+                required=False,
+            ),
+        ],
+        responses={
+            200: {"description": "List of job IDs"},
+            404: {"description": "Queue not found"},
+        },
+    )
+    @action(detail=True, methods=["get"], url_path="jobs")
+    def jobs(self, request, pk=None):
+        """Get list of jobs in queue."""
+        try:
+            import django_rq
+
+            try:
+                queue = django_rq.get_queue(pk)
+            except Exception:
+                return Response(
+                    {"error": f"Queue {pk} not found"},
+                    status=status.HTTP_404_NOT_FOUND,
+                )
+
+            # Pagination
+            limit = min(int(request.query_params.get("limit", 100)), 500)
+            offset = int(request.query_params.get("offset", 0))
+
+            # Get job IDs
+            job_ids = queue.get_job_ids(offset, offset + limit)
+
+            return Response({
+                "queue": pk,
+                "total_jobs": queue.count,
+                "limit": limit,
+                "offset": offset,
+                "job_ids": job_ids,
+            })
+
+        except Exception as e:
+            logger.error(f"Queue jobs error: {e}", exc_info=True)
+            return Response(
+                {"error": "Internal server error"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )