ccproxy-api 0.1.0__py3-none-any.whl

ccproxy/observability/pushgateway.py

@@ -0,0 +1,366 @@
+"""Prometheus Pushgateway integration for batch metrics."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import time
+from typing import Any
+
+from structlog import get_logger
+
+from ccproxy.config.observability import ObservabilitySettings
+
+
+logger = get_logger(__name__)
+
+
+# Import prometheus_client with graceful degradation (matching existing metrics.py pattern)
+try:
+    from prometheus_client import (
+        CollectorRegistry,
+        delete_from_gateway,
+        push_to_gateway,
+        pushadd_to_gateway,
+    )
+
+    PROMETHEUS_AVAILABLE = True
+except ImportError:
+    PROMETHEUS_AVAILABLE = False
+
+    # Dummy classes for graceful degradation
+    def push_to_gateway(*args: Any, **kwargs: Any) -> None:  # type: ignore[misc]
+        pass
+
+    def pushadd_to_gateway(*args: Any, **kwargs: Any) -> None:  # type: ignore[misc]
+        pass
+
+    def delete_from_gateway(*args: Any, **kwargs: Any) -> None:  # type: ignore[misc]
+        pass
+
+    class CollectorRegistry:  # type: ignore[no-redef]
+        pass
+
+
+class CircuitBreaker:
+    """Simple circuit breaker for pushgateway operations."""
+
+    def __init__(self, failure_threshold: int = 5, recovery_timeout: float = 60.0):
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.failure_count = 0
+        self.last_failure_time = 0.0
+        self.state = "CLOSED"  # CLOSED, OPEN, HALF_OPEN
+
+    def can_execute(self) -> bool:
+        """Check if operation can be executed."""
+        if self.state == "CLOSED":
+            return True
+        elif self.state == "OPEN":
+            if time.time() - self.last_failure_time > self.recovery_timeout:
+                self.state = "HALF_OPEN"
+                return True
+            return False
+        else:  # HALF_OPEN
+            return True
+
+    def record_success(self) -> None:
+        """Record successful operation."""
+        self.failure_count = 0
+        self.state = "CLOSED"
+
+    def record_failure(self) -> None:
+        """Record failed operation."""
+        self.failure_count += 1
+        self.last_failure_time = time.time()
+
+        if self.failure_count >= self.failure_threshold:
+            self.state = "OPEN"
+            logger.warning(
+                "pushgateway_circuit_breaker_opened",
+                failure_count=self.failure_count,
+                recovery_timeout=self.recovery_timeout,
+            )
+
+
+class PushgatewayClient:
+    """Prometheus Pushgateway client using official prometheus_client methods.
+
+    Supports standard pushgateway operations:
+    - push_to_gateway(): Replace all metrics for job/instance
+    - pushadd_to_gateway(): Add metrics to existing job/instance
+    - delete_from_gateway(): Delete metrics for job/instance
+
+    Also supports VictoriaMetrics remote write protocol for compatibility.
+    """
+
+    def __init__(self, settings: ObservabilitySettings) -> None:
+        """Initialize Pushgateway client.
+
+        Args:
+            settings: Observability configuration settings
+        """
+        self.settings = settings
+        # Pushgateway is enabled if URL is configured and prometheus_client is available
+        self._enabled = PROMETHEUS_AVAILABLE and bool(settings.pushgateway_url)
+        self._circuit_breaker = CircuitBreaker(
+            failure_threshold=5,
+            recovery_timeout=60.0,
+        )
+
+        # Only log if pushgateway URL is configured but prometheus is not available
+        if settings.pushgateway_url and not PROMETHEUS_AVAILABLE:
+            logger.warning(
+                "prometheus_client not available. Pushgateway will be disabled. "
+                "Install with: pip install prometheus-client"
+            )
+
+    def push_metrics(self, registry: CollectorRegistry, method: str = "push") -> bool:
+        """Push metrics to Pushgateway using official prometheus_client methods.
+
+        Args:
+            registry: Prometheus metrics registry to push
+            method: Push method - "push" (replace), "pushadd" (add), or "delete"
+
+        Returns:
+            True if push succeeded, False otherwise
+        """
+
+        if not self._enabled or not self.settings.pushgateway_url:
+            return False
+
+        # Check circuit breaker before attempting operation
+        if not self._circuit_breaker.can_execute():
+            logger.debug(
+                "pushgateway_circuit_breaker_blocking",
+                state=self._circuit_breaker.state,
+                failure_count=self._circuit_breaker.failure_count,
+            )
+            return False
+
+        try:
+            # Check if URL looks like VictoriaMetrics remote write endpoint
+            if "/api/v1/write" in self.settings.pushgateway_url:
+                success = self._push_remote_write(registry)
+            else:
+                success = self._push_standard(registry, method)
+
+            if success:
+                self._circuit_breaker.record_success()
+            else:
+                self._circuit_breaker.record_failure()
+
+            return success
+
+        except Exception as e:
+            self._circuit_breaker.record_failure()
+            logger.error(
+                "pushgateway_push_failed",
+                url=self.settings.pushgateway_url,
+                job=self.settings.pushgateway_job,
+                method=method,
+                error=str(e),
+                error_type=type(e).__name__,
+            )
+            return False
+
+    def _push_standard(self, registry: CollectorRegistry, method: str = "push") -> bool:
+        """Push using standard Prometheus pushgateway protocol with official client methods.
+
+        Args:
+            registry: Prometheus metrics registry
+            method: Push method - "push" (replace), "pushadd" (add), or "delete"
+        """
+        if not self.settings.pushgateway_url:
+            return False
+
+        try:
+            # Use the appropriate prometheus_client function based on method
+            if method == "push":
+                push_to_gateway(
+                    gateway=self.settings.pushgateway_url,
+                    job=self.settings.pushgateway_job,
+                    registry=registry,
+                )
+            elif method == "pushadd":
+                pushadd_to_gateway(
+                    gateway=self.settings.pushgateway_url,
+                    job=self.settings.pushgateway_job,
+                    registry=registry,
+                )
+            elif method == "delete":
+                delete_from_gateway(
+                    gateway=self.settings.pushgateway_url,
+                    job=self.settings.pushgateway_job,
+                )
+            else:
+                logger.error("pushgateway_invalid_method", method=method)
+                return False
+
+            logger.debug(
+                "pushgateway_push_success",
+                url=self.settings.pushgateway_url,
+                job=self.settings.pushgateway_job,
+                protocol="standard",
+                method=method,
+            )
+            return True
+
+        except Exception as e:
+            logger.error(
+                "pushgateway_standard_push_failed",
+                url=self.settings.pushgateway_url,
+                job=self.settings.pushgateway_job,
+                method=method,
+                error=str(e),
+                error_type=type(e).__name__,
+            )
+            return False
+
+    def _push_remote_write(self, registry: CollectorRegistry) -> bool:
+        """Push using VictoriaMetrics import protocol for exposition format data.
+
+        VictoriaMetrics supports importing Prometheus exposition format data
+        via the /api/v1/import/prometheus endpoint, which is simpler than
+        the full remote write protocol that requires protobuf encoding.
+        """
+        import httpx
+        from prometheus_client.exposition import generate_latest
+
+        if not self.settings.pushgateway_url:
+            return False
+
+        # Generate metrics in Prometheus exposition format
+        metrics_data = generate_latest(registry)
+
+        # Convert /api/v1/write URL to /api/v1/import/prometheus for VictoriaMetrics
+        # This endpoint accepts Prometheus exposition format directly
+        if "/api/v1/write" in self.settings.pushgateway_url:
+            import_url = self.settings.pushgateway_url.replace(
+                "/api/v1/write", "/api/v1/import/prometheus"
+            )
+        else:
+            # Fallback - assume it's already the correct import URL
+            import_url = self.settings.pushgateway_url
+
+        try:
+            # VictoriaMetrics import endpoint accepts text/plain exposition format
+            response = httpx.post(
+                import_url,
+                content=metrics_data,
+                headers={
+                    "Content-Type": "text/plain; charset=utf-8",
+                    "User-Agent": "ccproxy-pushgateway-client/1.0",
+                },
+                timeout=30,
+            )
+
+            if response.status_code in (200, 204):
+                logger.debug(
+                    "pushgateway_import_success",
+                    url=import_url,
+                    job=self.settings.pushgateway_job,
+                    protocol="victoriametrics_import",
+                    status=response.status_code,
+                )
+                return True
+            else:
+                logger.error(
+                    "pushgateway_import_failed",
+                    url=import_url,
+                    status=response.status_code,
+                    response=response.text[:500] if response.text else "empty",
+                )
+                return False
+        except httpx.RequestError as e:
+            logger.error(
+                "pushgateway_import_request_error",
+                url=import_url,
+                error=str(e),
+                error_type=type(e).__name__,
+            )
+            return False
+
+    def push_add_metrics(self, registry: CollectorRegistry) -> bool:
+        """Add metrics to existing job/instance (pushadd operation).
+
+        Args:
+            registry: Prometheus metrics registry to add
+
+        Returns:
+            True if push succeeded, False otherwise
+        """
+        return self.push_metrics(registry, method="pushadd")
+
+    def delete_metrics(self) -> bool:
+        """Delete all metrics for the configured job.
+
+        Returns:
+            True if delete succeeded, False otherwise
+        """
+
+        if not self._enabled or not self.settings.pushgateway_url:
+            return False
+
+        # Check circuit breaker before attempting operation
+        if not self._circuit_breaker.can_execute():
+            logger.debug(
+                "pushgateway_circuit_breaker_blocking_delete",
+                state=self._circuit_breaker.state,
+                failure_count=self._circuit_breaker.failure_count,
+            )
+            return False
+
+        try:
+            # Only standard pushgateway supports delete operation
+            if "/api/v1/write" in self.settings.pushgateway_url:
+                logger.warning("pushgateway_delete_not_supported_for_remote_write")
+                return False
+            else:
+                success = self._push_standard(None, method="delete")  # type: ignore[arg-type]
+
+                if success:
+                    self._circuit_breaker.record_success()
+                else:
+                    self._circuit_breaker.record_failure()
+
+                return success
+
+        except Exception as e:
+            self._circuit_breaker.record_failure()
+            logger.error(
+                "pushgateway_delete_failed",
+                url=self.settings.pushgateway_url,
+                job=self.settings.pushgateway_job,
+                error=str(e),
+                error_type=type(e).__name__,
+            )
+            return False
+
+    def is_enabled(self) -> bool:
+        """Check if Pushgateway client is enabled and configured."""
+        return self._enabled and bool(self.settings.pushgateway_url)
+
+
+# Global pushgateway client instance
+_global_pushgateway_client: PushgatewayClient | None = None
+
+
+def get_pushgateway_client() -> PushgatewayClient:
+    """Get or create global pushgateway client instance."""
+    global _global_pushgateway_client
+
+    if _global_pushgateway_client is None:
+        # Import here to avoid circular imports
+        from ccproxy.config.settings import get_settings
+
+        settings = get_settings()
+        _global_pushgateway_client = PushgatewayClient(settings.observability)
+
+    return _global_pushgateway_client
+
+
+def reset_pushgateway_client() -> None:
+    """Reset global pushgateway client instance (mainly for testing)."""
+    global _global_pushgateway_client
+    _global_pushgateway_client = None

ccproxy/observability/sse_events.py

@@ -0,0 +1,303 @@
+"""
+Server-Sent Events (SSE) event manager for real-time dashboard updates.
+
+This module provides centralized SSE connection management and event broadcasting
+for real-time dashboard notifications when requests start, complete, or error.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+import uuid
+from collections.abc import AsyncGenerator
+from typing import Any
+
+import structlog
+
+
+logger = structlog.get_logger(__name__)
+
+
+class SSEEventManager:
+    """
+    Centralized SSE connection management and event broadcasting.
+
+    Manages multiple SSE connections and broadcasts events to all connected clients.
+    Uses bounded queues to prevent memory issues with slow clients.
+    """
+
+    def __init__(self, max_queue_size: int = 100) -> None:
+        """
+        Initialize SSE event manager.
+
+        Args:
+            max_queue_size: Maximum events to queue per connection before dropping
+        """
+        self._connections: dict[str, asyncio.Queue[dict[str, Any]]] = {}
+        self._lock = asyncio.Lock()
+        self._max_queue_size = max_queue_size
+
+    async def add_connection(
+        self, connection_id: str | None = None, request_id: str | None = None
+    ) -> AsyncGenerator[str, None]:
+        """
+        Add SSE connection and yield events as JSON strings.
+
+        Args:
+            connection_id: Unique connection identifier (generated if not provided)
+            request_id: Request identifier for tracking
+
+        Yields:
+            JSON-formatted event strings for SSE
+        """
+        if connection_id is None:
+            connection_id = str(uuid.uuid4())
+
+        # Create bounded queue for this connection
+        queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(
+            maxsize=self._max_queue_size
+        )
+
+        async with self._lock:
+            self._connections[connection_id] = queue
+
+        logger.debug(
+            "sse_connection_added", connection_id=connection_id, request_id=request_id
+        )
+
+        try:
+            # Send initial connection event
+            connection_event = {
+                "type": "connection",
+                "message": "Connected to metrics stream",
+                "connection_id": connection_id,
+                "timestamp": time.time(),
+            }
+            yield self._format_sse_event(connection_event)
+
+            while True:
+                # Wait for next event
+                event = await queue.get()
+
+                # Check for special disconnect event
+                if event.get("type") == "_disconnect":
+                    break
+
+                # Yield formatted event
+                yield self._format_sse_event(event)
+
+        except asyncio.CancelledError:
+            logger.debug("sse_connection_cancelled", connection_id=connection_id)
+            raise
+        except GeneratorExit:
+            logger.debug("sse_connection_generator_exit", connection_id=connection_id)
+            raise
+        finally:
+            # Clean up connection
+            await self._cleanup_connection(connection_id)
+
+            # Send disconnect event only if not in shutdown
+            try:
+                disconnect_event = {
+                    "type": "disconnect",
+                    "message": "Stream disconnected",
+                    "connection_id": connection_id,
+                    "timestamp": time.time(),
+                }
+                yield self._format_sse_event(disconnect_event)
+            except (GeneratorExit, asyncio.CancelledError):
+                # Ignore errors during cleanup
+                pass
+
+    async def emit_event(self, event_type: str, data: dict[str, Any]) -> None:
+        """
+        Broadcast event to all connected clients.
+
+        Args:
+            event_type: Type of event (request_start, request_complete, request_error)
+            data: Event data dictionary
+        """
+        if not self._connections:
+            return  # No connected clients
+
+        event = {
+            "type": event_type,
+            "data": data,
+            "timestamp": time.time(),
+        }
+
+        async with self._lock:
+            # Get copy of connections to avoid modification during iteration
+            connections = dict(self._connections)
+
+        # Broadcast to all connections
+        failed_connections = []
+
+        for connection_id, queue in connections.items():
+            try:
+                # Try to put event in queue without blocking
+                queue.put_nowait(event)
+            except asyncio.QueueFull:
+                # Queue is full, handle overflow
+                try:
+                    # Try to drop oldest event and add overflow indicator
+                    queue.get_nowait()  # Remove oldest
+                    overflow_event = {
+                        "type": "overflow",
+                        "message": "Event queue full, some events dropped",
+                        "timestamp": time.time(),
+                    }
+                    try:
+                        queue.put_nowait(overflow_event)
+                        queue.put_nowait(event)
+                    except asyncio.QueueFull:
+                        # Still full after dropping, connection is problematic
+                        failed_connections.append(connection_id)
+                        continue
+
+                    logger.warning(
+                        "sse_queue_overflow",
+                        connection_id=connection_id,
+                        max_queue_size=self._max_queue_size,
+                    )
+                except asyncio.QueueEmpty:
+                    # Queue became empty, try again
+                    try:
+                        queue.put_nowait(event)
+                    except asyncio.QueueFull:
+                        # Still full, connection is problematic
+                        failed_connections.append(connection_id)
+                except Exception as e:
+                    logger.error(
+                        "sse_overflow_error",
+                        connection_id=connection_id,
+                        error=str(e),
+                    )
+                    failed_connections.append(connection_id)
+            except Exception as e:
+                logger.error(
+                    "sse_broadcast_error",
+                    connection_id=connection_id,
+                    error=str(e),
+                )
+                failed_connections.append(connection_id)
+
+        # Clean up failed connections
+        for connection_id in failed_connections:
+            await self._cleanup_connection(connection_id)
+
+        if failed_connections:
+            logger.debug(
+                "sse_connections_cleaned",
+                failed_count=len(failed_connections),
+                active_count=len(self._connections),
+            )
+
+    async def disconnect_all(self) -> None:
+        """Disconnect all active connections gracefully."""
+        async with self._lock:
+            connections = dict(self._connections)
+
+        for connection_id, queue in connections.items():
+            try:
+                # Send disconnect signal
+                disconnect_signal = {"type": "_disconnect"}
+                queue.put_nowait(disconnect_signal)
+            except asyncio.QueueFull:
+                # Queue is full, force cleanup
+                await self._cleanup_connection(connection_id)
+            except Exception as e:
+                logger.error(
+                    "sse_disconnect_error",
+                    connection_id=connection_id,
+                    error=str(e),
+                )
+
+        logger.debug("sse_all_connections_disconnected")
+
+    async def _cleanup_connection(self, connection_id: str) -> None:
+        """Remove connection from active connections."""
+        async with self._lock:
+            if connection_id in self._connections:
+                del self._connections[connection_id]
+                logger.debug("sse_connection_removed", connection_id=connection_id)
+
+    def _format_sse_event(self, event: dict[str, Any]) -> str:
+        """Format event as SSE data string."""
+        try:
+            json_data = json.dumps(event, default=self._json_serializer)
+            return f"data: {json_data}\n\n"
+        except (TypeError, ValueError) as e:
+            logger.error("sse_format_error", error=str(e), event_type=event.get("type"))
+            # Return error event instead
+            error_event = {
+                "type": "error",
+                "message": "Failed to format event",
+                "timestamp": time.time(),
+            }
+            json_data = json.dumps(error_event, default=self._json_serializer)
+            return f"data: {json_data}\n\n"
+
+    def _json_serializer(self, obj: Any) -> Any:
+        """Custom JSON serializer for datetime and other objects."""
+        from datetime import datetime
+
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+    async def get_connection_count(self) -> int:
+        """Get number of active connections."""
+        async with self._lock:
+            return len(self._connections)
+
+    async def get_connection_info(self) -> dict[str, Any]:
+        """Get connection status information."""
+        async with self._lock:
+            return {
+                "active_connections": len(self._connections),
+                "max_queue_size": self._max_queue_size,
+                "connection_ids": list(self._connections.keys()),
+            }
+
+
+# Global SSE event manager instance
+_global_sse_manager: SSEEventManager | None = None
+
+
+def get_sse_manager() -> SSEEventManager:
+    """Get or create global SSE event manager."""
+    global _global_sse_manager
+
+    if _global_sse_manager is None:
+        _global_sse_manager = SSEEventManager()
+
+    return _global_sse_manager
+
+
+async def emit_sse_event(event_type: str, data: dict[str, Any]) -> None:
+    """
+    Convenience function to emit SSE event using global manager.
+
+    Args:
+        event_type: Type of event (request_start, request_complete, request_error)
+        data: Event data dictionary
+    """
+    try:
+        manager = get_sse_manager()
+        await manager.emit_event(event_type, data)
+    except Exception as e:
+        # Log error but don't fail the request
+        logger.debug("sse_emit_failed", event_type=event_type, error=str(e))
+
+
+async def cleanup_sse_manager() -> None:
+    """Clean up global SSE manager and disconnect all clients."""
+    global _global_sse_manager
+
+    if _global_sse_manager is not None:
+        await _global_sse_manager.disconnect_all()
+        _global_sse_manager = None
+        logger.debug("sse_manager_cleaned_up")