django-cfg 1.4.46__py3-none-any.whl → 1.5.181__py3-none-any.whl

django_cfg/apps/integrations/grpc/services/streaming/integrations/circuit_breaker.py

@@ -0,0 +1,247 @@
+"""
+Centrifugo resilience layer for auto-publishing.
+
+Simple circuit breaker pattern without external dependencies.
+Designed for non-critical auto-publishing use case.
+
+Created: 2025-11-14
+Status: %%PRODUCTION%%
+Phase: Phase 1 - Universal Components
+"""
+
+import logging
+from datetime import datetime, timedelta, timezone as tz
+from typing import Optional
+from enum import Enum
+
+# Logger will be configured by BidirectionalStreamingService via setup_streaming_logger
+logger = logging.getLogger("grpc_streaming.circuit_breaker")
+
+
+class CircuitState(str, Enum):
+    """Circuit breaker states."""
+    CLOSED = "closed"      # Normal operation
+    OPEN = "open"          # Too many errors, blocking calls
+    HALF_OPEN = "half_open"  # Testing if service recovered
+
+
+class CentrifugoCircuitBreaker:
+    """
+    Simple circuit breaker for Centrifugo auto-publishing.
+
+    **States**:
+    - CLOSED: Normal operation, all calls go through
+    - OPEN: Too many failures, all calls blocked
+    - HALF_OPEN: Testing recovery, limited calls allowed
+
+    **Configuration**:
+    - failure_threshold: Max consecutive failures before opening (default: 5)
+    - recovery_timeout: Seconds to wait before testing recovery (default: 60)
+    - success_threshold: Successful calls needed to close from half-open (default: 2)
+
+    **Usage**:
+    ```python
+    circuit = CentrifugoCircuitBreaker(failure_threshold=5, recovery_timeout=60)
+
+    if circuit.can_execute():
+        try:
+            await publish_to_centrifugo(data)
+            circuit.record_success()
+        except Exception as e:
+            circuit.record_failure(e)
+    else:
+        # Circuit is open, skip publishing
+        pass
+    ```
+
+    **Thread Safety**: Not thread-safe, designed for single asyncio event loop.
+    """
+
+    def __init__(
+        self,
+        failure_threshold: int = 5,
+        recovery_timeout: float = 60.0,
+        success_threshold: int = 2,
+        enable_logging: bool = True,
+    ):
+        """
+        Initialize circuit breaker.
+
+        Args:
+            failure_threshold: Max consecutive failures before opening circuit
+            recovery_timeout: Seconds to wait in OPEN state before trying HALF_OPEN
+            success_threshold: Successful calls needed in HALF_OPEN to close circuit
+            enable_logging: Enable state transition logging
+        """
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.success_threshold = success_threshold
+        self.enable_logging = enable_logging
+
+        # State
+        self._state = CircuitState.CLOSED
+        self._failure_count = 0
+        self._success_count = 0
+        self._last_failure_time: Optional[datetime] = None
+        self._opened_at: Optional[datetime] = None
+
+        # Statistics
+        self._total_calls = 0
+        self._total_failures = 0
+        self._total_blocked = 0
+
+    @property
+    def state(self) -> CircuitState:
+        """Get current circuit state."""
+        return self._state
+
+    @property
+    def is_open(self) -> bool:
+        """Check if circuit is open (blocking calls)."""
+        return self._state == CircuitState.OPEN
+
+    def can_execute(self) -> bool:
+        """
+        Check if call can be executed.
+
+        Returns:
+            True if call should proceed, False if blocked by circuit breaker
+        """
+        self._total_calls += 1
+
+        # CLOSED: Allow all calls
+        if self._state == CircuitState.CLOSED:
+            return True
+
+        # OPEN: Check if recovery timeout passed
+        if self._state == CircuitState.OPEN:
+            if self._opened_at is None:
+                # Should never happen, but handle gracefully
+                self._transition_to(CircuitState.HALF_OPEN)
+                return True
+
+            elapsed = (datetime.now(tz.utc) - self._opened_at).total_seconds()
+            if elapsed >= self.recovery_timeout:
+                # Try recovery
+                self._transition_to(CircuitState.HALF_OPEN)
+                return True
+            else:
+                # Still in recovery timeout
+                self._total_blocked += 1
+                return False
+
+        # HALF_OPEN: Allow call to test recovery
+        if self._state == CircuitState.HALF_OPEN:
+            return True
+
+        # Should never reach here
+        return False
+
+    def record_success(self) -> None:
+        """Record successful call."""
+        if self._state == CircuitState.CLOSED:
+            # Reset failure counter on success
+            self._failure_count = 0
+
+        elif self._state == CircuitState.HALF_OPEN:
+            # Count successes to close circuit
+            self._success_count += 1
+            if self._success_count >= self.success_threshold:
+                self._transition_to(CircuitState.CLOSED)
+
+    def record_failure(self, error: Exception) -> None:
+        """
+        Record failed call.
+
+        Args:
+            error: Exception that caused the failure
+        """
+        self._total_failures += 1
+        self._last_failure_time = datetime.now(tz.utc)
+
+        if self._state == CircuitState.CLOSED:
+            self._failure_count += 1
+            if self._failure_count >= self.failure_threshold:
+                self._transition_to(CircuitState.OPEN)
+
+        elif self._state == CircuitState.HALF_OPEN:
+            # Failure during testing - reopen circuit
+            self._transition_to(CircuitState.OPEN)
+
+    def _transition_to(self, new_state: CircuitState) -> None:
+        """
+        Transition to new state.
+
+        Args:
+            new_state: New circuit state
+        """
+        old_state = self._state
+        self._state = new_state
+
+        # Reset counters based on new state
+        if new_state == CircuitState.CLOSED:
+            self._failure_count = 0
+            self._success_count = 0
+            self._opened_at = None
+
+        elif new_state == CircuitState.OPEN:
+            self._opened_at = datetime.now(tz.utc)
+            self._success_count = 0
+
+        elif new_state == CircuitState.HALF_OPEN:
+            self._failure_count = 0
+            self._success_count = 0
+
+        # Log transition
+        if self.enable_logging and old_state != new_state:
+            logger.warning(
+                f"⚡ Centrifugo Circuit Breaker: {old_state.value} → {new_state.value} "
+                f"(failures={self._failure_count}, total_blocked={self._total_blocked})"
+            )
+
+    def reset(self) -> None:
+        """Reset circuit breaker to initial state."""
+        self._state = CircuitState.CLOSED
+        self._failure_count = 0
+        self._success_count = 0
+        self._last_failure_time = None
+        self._opened_at = None
+
+        if self.enable_logging:
+            logger.info("🔄 Centrifugo Circuit Breaker reset")
+
+    def get_stats(self) -> dict:
+        """
+        Get circuit breaker statistics.
+
+        Returns:
+            Dictionary with statistics
+        """
+        return {
+            "state": self._state.value,
+            "total_calls": self._total_calls,
+            "total_failures": self._total_failures,
+            "total_blocked": self._total_blocked,
+            "failure_count": self._failure_count,
+            "success_count": self._success_count,
+            "failure_rate": (
+                self._total_failures / self._total_calls
+                if self._total_calls > 0
+                else 0.0
+            ),
+            "last_failure_time": (
+                self._last_failure_time.isoformat()
+                if self._last_failure_time
+                else None
+            ),
+        }
+
+
+# ============================================================================
+# Exports
+# ============================================================================
+
+__all__ = [
+    'CentrifugoCircuitBreaker',
+    'CircuitState',
+]

django_cfg/apps/integrations/grpc/services/streaming/processors/__init__.py

@@ -0,0 +1,18 @@
+"""
+Processors for bidirectional streaming.
+
+Input and output message/command processing with Centrifugo integration.
+
+Created: 2025-11-14
+Status: %%PRODUCTION%%
+Phase: Phase 1 - Universal Components (Refactored)
+"""
+
+from .input import InputProcessor
+from .output import OutputProcessor
+
+
+__all__ = [
+    'InputProcessor',
+    'OutputProcessor',
+]

django_cfg/apps/integrations/grpc/services/streaming/processors/input.py

@@ -0,0 +1,390 @@
+"""
+Input message processing for bidirectional streaming.
+
+Handles incoming messages from clients with async iteration support.
+
+Created: 2025-11-14
+Status: %%PRODUCTION%%
+Phase: Phase 1 - Universal Components (Refactored)
+"""
+
+import asyncio
+import logging
+import traceback
+from typing import AsyncIterator, Optional, Any
+import grpc
+
+from ..types import (
+    TMessage,
+    TCommand,
+    ClientIdExtractor,
+    MessageProcessor,
+    CommandAckExtractor,
+    HeartbeatExtractor,
+    HeartbeatCallback,
+)
+from ..integrations.centrifugo import CentrifugoPublisher
+from ..config import StreamingMode
+
+# Logger will be configured by BidirectionalStreamingService via setup_streaming_logger
+logger = logging.getLogger("grpc_streaming.input_processor")
+
+
+class InputProcessor:
+    """
+    Processes incoming messages from clients.
+
+    Responsibilities:
+    - Extract client_id from each message
+    - Auto-resolve CommandAck (for synchronous RPC)
+    - Auto-handle Heartbeat (for keepalive + metrics)
+    - Route messages to message processor callback
+    - Auto-publish to Centrifugo (if configured)
+    - Handle errors gracefully
+    - Support both async for and anext() iteration modes
+    """
+
+    def __init__(
+        self,
+        extract_client_id: ClientIdExtractor[TMessage],
+        process_message: MessageProcessor[TMessage, TCommand],
+        centrifugo_publisher: Optional[CentrifugoPublisher] = None,
+        command_ack_extractor: Optional[CommandAckExtractor[TMessage]] = None,
+        heartbeat_extractor: Optional[HeartbeatExtractor[TMessage]] = None,
+        heartbeat_callback: Optional[HeartbeatCallback] = None,
+        streaming_mode: StreamingMode = StreamingMode.ASYNC_FOR,
+        connection_timeout: Optional[float] = None,
+        yield_event_loop: bool = True,
+        enable_logging: bool = True
+    ):
+        """
+        Initialize input processor.
+
+        Args:
+            extract_client_id: Callback to extract client_id from message
+            process_message: Callback to process message (business logic)
+            centrifugo_publisher: Optional Centrifugo publisher for auto-publishing
+            command_ack_extractor: Optional callback to extract CommandAck from message
+            heartbeat_extractor: Optional callback to extract Heartbeat from message
+            heartbeat_callback: Optional callback to handle heartbeat (update DB, send ping)
+            streaming_mode: ASYNC_FOR or ANEXT iteration mode
+            connection_timeout: Timeout for anext() mode (None = no timeout)
+            yield_event_loop: Whether to yield event loop with asyncio.sleep(0)
+            enable_logging: Enable logging
+        """
+        self.extract_client_id = extract_client_id
+        self.process_message = process_message
+        self.centrifugo_publisher = centrifugo_publisher
+        self.command_ack_extractor = command_ack_extractor
+        self.heartbeat_extractor = heartbeat_extractor
+        self.heartbeat_callback = heartbeat_callback
+        self.streaming_mode = streaming_mode
+        self.connection_timeout = connection_timeout
+        self.yield_event_loop = yield_event_loop
+        self.enable_logging = enable_logging
+
+    async def process_stream(
+        self,
+        request_iterator: AsyncIterator[TMessage],
+        output_queue: asyncio.Queue[TCommand],
+        context: grpc.aio.ServicerContext,
+        streaming_service: Optional[Any] = None,
+        on_connect: Optional[Any] = None
+    ) -> Optional[str]:
+        """
+        Process incoming message stream.
+
+        Args:
+            request_iterator: Async iterator of incoming messages
+            output_queue: Queue for outgoing commands
+            context: gRPC service context
+            streaming_service: Reference to BidirectionalStreamingService (for callbacks)
+            on_connect: Optional callback when client connects
+
+        Returns:
+            client_id if extracted, None otherwise
+
+        Raises:
+            Exception: If critical error occurs during processing
+        """
+        if self.streaming_mode == StreamingMode.ASYNC_FOR:
+            return await self._process_async_for(
+                request_iterator,
+                output_queue,
+                context,
+                streaming_service,
+                on_connect
+            )
+        else:  # StreamingMode.ANEXT
+            return await self._process_anext(
+                request_iterator,
+                output_queue,
+                context,
+                streaming_service,
+                on_connect
+            )
+
+    async def _process_async_for(
+        self,
+        request_iterator: AsyncIterator[TMessage],
+        output_queue: asyncio.Queue[TCommand],
+        context: grpc.aio.ServicerContext,
+        streaming_service: Optional[Any] = None,
+        on_connect: Optional[Any] = None
+    ) -> Optional[str]:
+        """Process input stream using async for iteration."""
+        client_id: Optional[str] = None
+        is_first_message = True
+
+        try:
+            if self.enable_logging:
+                logger.info("Starting async for loop")
+
+            async for message in request_iterator:
+                # DEBUG: Log every received message
+                msg_type = message.WhichOneof('payload') if hasattr(message, 'WhichOneof') else 'unknown'
+                print(f"\n📥📥📥 [INPUT_PROCESSOR] Received message type={msg_type}", flush=True)
+                logger.info(f"📥 [INPUT_PROCESSOR] Received message type={msg_type}")
+
+                # Extract client ID from first message
+                if is_first_message:
+                    client_id = self.extract_client_id(message)
+                    is_first_message = False
+
+                    # Register connection in ConnectionManager
+                    if streaming_service and hasattr(streaming_service, 'connection_manager'):
+                        streaming_service.connection_manager.register(
+                            client_id=client_id,
+                            output_queue=output_queue
+                        )
+
+                    if self.enable_logging:
+                        logger.info(f"Client {client_id[:8]}... connected")
+
+                    if on_connect:
+                        await on_connect(client_id)
+
+                # Auto-publish to Centrifugo (Client � Server message)
+                if self.centrifugo_publisher:
+                    await self.centrifugo_publisher.publish_message(
+                        client_id=client_id,
+                        message=message
+                    )
+
+
+                # Auto-resolve CommandAck (for synchronous RPC)
+                if self.command_ack_extractor and streaming_service:
+                    command_ack = self.command_ack_extractor(message)
+                    if command_ack and hasattr(command_ack, 'command_id') and command_ack.command_id:
+                        # Resolve in ResponseRegistry (only if command_id is not empty)
+                        if hasattr(streaming_service, 'response_registry'):
+                            await streaming_service.response_registry.resolve_command(
+                                command_ack.command_id,
+                                command_ack
+                            )
+                            if self.enable_logging:
+                                logger.debug(
+                                    f"✅ Auto-resolved CommandAck: {command_ack.command_id} "
+                                    f"(client={client_id[:8]}...)"
+                                )
+                            # CommandAck processed, skip user's message_processor
+                            continue
+
+                # Auto-handle Heartbeat (for keepalive)
+                if self.heartbeat_extractor and self.heartbeat_callback:
+                    heartbeat = self.heartbeat_extractor(message)
+                    if heartbeat:
+                        # Run in background (non-blocking - DB queries can be slow)
+                        asyncio.create_task(
+                            self.heartbeat_callback(
+                                client_id=client_id,
+                                heartbeat=heartbeat,
+                                output_queue=output_queue
+                            )
+                        )
+                        if self.enable_logging:
+                            logger.debug(
+                                f"💓 Auto-handled Heartbeat (client={client_id[:8]}...)"
+                            )
+                        # Heartbeat processed, skip user's message_processor
+                        continue
+
+                # Process message (business logic)
+                await self.process_message(
+                    client_id=client_id,
+                    message=message,
+                    output_queue=output_queue,
+                    streaming_service=streaming_service
+                )
+
+                # � CRITICAL: Yield to event loop!
+                # Without this, the next message read blocks output loop from yielding.
+                if self.yield_event_loop:
+                    await asyncio.sleep(0)
+
+        except asyncio.CancelledError:
+            if self.enable_logging:
+                logger.info(f"Input stream cancelled for client {client_id}")
+            raise
+
+        except Exception as e:
+            if self.enable_logging:
+                logger.error(
+                    f"❌ [INPUT_PROCESSOR] Input stream error for client {client_id}:\n"
+                    f"   Error: {type(e).__name__}: {e}\n"
+                    f"   Traceback:\n{traceback.format_exc()}"
+                )
+            raise
+
+        return client_id
+
+    async def _process_anext(
+        self,
+        request_iterator: AsyncIterator[TMessage],
+        output_queue: asyncio.Queue[TCommand],
+        context: grpc.aio.ServicerContext,
+        streaming_service: Optional[Any] = None,
+        on_connect: Optional[Any] = None
+    ) -> Optional[str]:
+        """Process input stream using anext() calls."""
+        client_id: Optional[str] = None
+        is_first_message = True
+
+        try:
+            # DEBUG: Check context state before loop
+            print(f"\n🔍 [INPUT_PROCESSOR] _process_anext starting", flush=True)
+            print(f"   context.cancelled()={context.cancelled()}", flush=True)
+            print(f"   request_iterator type={type(request_iterator)}", flush=True)
+            if self.enable_logging:
+                logger.info(f"_process_anext starting, context.cancelled()={context.cancelled()}")
+
+            while not context.cancelled():
+                try:
+                    # Get next message with optional timeout
+                    # NOTE: connection_timeout is for non-blocking read, not for closing connection
+                    if self.connection_timeout:
+                        message = await asyncio.wait_for(
+                            anext(request_iterator),
+                            timeout=self.connection_timeout,
+                        )
+                    else:
+                        message = await anext(request_iterator)
+
+                    # Extract client ID from first message
+                    if is_first_message:
+                        client_id = self.extract_client_id(message)
+                        is_first_message = False
+
+                        # Register connection in ConnectionManager
+                        if streaming_service and hasattr(streaming_service, 'connection_manager'):
+                            streaming_service.connection_manager.register(
+                                client_id=client_id,
+                                output_queue=output_queue
+                            )
+
+                        if self.enable_logging:
+                            logger.info(f"Client {client_id[:8]}... connected")
+
+                        if on_connect:
+                            await on_connect(client_id)
+
+                    # Auto-publish to Centrifugo (Client � Server message)
+                    if self.centrifugo_publisher:
+                        await self.centrifugo_publisher.publish_message(
+                            client_id=client_id,
+                            message=message
+                        )
+
+                    # Auto-resolve CommandAck (for synchronous RPC)
+                    if self.command_ack_extractor and streaming_service:
+                        command_ack = self.command_ack_extractor(message)
+                        if command_ack and hasattr(command_ack, 'command_id') and command_ack.command_id:
+                            # Resolve in ResponseRegistry (only if command_id is not empty)
+                            if hasattr(streaming_service, 'response_registry'):
+                                await streaming_service.response_registry.resolve_command(
+                                    command_ack.command_id,
+                                    command_ack
+                                )
+                                if self.enable_logging:
+                                    logger.debug(
+                                        f"✅ Auto-resolved CommandAck: {command_ack.command_id} "
+                                        f"(client={client_id[:8]}...)"
+                                    )
+                                # CommandAck processed, skip user's message_processor
+                                continue
+
+                    # Auto-handle Heartbeat (for keepalive)
+                    if self.heartbeat_extractor and self.heartbeat_callback:
+                        heartbeat = self.heartbeat_extractor(message)
+                        if heartbeat:
+                            # Run in background (non-blocking - DB queries can be slow)
+                            asyncio.create_task(
+                                self.heartbeat_callback(
+                                    client_id=client_id,
+                                    heartbeat=heartbeat,
+                                    output_queue=output_queue
+                                )
+                            )
+                            if self.enable_logging:
+                                logger.debug(
+                                    f"💓 Auto-handled Heartbeat (client={client_id[:8]}...)"
+                                )
+                            # Heartbeat processed, skip user's message_processor
+                            continue
+
+                    # Process message (business logic)
+                    await self.process_message(
+                        client_id=client_id,
+                        message=message,
+                        output_queue=output_queue,
+                        streaming_service=streaming_service
+                    )
+
+                    # � CRITICAL: Yield to event loop!
+                    if self.yield_event_loop:
+                        await asyncio.sleep(0)
+
+                except StopAsyncIteration:
+                    # Stream ended normally
+                    print(f"   ⚠️ StopAsyncIteration - stream ended", flush=True)
+                    if self.enable_logging:
+                        logger.info(f"Client {client_id} stream ended")
+                    break
+
+                except asyncio.TimeoutError:
+                    # connection_timeout is for non-blocking read, NOT for closing connection.
+                    # Just continue the loop to check context.cancelled() and wait for more messages.
+                    # The real connection timeout should be handled via ping_timeout in OutputProcessor.
+                    continue
+
+            # DEBUG: Log when while loop exits normally (context.cancelled() == True)
+            if self.enable_logging:
+                logger.info(f"_process_anext while loop exited, context.cancelled()={context.cancelled()}, client_id={client_id}")
+
+        except asyncio.CancelledError:
+            print(f"   🚫 CancelledError - input stream cancelled (client_id={client_id})", flush=True)
+            if self.enable_logging:
+                logger.info(f"Input stream cancelled for client {client_id}")
+            raise
+
+        except Exception as e:
+            print(f"   ❌ Exception: {type(e).__name__}: {e}", flush=True)
+            if self.enable_logging:
+                logger.error(
+                    f"❌ [INPUT_PROCESSOR] Input stream error (anext) for client {client_id}:\n"
+                    f"   Error: {type(e).__name__}: {e}\n"
+                    f"   Traceback:\n{traceback.format_exc()}"
+                )
+            raise
+
+        print(f"   🏁 _process_anext returning, client_id={client_id}", flush=True)
+        return client_id
+
+
+# ============================================================================
+# Exports
+# ============================================================================
+
+__all__ = [
+    'InputProcessor',
+]