django-cfg 1.5.14__py3-none-any.whl → 1.5.29__py3-none-any.whl

django_cfg/apps/integrations/grpc/management/commands/rungrpc.py

@@ -1,21 +1,53 @@
 """
-Django management command to run async gRPC server.
+Django management command to run async gRPC server with auto-reload support.
 
 Usage:
+    # Development mode (with auto-reload)
     python manage.py rungrpc
+
+    # Production mode (no auto-reload)
+    python manage.py rungrpc --noreload
+
+    # Custom host and port
     python manage.py rungrpc --host 0.0.0.0 --port 50051
+
+    # With Centrifugo test event on startup
+    python manage.py rungrpc --test
+
+    # Disable specific features
+    python manage.py rungrpc --no-reflection --no-health-check
+
+Auto-reload behavior:
+    - Enabled by default in development mode (ENV_MODE != "production")
+    - Disabled by default in production mode (ENV_MODE == "production")
+    - Use --noreload to explicitly disable auto-reload
+    - Server will restart automatically when Python files change
+
+Test mode:
+    - Use --test to send a test Centrifugo event on server startup
+    - Useful for verifying Centrifugo integration is working
+    - Event published to: grpc#rungrpc#startup#test
 """
 
 from __future__ import annotations
 
 import asyncio
+import logging
 import signal
 import sys
+import threading
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.utils import autoreload
 
+from django_cfg.core.config import get_current_config
 from django_cfg.modules.django_logging import get_logger
+from django_cfg.apps.integrations.grpc.utils.streaming_logger import (
+    setup_streaming_logger,
+    log_server_start,
+    log_server_shutdown,
+)
 
 # Check dependencies before importing grpc
 from django_cfg.apps.integrations.grpc._cfg import check_grpc_dependencies
@@ -33,16 +65,23 @@ import grpc.aio
 
 class Command(BaseCommand):
     """
-    Run async gRPC server with auto-discovered services.
+    Run async gRPC server with auto-discovered services and hot-reload.
 
     Features:
     - Async server with grpc.aio
     - Auto-discovers and registers services
+    - Hot-reload in development mode (watches for file changes)
     - Configurable host, port
     - Health check support
     - Reflection support
     - Graceful shutdown
     - Signal handling
+
+    Hot-reload:
+    - Automatically enabled in development mode (ENV_MODE != "production")
+    - Automatically disabled in production mode (ENV_MODE == "production")
+    - Use --noreload to explicitly disable in development
+    - Works like Django's runserver - restarts on code changes
     """
 
     # Web execution metadata
@@ -50,14 +89,17 @@ class Command(BaseCommand):
     requires_input = False
     is_destructive = False
 
-    help = "Run async gRPC server"
+    help = "Run async gRPC server with optional hot-reload support"
 
     def __init__(self, *args, **kwargs):
         """Initialize with self.logger and async server reference."""
         super().__init__(*args, **kwargs)
         self.logger = get_logger('rungrpc')
+        self.streaming_logger = None  # Will be initialized when server starts
         self.server = None
         self.shutdown_event = None
+        self.server_status = None
+        self.server_config = None  # Store config for re-registration
 
     def add_arguments(self, parser):
         """Add command arguments."""
@@ -88,22 +130,114 @@ class Command(BaseCommand):
             action="store_true",
             help="Enable asyncio debug mode",
         )
+        parser.add_argument(
+            "--noreload",
+            action="store_false",
+            dest="use_reloader",
+            help="Disable auto-reloader (default: enabled in dev mode)",
+        )
+        parser.add_argument(
+            "--test",
+            action="store_true",
+            help="Send test Centrifugo event on startup (for testing integration)",
+        )
 
     def handle(self, *args, **options):
-        """Run async gRPC server."""
+        """Run async gRPC server with optional auto-reload."""
+        config = get_current_config()
+
+        # Determine if we should use auto-reload
+        # Changed default to False for stability with bidirectional streaming
+        use_reloader = options.get("use_reloader", False)
+
+        # Check if we're in production mode (disable reloader in production)
+        if config and hasattr(config, 'is_production'):
+            is_production = config.is_production  # property, not method
+        else:
+            # Fallback to settings
+            env_mode = getattr(settings, "ENV_MODE", "development").lower()
+            is_production = env_mode == "production"
+
+        # Disable reloader in production by default
+        if is_production and options.get("use_reloader") is None:
+            use_reloader = False
+            self.stdout.write(
+                self.style.WARNING(
+                    "Production mode - auto-reloader disabled"
+                )
+            )
+
         # Enable asyncio debug if requested
         if options.get("asyncio_debug"):
             asyncio.get_event_loop().set_debug(True)
             self.logger.info("Asyncio debug mode enabled")
 
-        # Run async main
-        asyncio.run(self._async_main(*args, **options))
+        # Run with or without reloader
+        if use_reloader:
+            from datetime import datetime
+            current_time = datetime.now().strftime('%H:%M:%S')
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"🔄 Auto-reloader enabled - watching for file changes [{current_time}]"
+                )
+            )
+            self.stdout.write(
+                self.style.WARNING(
+                    "⚠️  Note: Active streaming connections will be dropped on reload. Use --noreload for stable bots."
+                )
+            )
+
+            # Setup autoreload to watch project directory
+            from django.utils.autoreload import autoreload_started
+
+            def watch_project_files(sender, **kwargs):
+                """Automatically watch Django project directory for changes."""
+                base_dir = getattr(settings, 'BASE_DIR', None)
+                if base_dir:
+                    sender.watch_dir(str(base_dir), '*.py')
+                    self.logger.debug(f"Watching project directory: {base_dir}")
+
+            autoreload_started.connect(watch_project_files)
+
+            # Use autoreload to restart on code changes
+            autoreload.run_with_reloader(
+                lambda: asyncio.run(self._async_main(*args, **options))
+            )
+        else:
+            # Run directly without reloader
+            asyncio.run(self._async_main(*args, **options))
 
     async def _async_main(self, *args, **options):
         """Main async server loop."""
+        # Setup streaming logger for detailed gRPC logging
+        self.streaming_logger = setup_streaming_logger(
+            name='grpc_rungrpc',
+            level=logging.DEBUG,
+            console_level=logging.INFO
+        )
+
+        config = get_current_config()
+        use_reloader = options.get("use_reloader", True)
+
+        # Determine production mode
+        if config and hasattr(config, 'is_production'):
+            is_production = config.is_production  # property, not method
+        else:
+            env_mode = getattr(settings, "ENV_MODE", "development").lower()
+            is_production = env_mode == "production"
+
+        # Log startup using reusable function
+        start_time = log_server_start(
+            self.streaming_logger,
+            server_type="gRPC Server",
+            mode="Production" if is_production else "Development",
+            hotreload_enabled=use_reloader
+        )
+
         # Import models here to avoid AppRegistryNotReady
         from django_cfg.apps.integrations.grpc.models import GRPCServerStatus
-        from django_cfg.apps.integrations.grpc.services.config_helper import (
+        from django_cfg.apps.integrations.grpc.services.management.config_helper import (
             get_grpc_server_config,
         )
 
@@ -182,14 +316,23 @@ class Command(BaseCommand):
             import os
             from django_cfg.apps.integrations.grpc.services import ServiceDiscovery
 
+            # Store config for re-registration
+            self.server_config = {
+                'host': host,
+                'port': port,
+                'pid': os.getpid(),
+                'max_workers': 0,
+                'enable_reflection': enable_reflection,
+                'enable_health_check': enable_health_check,
+            }
+
             # Get registered services metadata (run in thread to avoid blocking)
             discovery = ServiceDiscovery()
             services_metadata = await asyncio.to_thread(
                 discovery.get_registered_services
             )
 
-            server_status = await asyncio.to_thread(
-                GRPCServerStatus.objects.start_server,
+            server_status = await GRPCServerStatus.objects.astart_server(
                 host=host,
                 port=port,
                 pid=os.getpid(),
@@ -200,11 +343,13 @@ class Command(BaseCommand):
 
             # Store registered services in database
             server_status.registered_services = services_metadata
-            await asyncio.to_thread(
-                server_status.save,
+            await server_status.asave(
                 update_fields=["registered_services"]
             )
 
+            # Store in instance for heartbeat
+            self.server_status = server_status
+
         except Exception as e:
             self.logger.warning(f"Could not start server status tracking: {e}")
 
@@ -214,10 +359,18 @@ class Command(BaseCommand):
         # Mark server as running
         if server_status:
             try:
-                await asyncio.to_thread(server_status.mark_running)
+                await server_status.amark_running()
             except Exception as e:
                 self.logger.warning(f"Could not mark server as running: {e}")
 
+        # Start heartbeat background task
+        heartbeat_task = None
+        if server_status:
+            heartbeat_task = asyncio.create_task(
+                self._heartbeat_loop(interval=30)
+            )
+            self.logger.info("Started heartbeat background task (30s interval)")
+
         # Display gRPC-specific startup info
         try:
             from django_cfg.core.integration.display import GRPCDisplayManager
@@ -250,13 +403,75 @@ class Command(BaseCommand):
 
         # Keep server running
         self.stdout.write(self.style.SUCCESS("\n✅ Async gRPC server is running..."))
+
+        # Show reloader status
+        if use_reloader and not is_production:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "🔄 Auto-reloader active - server will restart on code changes"
+                )
+            )
+
         self.stdout.write("Press CTRL+C to stop\n")
 
+        # Log server ready
+        self.streaming_logger.info("✅ Server ready and accepting connections")
+        if use_reloader:
+            self.streaming_logger.info("🔄 Watching for file changes...")
+
+        # Send test Centrifugo event if --test flag is set
+        if options.get("test"):
+            self.streaming_logger.info("🧪 Sending test Centrifugo event...")
+            try:
+                from django_cfg.apps.integrations.grpc.services.centrifugo.demo import send_demo_event
+
+                test_result = await send_demo_event(
+                    channel="grpc#rungrpc#startup#test",
+                    metadata={
+                        "source": "rungrpc",
+                        "action": "startup_test",
+                        "mode": "Development" if not is_production else "Production",
+                        "host": host,
+                        "port": port,
+                    }
+                )
+
+                if test_result:
+                    self.streaming_logger.info("✅ Test Centrifugo event sent successfully")
+                    self.stdout.write(self.style.SUCCESS("🧪 Test event published to Centrifugo"))
+                else:
+                    self.streaming_logger.warning("⚠️ Test Centrifugo event failed")
+                    self.stdout.write(self.style.WARNING("⚠️ Test event failed (check Centrifugo config)"))
+
+            except Exception as e:
+                self.streaming_logger.error(f"❌ Failed to send test event: {e}")
+                self.stdout.write(
+                    self.style.ERROR(f"❌ Test event error: {e}")
+                )
+
+        shutdown_reason = "Unknown"
         try:
             await self.server.wait_for_termination()
+            shutdown_reason = "Normal termination"
         except KeyboardInterrupt:
-            # Signal handler will take care of graceful shutdown
+            shutdown_reason = "Keyboard interrupt"
             pass
+        finally:
+            # Log shutdown using reusable function
+            log_server_shutdown(
+                self.streaming_logger,
+                start_time,
+                server_type="gRPC Server",
+                reason=shutdown_reason
+            )
+
+            # Cancel heartbeat task
+            if heartbeat_task and not heartbeat_task.done():
+                heartbeat_task.cancel()
+                try:
+                    await heartbeat_task
+                except asyncio.CancelledError:
+                    pass
 
     def _build_grpc_options(self, config: dict) -> list:
         """
@@ -462,6 +677,78 @@ class Command(BaseCommand):
             )
             return 0
 
+    async def _heartbeat_loop(self, interval: int = 30):
+        """
+        Periodically update server heartbeat with auto-recovery.
+
+        If server record is deleted from database, automatically re-registers
+        the server to maintain monitoring continuity.
+
+        Args:
+            interval: Heartbeat interval in seconds (default: 30)
+        """
+        from django_cfg.apps.integrations.grpc.models import GRPCServerStatus
+
+        try:
+            while True:
+                await asyncio.sleep(interval)
+
+                if not self.server_status or not self.server_config:
+                    self.logger.warning("No server status or config available")
+                    continue
+
+                try:
+                    # Check if record still exists (Django 5.2: Native async ORM)
+                    record_exists = await GRPCServerStatus.objects.filter(
+                        id=self.server_status.id
+                    ).aexists()
+
+                    if not record_exists:
+                        # Record was deleted - re-register server
+                        self.logger.warning(
+                            "Server record was deleted from database, "
+                            "re-registering..."
+                        )
+
+                        # Get services metadata for re-registration
+                        from django_cfg.apps.integrations.grpc.services import ServiceDiscovery
+                        discovery = ServiceDiscovery()
+                        services_metadata = await asyncio.to_thread(
+                            discovery.get_registered_services
+                        )
+
+                        # Re-register server (Django 5.2: Native async ORM)
+                        new_server_status = await GRPCServerStatus.objects.astart_server(
+                            **self.server_config
+                        )
+
+                        # Store registered services
+                        new_server_status.registered_services = services_metadata
+                        await new_server_status.asave(
+                            update_fields=["registered_services"]
+                        )
+
+                        # Mark as running (Django 5.2: Native async ORM)
+                        await new_server_status.amark_running()
+
+                        # Update reference
+                        self.server_status = new_server_status
+
+                        self.logger.warning(
+                            f"✅ Successfully re-registered server (ID: {new_server_status.id})"
+                        )
+                    else:
+                        # Record exists - just update heartbeat (Django 5.2: Native async ORM)
+                        await self.server_status.amark_running()
+                        self.logger.debug(f"Heartbeat updated (interval: {interval}s)")
+
+                except Exception as e:
+                    self.logger.warning(f"Failed to update heartbeat: {e}")
+
+        except asyncio.CancelledError:
+            self.logger.info("Heartbeat task cancelled")
+            raise
+
     def _setup_signal_handlers_async(self, server, server_status=None):
         """
         Setup signal handlers for graceful async server shutdown.
@@ -469,7 +756,17 @@ class Command(BaseCommand):
         Args:
             server: Async gRPC server instance
             server_status: GRPCServerStatus instance (optional)
+
+        Note:
+            Signal handlers can only be set in the main thread.
+            When running with autoreload, we're in a separate thread,
+            so we skip signal handler setup.
         """
+        # Check if we're in the main thread
+        if threading.current_thread() is not threading.main_thread():
+            # In autoreload mode, Django handles signals - we don't need to set them up
+            return
+
         # Flag to prevent multiple shutdown attempts
         shutdown_initiated = {'value': False}
 
@@ -481,17 +778,10 @@ class Command(BaseCommand):
 
             self.stdout.write("\n🛑 Shutting down gracefully...")
 
-            # Mark server as stopping
+            # Mark server as stopping (sync context - signal handlers are sync)
             if server_status:
                 try:
-                    import django
-                    if django.VERSION >= (3, 0):
-                        from asgiref.sync import sync_to_async
-                        # Run in sync context
-                        try:
-                            server_status.mark_stopping()
-                        except:
-                            pass
+                    server_status.mark_stopping()
                 except Exception as e:
                     self.logger.warning(f"Could not mark server as stopping: {e}")
 
@@ -503,12 +793,11 @@ class Command(BaseCommand):
             except Exception as e:
                 self.logger.error(f"Error stopping server: {e}")
 
-            # Mark server as stopped (async-safe)
+            # Mark server as stopped (async-safe, Django 5.2: Native async ORM)
             if server_status:
                 try:
-                    from asgiref.sync import sync_to_async
-                    # Wrap sync DB operation in sync_to_async
-                    asyncio.create_task(sync_to_async(server_status.mark_stopped)())
+                    # Use native async method
+                    asyncio.create_task(server_status.amark_stopped())
                 except Exception as e:
                     self.logger.warning(f"Could not mark server as stopped: {e}")
 

django_cfg/apps/integrations/grpc/management/proto/__init__.py

@@ -0,0 +1,3 @@
+"""
+Proto utilities for django-cfg gRPC integration.
+"""

django_cfg/apps/integrations/grpc/management/proto/compiler.py

@@ -0,0 +1,194 @@
+"""
+Proto compiler utilities.
+Shared functionality for compiling .proto files to Python.
+"""
+
+import logging
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ProtoCompiler:
+    """Compiles .proto files to Python using grpc_tools.protoc."""
+
+    def __init__(
+        self,
+        output_dir: Optional[Path] = None,
+        proto_import_path: Optional[Path] = None,
+        fix_imports: bool = True,
+        verbose: bool = True,
+    ):
+        """
+        Initialize proto compiler.
+
+        Args:
+            output_dir: Output directory for generated files (default: same as proto file)
+            proto_import_path: Additional proto import path (passed to protoc -I flag)
+            fix_imports: Fix imports in generated _grpc.py files (default: True)
+            verbose: Print compilation progress (default: True)
+        """
+        self.output_dir = output_dir
+        self.proto_import_path = proto_import_path
+        self.fix_imports = fix_imports
+        self.verbose = verbose
+
+    def compile_file(self, proto_file: Path) -> bool:
+        """
+        Compile a single .proto file.
+
+        Args:
+            proto_file: Path to .proto file
+
+        Returns:
+            True if compilation succeeded, False otherwise
+        """
+        if self.verbose:
+            logger.info(f"📦 Compiling: {proto_file}")
+
+        # Determine output directory
+        output_dir = self.output_dir or proto_file.parent
+
+        # Determine proto import path
+        proto_import_path = self.proto_import_path or proto_file.parent
+
+        # Ensure output directory exists
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Build protoc command
+        cmd = [
+            sys.executable,
+            "-m",
+            "grpc_tools.protoc",
+            f"-I{proto_import_path}",
+            f"--python_out={output_dir}",
+            f"--grpc_python_out={output_dir}",
+            str(proto_file),
+        ]
+
+        # Run protoc
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+
+            if result.stdout and self.verbose:
+                logger.info(f"   {result.stdout}")
+
+            if self.verbose:
+                logger.info(f"   ✅ Compiled successfully")
+
+            # Fix imports if requested
+            if self.fix_imports:
+                self._fix_imports(proto_file, output_dir)
+
+            return True
+
+        except subprocess.CalledProcessError as e:
+            logger.error(f"   ❌ Compilation failed")
+            logger.error(f"   Error: {e.stderr}")
+            return False
+
+    def compile_directory(
+        self,
+        proto_path: Path,
+        recursive: bool = False,
+    ) -> tuple[int, int]:
+        """
+        Compile all .proto files in a directory.
+
+        Args:
+            proto_path: Directory containing .proto files
+            recursive: Recursively compile all .proto files
+
+        Returns:
+            Tuple of (success_count, failure_count)
+        """
+        # Collect proto files
+        proto_files = self._collect_proto_files(proto_path, recursive)
+
+        if not proto_files:
+            logger.warning(f"No .proto files found in: {proto_path}")
+            return 0, 0
+
+        if self.verbose:
+            logger.info(f"🔧 Compiling {len(proto_files)} proto file(s)...")
+
+        # Compile each proto file
+        success_count = 0
+        failure_count = 0
+
+        for proto_file in proto_files:
+            if self.compile_file(proto_file):
+                success_count += 1
+            else:
+                failure_count += 1
+
+        return success_count, failure_count
+
+    def _collect_proto_files(self, path: Path, recursive: bool) -> List[Path]:
+        """Collect all .proto files from path."""
+        if path.is_file():
+            if path.suffix == ".proto":
+                return [path]
+            else:
+                raise ValueError(f"File is not a .proto file: {path}")
+
+        # Directory
+        if recursive:
+            return list(path.rglob("*.proto"))
+        else:
+            return list(path.glob("*.proto"))
+
+    def _fix_imports(self, proto_file: Path, output_dir: Path):
+        """
+        Fix imports in generated _grpc.py files.
+
+        Changes: import foo_pb2 as foo__pb2
+        To:      from . import foo_pb2 as foo__pb2
+        """
+        # Find generated _grpc.py file
+        grpc_file = output_dir / f"{proto_file.stem}_pb2_grpc.py"
+
+        if not grpc_file.exists():
+            if self.verbose:
+                logger.warning(f"   ⚠️  Skipping import fix: {grpc_file.name} not found")
+            return
+
+        if self.verbose:
+            logger.info(f"   🔧 Fixing imports in {grpc_file.name}...")
+
+        # Read file
+        content = grpc_file.read_text()
+
+        # Pattern to match: import xxx_pb2 as yyy
+        # But NOT: from xxx import ...
+        pattern = r"^import (\w+_pb2) as (\w+)$"
+
+        # Replace with: from . import xxx_pb2 as yyy
+        def replace_func(match):
+            module = match.group(1)
+            alias = match.group(2)
+            return f"from . import {module} as {alias}"
+
+        # Apply replacement
+        new_content = re.sub(pattern, replace_func, content, flags=re.MULTILINE)
+
+        # Count changes
+        changes = content.count("\nimport ") - new_content.count("\nimport ")
+
+        if changes > 0:
+            # Write back
+            grpc_file.write_text(new_content)
+            if self.verbose:
+                logger.info(f"   ✅ Fixed {changes} import(s) in {grpc_file.name}")
+        else:
+            if self.verbose:
+                logger.info(f"   ℹ️  No imports to fix in {grpc_file.name}")