django-cfg 1.5.8__py3-none-any.whl → 1.5.14__py3-none-any.whl

django_cfg/apps/integrations/grpc/utils/integration_test.py

@@ -0,0 +1,334 @@
+"""
+gRPC Integration Test Utility.
+
+Comprehensive integration test for gRPC with API keys.
+Automatically performs all steps from proto generation to log verification.
+"""
+
+import sys
+import time
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+import grpc
+from django.conf import settings
+from django.contrib.auth import get_user_model
+
+from django_cfg.apps.integrations.grpc.models import GrpcApiKey, GRPCRequestLog
+
+User = get_user_model()
+
+
+class GRPCIntegrationTest:
+    """Comprehensive integration test for gRPC with API keys."""
+
+    def __init__(self, app_label: str = "crypto", quiet: bool = False):
+        """
+        Initialize integration test.
+
+        Args:
+            app_label: Django app label to generate protos for
+            quiet: Suppress verbose output
+        """
+        self.app_label = app_label
+        self.quiet = quiet
+        self.server_process: Optional[subprocess.Popen] = None
+        self.api_key: Optional[GrpcApiKey] = None
+        self.grpc_port = settings.GRPC_SERVER.get("port", 50051)
+
+    def log(self, message: str):
+        """Print message if not quiet."""
+        if not self.quiet:
+            print(message)
+
+    def print_step(self, step_num: int, message: str):
+        """Print step with formatting."""
+        if not self.quiet:
+            print(f"\n{'='*70}")
+            print(f"🔹 Step {step_num}: {message}")
+            print(f"{'='*70}")
+
+    def step1_generate_protos(self) -> bool:
+        """Step 1: Generate proto files."""
+        self.print_step(1, f"Generating proto files for {self.app_label}")
+
+        try:
+            result = subprocess.run(
+                [sys.executable, "manage.py", "generate_protos", self.app_label],
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            self.log("✅ Proto files generated successfully")
+            return True
+
+        except subprocess.CalledProcessError as e:
+            self.log(f"❌ Proto generation error: {e}")
+            if e.stderr:
+                self.log(f"   STDERR: {e.stderr}")
+            return False
+
+    def step2_start_server(self) -> bool:
+        """Step 2: Start gRPC server."""
+        self.print_step(2, "Starting gRPC server")
+
+        try:
+            self.server_process = subprocess.Popen(
+                [sys.executable, "manage.py", "rungrpc"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True
+            )
+
+            self.log(f"✅ gRPC server started (PID: {self.server_process.pid})")
+            self.log(f"   Port: {self.grpc_port}")
+            self.log("   Waiting for server to start...")
+            time.sleep(3)
+
+            if self.server_process.poll() is not None:
+                stdout, stderr = self.server_process.communicate()
+                self.log(f"❌ Server terminated prematurely")
+                self.log(f"   STDOUT: {stdout}")
+                self.log(f"   STDERR: {stderr}")
+                return False
+
+            self.log("✅ Server is running")
+            return True
+
+        except Exception as e:
+            self.log(f"❌ Server startup error: {e}")
+            return False
+
+    def step3_create_api_key(self) -> bool:
+        """Step 3: Create test API key."""
+        self.print_step(3, "Creating test API key")
+
+        try:
+            user = User.objects.filter(is_active=True).first()
+            if not user:
+                self.log("   Creating test user...")
+                user = User.objects.create_user(
+                    username="grpc_integration_test",
+                    email="grpc_test@example.com",
+                    password="test_password_123",
+                    is_active=True
+                )
+                self.log(f"   ✅ Created user: {user.username}")
+            else:
+                self.log(f"   ✅ Using existing user: {user.username}")
+
+            self.api_key = GrpcApiKey.objects.create_for_user(
+                user=user,
+                name="Integration Test Key",
+                key_type="development",
+                expires_in_days=None,
+            )
+
+            self.log(f"✅ API key created")
+            self.log(f"   Name: {self.api_key.name}")
+            self.log(f"   Key: {self.api_key.key[:32]}...")
+            self.log(f"   User: {self.api_key.user.username}")
+            self.log(f"   Valid: {self.api_key.is_valid}")
+
+            return True
+
+        except Exception as e:
+            self.log(f"❌ API key creation error: {e}")
+            import traceback
+            if not self.quiet:
+                traceback.print_exc()
+            return False
+
+    def step4_test_client(self) -> bool:
+        """Step 4: Test gRPC client."""
+        self.print_step(4, "Testing gRPC client with API key")
+
+        try:
+            import importlib
+            module_path = f"apps.{self.app_label}.grpc_services.generated"
+
+            try:
+                service_name = f"{self.app_label}_service"
+                proto_module = importlib.import_module(f"{module_path}.{service_name}_pb2")
+                grpc_module = importlib.import_module(f"{module_path}.{service_name}_pb2_grpc")
+                service_stub_name = f"{self.app_label.capitalize()}ServiceStub"
+            except ImportError:
+                try:
+                    proto_module = importlib.import_module(f"{module_path}.coin_pb2")
+                    grpc_module = importlib.import_module(f"{module_path}.coin_pb2_grpc")
+                    service_stub_name = "CoinServiceStub"
+                except ImportError:
+                    self.log(f"❌ Failed to import proto files from {module_path}")
+                    return False
+
+            server_address = f"localhost:{self.grpc_port}"
+            self.log(f"   Connecting to: {server_address}")
+
+            StubClass = getattr(grpc_module, service_stub_name)
+
+            # Test 1: Valid API key
+            self.log("\n   📝 Test 1: Authentication with valid API key")
+            with grpc.insecure_channel(server_address) as channel:
+                stub = StubClass(channel)
+                metadata = [("x-api-key", self.api_key.key)]
+
+                request = proto_module.GetCoinRequest(symbol="BTC")
+                response = stub.GetCoin(request, metadata=metadata)
+
+                self.log(f"   ✅ Request successful: {response.coin.symbol} - {response.coin.name}")
+
+            # Test 2: Django SECRET_KEY
+            self.log("\n   📝 Test 2: Authentication with Django SECRET_KEY")
+            with grpc.insecure_channel(server_address) as channel:
+                stub = StubClass(channel)
+                metadata = [("x-api-key", settings.SECRET_KEY)]
+
+                request = proto_module.GetCoinRequest(symbol="ETH")
+                response = stub.GetCoin(request, metadata=metadata)
+
+                self.log(f"   ✅ SECRET_KEY works: {response.coin.symbol} - {response.coin.name}")
+
+            # Test 3: Invalid key
+            self.log("\n   📝 Test 3: Testing invalid key")
+            try:
+                with grpc.insecure_channel(server_address) as channel:
+                    stub = StubClass(channel)
+                    metadata = [("x-api-key", "invalid_key_12345")]
+
+                    request = proto_module.GetCoinRequest(symbol="BTC")
+                    response = stub.GetCoin(request, metadata=metadata)
+
+                    self.log(f"   ⚠️  Invalid key was accepted (require_auth=False)")
+
+            except grpc.RpcError as e:
+                if e.code() == grpc.StatusCode.UNAUTHENTICATED:
+                    self.log(f"   ✅ Invalid key correctly rejected")
+                else:
+                    self.log(f"   ⚠️  Unexpected error: {e.code()} - {e.details()}")
+
+            self.log("\n✅ All client tests passed")
+            return True
+
+        except grpc.RpcError as e:
+            self.log(f"❌ gRPC error: {e.code()} - {e.details()}")
+            return False
+        except Exception as e:
+            self.log(f"❌ Client testing error: {e}")
+            import traceback
+            if not self.quiet:
+                traceback.print_exc()
+            return False
+
+    def step5_verify_logs(self) -> bool:
+        """Step 5: Verify request logs."""
+        self.print_step(5, "Verifying request logs")
+
+        try:
+            self.api_key.refresh_from_db()
+
+            self.log(f"📊 API key statistics:")
+            self.log(f"   Request count: {self.api_key.request_count}")
+            self.log(f"   Last used: {self.api_key.last_used_at}")
+
+            logs_with_key = GRPCRequestLog.objects.filter(api_key=self.api_key)
+            logs_without_key = GRPCRequestLog.objects.filter(
+                api_key__isnull=True,
+                is_authenticated=True
+            )
+
+            self.log(f"\n📝 Request logs:")
+            self.log(f"   With API key: {logs_with_key.count()}")
+            self.log(f"   With SECRET_KEY: {logs_without_key.count()}")
+            self.log(f"   Total logs: {GRPCRequestLog.objects.count()}")
+
+            if logs_with_key.exists() and not self.quiet:
+                self.log(f"\n   Recent requests with API key:")
+                for log in logs_with_key.order_by("-created_at")[:3]:
+                    self.log(f"   - {log.method_name}: {log.status} ({log.duration_ms}ms)")
+                    self.log(f"     API Key: {log.api_key.name if log.api_key else 'None'}")
+                    self.log(f"     User: {log.user.username if log.user else 'None'}")
+
+            self.log("\n✅ Logs correctly recorded with api_key")
+            return True
+
+        except Exception as e:
+            self.log(f"❌ Log verification error: {e}")
+            import traceback
+            if not self.quiet:
+                traceback.print_exc()
+            return False
+
+    def step6_cleanup(self) -> bool:
+        """Step 6: Clean up test data."""
+        self.print_step(6, "Cleaning up test data")
+
+        try:
+            if self.server_process:
+                self.log("   Stopping gRPC server...")
+                self.server_process.terminate()
+                try:
+                    self.server_process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    self.server_process.kill()
+                self.log(f"   ✅ Server stopped (PID: {self.server_process.pid})")
+
+            if self.api_key:
+                self.log(f"   Deleting API key: {self.api_key.name}...")
+                self.api_key.delete()
+                self.log("   ✅ API key deleted")
+
+            self.log("\n✅ Cleanup completed")
+            return True
+
+        except Exception as e:
+            self.log(f"❌ Cleanup error: {e}")
+            return False
+
+    def run(self) -> bool:
+        """Run full integration test."""
+        self.log("=" * 70)
+        self.log("🧪 Comprehensive gRPC API Keys Integration Test")
+        self.log("=" * 70)
+
+        results = []
+
+        results.append(("Proto generation", self.step1_generate_protos()))
+
+        if results[-1][1]:
+            results.append(("Server startup", self.step2_start_server()))
+
+        if results[-1][1]:
+            results.append(("API key creation", self.step3_create_api_key()))
+
+        if results[-1][1]:
+            results.append(("Client testing", self.step4_test_client()))
+
+        if results[-1][1]:
+            results.append(("Log verification", self.step5_verify_logs()))
+
+        results.append(("Cleanup", self.step6_cleanup()))
+
+        self.log("\n" + "=" * 70)
+        self.log("📊 Integration Test Results")
+        self.log("=" * 70)
+
+        success_count = sum(1 for _, success in results if success)
+        total_count = len(results)
+
+        for step_name, success in results:
+            status = "✅" if success else "❌"
+            self.log(f"{status} {step_name}")
+
+        self.log("\n" + "=" * 70)
+        if success_count == total_count:
+            self.log(f"🎉 All tests passed successfully! ({success_count}/{total_count})")
+            self.log("=" * 70)
+            return True
+        else:
+            self.log(f"⚠️  Tests passed: {success_count}/{total_count}")
+            self.log("=" * 70)
+            return False
+
+
+__all__ = ["GRPCIntegrationTest"]

django_cfg/apps/integrations/grpc/utils/proto_gen.py

@@ -184,7 +184,21 @@ class ProtoGenerator:
 
         field_number = 1
 
-        # Add id field
+        # Track fields to skip to avoid duplicates
+        fields_to_skip = set()
+
+        # If we'll add id separately, skip it in the field iteration
+        if include_id:
+            fields_to_skip.add('id')
+
+        # If we'll add timestamps separately, skip them in the field iteration
+        if include_timestamps:
+            if hasattr(model, "created_at"):
+                fields_to_skip.add('created_at')
+            if hasattr(model, "updated_at"):
+                fields_to_skip.add('updated_at')
+
+        # Add id field first if requested
         if include_id:
             lines.append(f"  int64 id = {field_number};")
             field_number += 1
@@ -199,6 +213,10 @@ class ProtoGenerator:
             if isinstance(field, models.ManyToManyField):
                 continue
 
+            # Skip fields that will be added separately to avoid duplicates
+            if field.name in fields_to_skip:
+                continue
+
             # Get field info
             field_name = self._format_field_name(field.name)
             proto_type = self.mapper.get_proto_type(field)
@@ -213,13 +231,13 @@ class ProtoGenerator:
             lines.append(field_def)
             field_number += 1
 
-        # Add timestamp fields if requested
+        # Add timestamp fields last if requested
         if include_timestamps:
             if hasattr(model, "created_at"):
-                lines.append(f"  string created_at = {field_number};")
+                lines.append(f"  optional string created_at = {field_number};")
                 field_number += 1
             if hasattr(model, "updated_at"):
-                lines.append(f"  string updated_at = {field_number};")
+                lines.append(f"  optional string updated_at = {field_number};")
                 field_number += 1
 
         lines.append("}")
@@ -375,12 +393,25 @@ def generate_proto_for_app(app_label: str, output_dir: Optional[Path] = None) ->
         print(f"Generated {count} proto file(s)")
         ```
     """
+    # Get gRPC config from django-cfg (Pydantic)
+    from ..services.config_helper import get_grpc_config
+
+    grpc_config = get_grpc_config()
+    proto_config = grpc_config.proto if grpc_config else None
+
     # Get output directory
     if output_dir is None:
-        grpc_proto_config = getattr(settings, "GRPC_PROTO", {})
-        output_dir_str = grpc_proto_config.get("output_dir", "protos")
+        if proto_config:
+            output_dir_str = proto_config.output_dir
+        else:
+            output_dir_str = "protos"  # Fallback
+
         output_dir = Path(output_dir_str)
 
+        # Make absolute if relative
+        if not output_dir.is_absolute():
+            output_dir = settings.BASE_DIR / output_dir
+
     # Get app config
     try:
         app_config = apps.get_app_config(app_label)
@@ -398,10 +429,19 @@ def generate_proto_for_app(app_label: str, output_dir: Optional[Path] = None) ->
         logger.warning(f"No models found in app '{app_label}'")
         return 0
 
+    # Build package name: combine prefix + app_label
+    if proto_config and proto_config.package_prefix:
+        full_package = f"{proto_config.package_prefix}.{app_label}"
+    else:
+        full_package = app_label
+
+    # Get field naming
+    field_naming = proto_config.field_naming if proto_config else "snake_case"
+
     # Generate proto file
     generator = ProtoGenerator(
-        package_prefix=app_label,
-        field_naming="snake_case",
+        package_prefix=full_package,
+        field_naming=field_naming,
     )
 
     output_path = output_dir / f"{app_label}.proto"

django_cfg/apps/integrations/grpc/utils/streaming_logger.py

@@ -0,0 +1,177 @@
+"""
+Streaming Logger Utilities for gRPC Services.
+
+Provides reusable logger configuration for gRPC streaming services.
+Follows django-cfg logging patterns for consistency.
+"""
+
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+from django.utils import timezone
+
+
+class AutoTracebackHandler(logging.Handler):
+    """
+    Custom handler that automatically adds exception info to ERROR and CRITICAL logs.
+
+    This ensures full tracebacks are always logged for errors, even if exc_info=True
+    is not explicitly specified.
+    """
+
+    def __init__(self, base_handler: logging.Handler):
+        super().__init__()
+        self.base_handler = base_handler
+        self.setLevel(base_handler.level)
+        self.setFormatter(base_handler.formatter)
+
+    def emit(self, record: logging.LogRecord):
+        """Emit log record, automatically adding exc_info for errors."""
+        # If ERROR or CRITICAL and no exc_info yet, add current exception if any
+        if record.levelno >= logging.ERROR and not record.exc_info:
+            # Check if we're in exception context
+            exc_info = sys.exc_info()
+            if exc_info[0] is not None:
+                record.exc_info = exc_info
+
+        # Delegate to base handler
+        self.base_handler.emit(record)
+
+
+def setup_streaming_logger(
+    name: str = "grpc_streaming",
+    logs_dir: Optional[Path] = None,
+    level: int = logging.DEBUG,
+    console_level: int = logging.INFO
+) -> logging.Logger:
+    """
+    Setup dedicated logger for gRPC streaming with file and console handlers.
+
+    Follows django-cfg logging pattern:
+    - Uses os.getcwd() / 'logs' / 'grpc_streaming' for log directory
+    - Time-based log file names (streaming_YYYYMMDD_HHMMSS.log)
+    - Detailed file logging (DEBUG level by default)
+    - Concise console logging (INFO level by default)
+
+    Args:
+        name: Logger name (default: "grpc_streaming")
+        logs_dir: Directory for log files (default: <cwd>/logs/grpc_streaming)
+        level: File logging level (default: DEBUG)
+        console_level: Console logging level (default: INFO)
+
+    Returns:
+        Configured logger instance
+
+    Example:
+        ```python
+        from django_cfg.apps.integrations.grpc.utils import setup_streaming_logger
+
+        # Basic usage
+        logger = setup_streaming_logger()
+
+        # Custom configuration
+        logger = setup_streaming_logger(
+            name="my_streaming_service",
+            logs_dir=Path("/var/log/grpc"),
+            level=logging.INFO
+        )
+
+        logger.info("Service started")
+        logger.debug("Detailed debug info")
+        ```
+
+    Features:
+        - Automatic log directory creation
+        - Time-based log file names
+        - No duplicate logs (propagate=False)
+        - UTF-8 encoding
+        - Reusable across all django-cfg gRPC projects
+    """
+    # Create logger
+    streaming_logger = logging.getLogger(name)
+    streaming_logger.setLevel(level)
+
+    # Avoid duplicate handlers if logger already configured
+    if streaming_logger.handlers:
+        return streaming_logger
+
+    # Determine logs directory using django-cfg pattern
+    if logs_dir is None:
+        # Pattern from django_cfg.modules.django_logging:
+        # current_dir = Path(os.getcwd())
+        # logs_dir = current_dir / 'logs' / 'grpc_streaming'
+        current_dir = Path(os.getcwd())
+        logs_dir = current_dir / 'logs' / 'grpc_streaming'
+
+    # Create logs directory
+    logs_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create log filename with timestamp
+    log_filename = f'streaming_{timezone.now().strftime("%Y%m%d_%H%M%S")}.log'
+    log_file_path = logs_dir / log_filename
+
+    # File handler - detailed logs with auto-traceback
+    base_file_handler = logging.FileHandler(
+        log_file_path,
+        encoding='utf-8'
+    )
+    base_file_handler.setLevel(level)
+    file_formatter = logging.Formatter(
+        '%(asctime)s | %(levelname)-8s | %(message)s',
+        datefmt='%H:%M:%S'
+    )
+    base_file_handler.setFormatter(file_formatter)
+
+    # Wrap with auto-traceback handler for automatic exc_info on errors
+    file_handler = AutoTracebackHandler(base_file_handler)
+    streaming_logger.addHandler(file_handler)
+
+    # Console handler - important messages only
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(console_level)
+    console_formatter = logging.Formatter('%(levelname)s: %(message)s')
+    console_handler.setFormatter(console_formatter)
+    streaming_logger.addHandler(console_handler)
+
+    # Prevent propagation to avoid duplicate logs
+    streaming_logger.propagate = False
+
+    # Log initialization
+    streaming_logger.info("=" * 80)
+    streaming_logger.info(f"🌊 {name.title()} Logger Initialized")
+    streaming_logger.info(f"📁 Log file: {log_file_path}")
+    streaming_logger.info("=" * 80)
+
+    return streaming_logger
+
+
+def get_streaming_logger(name: str = "grpc_streaming") -> logging.Logger:
+    """
+    Get existing streaming logger or create new one.
+
+    Args:
+        name: Logger name (default: "grpc_streaming")
+
+    Returns:
+        Logger instance
+
+    Example:
+        ```python
+        from django_cfg.apps.integrations.grpc.utils import get_streaming_logger
+
+        logger = get_streaming_logger()
+        logger.info("Using existing logger")
+        ```
+    """
+    logger = logging.getLogger(name)
+
+    # If not configured yet, set it up
+    if not logger.handlers:
+        return setup_streaming_logger(name)
+
+    return logger
+
+
+__all__ = ["setup_streaming_logger", "get_streaming_logger"]

django_cfg/apps/integrations/grpc/views/__init__.py

@@ -2,8 +2,10 @@
 Views for gRPC monitoring API.
 """
 
+from .api_keys import GRPCApiKeyViewSet
 from .config import GRPCConfigViewSet
 from .monitoring import GRPCMonitorViewSet
+from .proto_files import GRPCProtoFilesViewSet
 from .services import GRPCServiceViewSet
 from .testing import GRPCTestingViewSet
 
@@ -12,4 +14,6 @@ __all__ = [
     "GRPCConfigViewSet",
     "GRPCServiceViewSet",
     "GRPCTestingViewSet",
+    "GRPCApiKeyViewSet",
+    "GRPCProtoFilesViewSet",
 ]