unrealon 1.1.1__py3-none-any.whl → 1.1.4__py3-none-any.whl

unrealon_driver/websocket/client.py

@@ -0,0 +1,249 @@
+"""
+Independent WebSocket Client for unrealon_driver.
+
+Provides WebSocket connectivity without dependencies on unrealon_server or unrealon_rpc.
+"""
+
+import asyncio
+import json
+import logging
+from typing import Optional, Dict, Any, Callable, Awaitable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+import websockets
+
+
+@dataclass
+class WebSocketConfig:
+    """WebSocket client configuration"""
+    url: str
+    api_key: Optional[str] = None
+    parser_id: Optional[str] = None
+    reconnect_interval: float = 5.0
+    max_reconnect_attempts: int = 10
+    ping_interval: float = 30.0
+    ping_timeout: float = 10.0
+
+
+class WebSocketClient:
+    """
+    Independent WebSocket client for driver-server communication.
+    
+    Features:
+    - Auto-reconnection
+    - Message queuing during disconnection
+    - Request-response pattern support
+    - Event-based message handling
+    """
+    
+    def __init__(self, config: WebSocketConfig):
+        self.config = config
+        self._websocket = None
+        self._connected = False
+        self._reconnect_task = None
+        self._message_handlers: Dict[str, Callable[[Dict[str, Any]], Awaitable[None]]] = {}
+        self._pending_requests: Dict[str, asyncio.Future] = {}
+        self._message_queue = []
+        self._logger = logging.getLogger(__name__)
+        
+    async def connect(self) -> bool:
+        """Connect to WebSocket server"""
+        try:
+            # Build connection parameters
+            connect_params = {
+                "ping_interval": self.config.ping_interval,
+                "ping_timeout": self.config.ping_timeout
+            }
+            
+            # Add headers if supported (websockets >= 10.0)
+            headers = {}
+            if self.config.api_key:
+                headers["Authorization"] = f"Bearer {self.config.api_key}"
+            if self.config.parser_id:
+                headers["X-Parser-ID"] = self.config.parser_id
+            
+            if headers:
+                try:
+                    # Try with extra_headers first (newer versions)
+                    connect_params["extra_headers"] = headers
+                    self._websocket = await websockets.connect(self.config.url, **connect_params)
+                except TypeError:
+                    # Fallback for older versions without extra_headers support
+                    connect_params.pop("extra_headers", None)
+                    self._websocket = await websockets.connect(self.config.url, **connect_params)
+            else:
+                self._websocket = await websockets.connect(self.config.url, **connect_params)
+            
+            self._connected = True
+            self._logger.info(f"Connected to WebSocket: {self.config.url}")
+            
+            # Start message listener
+            asyncio.create_task(self._message_listener())
+            
+            # Send queued messages
+            await self._send_queued_messages()
+            
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to connect to WebSocket: {e}")
+            self._connected = False
+            return False
+    
+    async def disconnect(self):
+        """Disconnect from WebSocket server"""
+        self._connected = False
+        
+        if self._reconnect_task:
+            self._reconnect_task.cancel()
+            
+        if self._websocket:
+            await self._websocket.close()
+            self._websocket = None
+            
+        self._logger.info("Disconnected from WebSocket")
+    
+    async def send_message(self, message: Dict[str, Any]) -> bool:
+        """Send message to server"""
+        if not self._connected or not self._websocket:
+            # Queue message for later
+            self._message_queue.append(message)
+            self._logger.debug("Message queued (not connected)")
+            return False
+            
+        try:
+            await self._websocket.send(json.dumps(message))
+            return True
+        except Exception as e:
+            self._logger.error(f"Failed to send message: {e}")
+            self._connected = False
+            self._message_queue.append(message)  # Re-queue
+            asyncio.create_task(self._reconnect())
+            return False
+    
+    async def send_request(self, message: Dict[str, Any], timeout: float = 30.0) -> Optional[Dict[str, Any]]:
+        """Send request and wait for response"""
+        import uuid
+        
+        request_id = str(uuid.uuid4())
+        message["request_id"] = request_id
+        
+        # Create future for response
+        future = asyncio.Future()
+        self._pending_requests[request_id] = future
+        
+        try:
+            # Send request
+            success = await self.send_message(message)
+            if not success:
+                return None
+                
+            # Wait for response
+            response = await asyncio.wait_for(future, timeout=timeout)
+            return response
+            
+        except asyncio.TimeoutError:
+            self._logger.error(f"Request timeout: {request_id}")
+            return None
+        except Exception as e:
+            self._logger.error(f"Request failed: {e}")
+            return None
+        finally:
+            # Clean up
+            self._pending_requests.pop(request_id, None)
+    
+    def add_message_handler(self, message_type: str, handler: Callable[[Dict[str, Any]], Awaitable[None]]):
+        """Add handler for specific message type"""
+        self._message_handlers[message_type] = handler
+    
+    async def _message_listener(self):
+        """Listen for incoming messages"""
+        try:
+            async for message_str in self._websocket:
+                try:
+                    message = json.loads(message_str)
+                    await self._handle_message(message)
+                except json.JSONDecodeError:
+                    self._logger.error(f"Invalid JSON received: {message_str}")
+                except Exception as e:
+                    self._logger.error(f"Error handling message: {e}")
+                    
+        except websockets.exceptions.ConnectionClosed:
+            self._logger.warning("WebSocket connection closed")
+            self._connected = False
+            asyncio.create_task(self._reconnect())
+        except Exception as e:
+            self._logger.error(f"Message listener error: {e}")
+            self._connected = False
+            asyncio.create_task(self._reconnect())
+    
+    async def _handle_message(self, message: Dict[str, Any]):
+        """Handle incoming message"""
+        # Check if it's a response to a pending request
+        request_id = message.get("request_id")
+        if request_id and request_id in self._pending_requests:
+            future = self._pending_requests[request_id]
+            if not future.done():
+                future.set_result(message)
+            return
+        
+        # Handle by message type
+        message_type = message.get("type")
+        if message_type and message_type in self._message_handlers:
+            try:
+                await self._message_handlers[message_type](message)
+            except Exception as e:
+                self._logger.error(f"Handler error for {message_type}: {e}")
+    
+    async def _send_queued_messages(self):
+        """Send all queued messages"""
+        while self._message_queue and self._connected:
+            message = self._message_queue.pop(0)
+            success = await self.send_message(message)
+            if not success:
+                # Re-queue and stop
+                self._message_queue.insert(0, message)
+                break
+    
+    async def _reconnect(self):
+        """Auto-reconnect with exponential backoff"""
+        if self._reconnect_task:
+            return  # Already reconnecting
+            
+        self._reconnect_task = asyncio.create_task(self._reconnect_loop())
+    
+    async def _reconnect_loop(self):
+        """Reconnection loop with backoff"""
+        attempt = 0
+        
+        while attempt < self.config.max_reconnect_attempts and not self._connected:
+            attempt += 1
+            wait_time = min(self.config.reconnect_interval * (2 ** (attempt - 1)), 60)
+            
+            self._logger.info(f"Reconnecting in {wait_time}s (attempt {attempt}/{self.config.max_reconnect_attempts})")
+            await asyncio.sleep(wait_time)
+            
+            if await self.connect():
+                self._logger.info("Reconnected successfully")
+                break
+        
+        if not self._connected:
+            self._logger.error("Max reconnection attempts reached")
+        
+        self._reconnect_task = None
+    
+    @property
+    def connected(self) -> bool:
+        """Check if connected"""
+        return self._connected and self._websocket is not None
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get connection statistics"""
+        return {
+            "connected": self._connected,
+            "url": self.config.url,
+            "queued_messages": len(self._message_queue),
+            "pending_requests": len(self._pending_requests),
+            "handlers": list(self._message_handlers.keys())
+        }

unrealon_driver/websocket/config.py

@@ -0,0 +1,188 @@
+"""
+Global WebSocket Configuration for UnrealOn Driver
+
+Provides automatic WebSocket URL detection and configuration management.
+No need to specify URLs in parser config files - everything is handled automatically.
+
+Strict compliance with CRITICAL_REQUIREMENTS.md:
+- Pydantic v2 models everywhere
+- No Dict[str, Any] usage
+- Complete type annotations
+- Proper error handling
+"""
+
+import os
+import socket
+from typing import Optional, Dict, Any
+from enum import Enum
+from pydantic import BaseModel, Field, ConfigDict, field_validator
+
+
+class Environment(str, Enum):
+    """Environment types for automatic URL detection"""
+    DEVELOPMENT = "development"
+    PRODUCTION = "production"
+    LOCAL = "local"
+
+
+class EnvironmentInfo(BaseModel):
+    """Environment detection information with full typing"""
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+    
+    detected_environment: Environment
+    websocket_url: str = Field(..., description="Detected WebSocket URL")
+    localhost_available: bool = Field(..., description="Whether localhost service is available")
+    environment_variables: Dict[str, Optional[str]] = Field(
+        default_factory=dict, 
+        description="Relevant environment variables"
+    )
+
+
+class GlobalWebSocketConfig(BaseModel):
+    """Global WebSocket configuration with automatic URL detection"""
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+    
+    # Production WebSocket URL
+    production_websocket_url: str = Field(
+        default="wss://ws.unrealon.com/ws",
+        description="Production WebSocket URL"
+    )
+    
+    # Development WebSocket URL
+    development_websocket_url: str = Field(
+        default="ws://localhost:8002/ws",
+        description="Development WebSocket URL"
+    )
+    
+    # Default environment
+    default_environment: Environment = Field(
+        default=Environment.PRODUCTION,
+        description="Default environment when detection fails"
+    )
+    
+    @field_validator('production_websocket_url', 'development_websocket_url')
+    @classmethod
+    def validate_websocket_url(cls, v: str) -> str:
+        """Validate WebSocket URL format"""
+        if not v.startswith(('ws://', 'wss://')):
+            raise ValueError("WebSocket URL must start with ws:// or wss://")
+        return v
+    
+    def get_websocket_url(self, environment: Optional[Environment] = None) -> str:
+        """Get WebSocket URL for specified environment"""
+        env = environment or self._detect_environment()
+        
+        if env == Environment.PRODUCTION:
+            return self.production_websocket_url
+        else:
+            # Development and Local both use localhost
+            return self.development_websocket_url
+    
+    def _detect_environment(self) -> Environment:
+        """Automatically detect environment based on various indicators"""
+        
+        # Check explicit environment variable
+        env_var = os.getenv("UNREALON_ENV", "").lower()
+        if env_var == Environment.PRODUCTION.value:
+            return Environment.PRODUCTION
+        elif env_var == Environment.DEVELOPMENT.value:
+            return Environment.DEVELOPMENT
+        elif env_var == Environment.LOCAL.value:
+            return Environment.LOCAL
+        
+        # Check if we're in development mode
+        if os.getenv("DEBUG", "").lower() in ("true", "1", "yes"):
+            return Environment.DEVELOPMENT
+        
+        # Check if localhost services are available
+        if self._is_localhost_available():
+            return Environment.DEVELOPMENT
+        
+        # Check common development indicators
+        development_indicators = [
+            os.getenv("NODE_ENV") == "development",
+            os.getenv("DJANGO_DEBUG", "").lower() in ("true", "1"),
+            os.getenv("FLASK_ENV") == "development",
+            os.path.exists(".env"),
+            os.path.exists("docker-compose.yml"),
+            os.path.exists("pyproject.toml") and os.getcwd().endswith("unrealon-rpc")
+        ]
+        
+        if any(development_indicators):
+            return Environment.DEVELOPMENT
+        
+        # Default to production for safety
+        return Environment.PRODUCTION
+    
+    def _is_localhost_available(self) -> bool:
+        """Check if localhost WebSocket service is available"""
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                sock.settimeout(1)
+                result = sock.connect_ex(('localhost', 8002))
+                return result == 0
+        except OSError:
+            return False
+        except Exception:
+            return False
+    
+    def get_environment_info(self) -> EnvironmentInfo:
+        """Get current environment information for debugging"""
+        current_env = self._detect_environment()
+        
+        env_vars = {
+            "UNREALON_ENV": os.getenv("UNREALON_ENV"),
+            "DEBUG": os.getenv("DEBUG"),
+            "NODE_ENV": os.getenv("NODE_ENV"),
+            "DJANGO_DEBUG": os.getenv("DJANGO_DEBUG"),
+        }
+        
+        return EnvironmentInfo(
+            detected_environment=current_env,
+            websocket_url=self.get_websocket_url(current_env),
+            localhost_available=self._is_localhost_available(),
+            environment_variables=env_vars
+        )
+
+
+# Global configuration instance
+global_websocket_config = GlobalWebSocketConfig()
+
+
+def get_websocket_url(environment: Optional[Environment] = None) -> str:
+    """Get WebSocket URL for current or specified environment"""
+    return global_websocket_config.get_websocket_url(environment)
+
+
+
+
+
+def get_environment() -> Environment:
+    """Get current detected environment"""
+    return global_websocket_config._detect_environment()
+
+
+def set_environment(environment: Environment) -> None:
+    """Override environment detection (for testing)"""
+    os.environ["UNREALON_ENV"] = environment.value
+
+
+def get_debug_info() -> EnvironmentInfo:
+    """Get debug information about current configuration"""
+    return global_websocket_config.get_environment_info()
+
+
+# Convenience functions for common use cases
+def is_production() -> bool:
+    """Check if running in production environment"""
+    return get_environment() == Environment.PRODUCTION
+
+
+def is_development() -> bool:
+    """Check if running in development environment"""
+    return get_environment() == Environment.DEVELOPMENT
+
+
+def is_local() -> bool:
+    """Check if running in local environment"""
+    return get_environment() == Environment.LOCAL

unrealon_driver/websocket/manager.py

@@ -0,0 +1,90 @@
+"""
+WebSocket Manager for unrealon_driver.
+
+Manages shared WebSocket connection for multiple use cases:
+- Logging transport
+- HTML analysis requests  
+- Other driver-server communication
+"""
+
+import asyncio
+from typing import Optional, Dict, Any, Callable, Awaitable
+from .client import WebSocketClient, WebSocketConfig
+
+
+class WebSocketManager:
+    """
+    Singleton WebSocket manager for the driver.
+    
+    Provides shared WebSocket connection for:
+    - SmartLogger (log batching)
+    - ParserManager (HTML analysis)
+    - Other driver components
+    """
+    
+    _instance: Optional['WebSocketManager'] = None
+    _client: Optional[WebSocketClient] = None
+    
+    def __new__(cls) -> 'WebSocketManager':
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        if not hasattr(self, '_initialized'):
+            self._initialized = True
+            self._client = None
+            self._config = None
+    
+    async def initialize(self, config: WebSocketConfig) -> bool:
+        """Initialize WebSocket connection"""
+        if self._client:
+            await self._client.disconnect()
+        
+        self._config = config
+        self._client = WebSocketClient(config)
+        
+        return await self._client.connect()
+    
+    async def send_message(self, message: Dict[str, Any]) -> bool:
+        """Send message via WebSocket"""
+        if not self._client:
+            return False
+        return await self._client.send_message(message)
+    
+    async def send_request(self, message: Dict[str, Any], timeout: float = 30.0) -> Optional[Dict[str, Any]]:
+        """Send request and wait for response"""
+        if not self._client:
+            return None
+        return await self._client.send_request(message, timeout)
+    
+    def add_message_handler(self, message_type: str, handler: Callable[[Dict[str, Any]], Awaitable[None]]):
+        """Add message handler"""
+        if self._client:
+            self._client.add_message_handler(message_type, handler)
+    
+    async def disconnect(self):
+        """Disconnect WebSocket"""
+        if self._client:
+            await self._client.disconnect()
+            self._client = None
+    
+    @property
+    def connected(self) -> bool:
+        """Check if connected"""
+        return self._client is not None and self._client.connected
+    
+    @property
+    def client(self) -> Optional[WebSocketClient]:
+        """Get underlying client"""
+        return self._client
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get connection statistics"""
+        if self._client:
+            return self._client.get_stats()
+        return {"connected": False}
+
+
+# Global instance
+websocket_manager = WebSocketManager()