unrealon 1.1.0__py3-none-any.whl → 1.1.4__py3-none-any.whl

unrealon_driver/websocket/client.py

@@ -0,0 +1,249 @@
+"""
+Independent WebSocket Client for unrealon_driver.
+
+Provides WebSocket connectivity without dependencies on unrealon_server or unrealon_rpc.
+"""
+
+import asyncio
+import json
+import logging
+from typing import Optional, Dict, Any, Callable, Awaitable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+import websockets
+
+
+@dataclass
+class WebSocketConfig:
+    """WebSocket client configuration"""
+    url: str
+    api_key: Optional[str] = None
+    parser_id: Optional[str] = None
+    reconnect_interval: float = 5.0
+    max_reconnect_attempts: int = 10
+    ping_interval: float = 30.0
+    ping_timeout: float = 10.0
+
+
+class WebSocketClient:
+    """
+    Independent WebSocket client for driver-server communication.
+    
+    Features:
+    - Auto-reconnection
+    - Message queuing during disconnection
+    - Request-response pattern support
+    - Event-based message handling
+    """
+    
+    def __init__(self, config: WebSocketConfig):
+        self.config = config
+        self._websocket = None
+        self._connected = False
+        self._reconnect_task = None
+        self._message_handlers: Dict[str, Callable[[Dict[str, Any]], Awaitable[None]]] = {}
+        self._pending_requests: Dict[str, asyncio.Future] = {}
+        self._message_queue = []
+        self._logger = logging.getLogger(__name__)
+        
+    async def connect(self) -> bool:
+        """Connect to WebSocket server"""
+        try:
+            # Build connection parameters
+            connect_params = {
+                "ping_interval": self.config.ping_interval,
+                "ping_timeout": self.config.ping_timeout
+            }
+            
+            # Add headers if supported (websockets >= 10.0)
+            headers = {}
+            if self.config.api_key:
+                headers["Authorization"] = f"Bearer {self.config.api_key}"
+            if self.config.parser_id:
+                headers["X-Parser-ID"] = self.config.parser_id
+            
+            if headers:
+                try:
+                    # Try with extra_headers first (newer versions)
+                    connect_params["extra_headers"] = headers
+                    self._websocket = await websockets.connect(self.config.url, **connect_params)
+                except TypeError:
+                    # Fallback for older versions without extra_headers support
+                    connect_params.pop("extra_headers", None)
+                    self._websocket = await websockets.connect(self.config.url, **connect_params)
+            else:
+                self._websocket = await websockets.connect(self.config.url, **connect_params)
+            
+            self._connected = True
+            self._logger.info(f"Connected to WebSocket: {self.config.url}")
+            
+            # Start message listener
+            asyncio.create_task(self._message_listener())
+            
+            # Send queued messages
+            await self._send_queued_messages()
+            
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to connect to WebSocket: {e}")
+            self._connected = False
+            return False
+    
+    async def disconnect(self):
+        """Disconnect from WebSocket server"""
+        self._connected = False
+        
+        if self._reconnect_task:
+            self._reconnect_task.cancel()
+            
+        if self._websocket:
+            await self._websocket.close()
+            self._websocket = None
+            
+        self._logger.info("Disconnected from WebSocket")
+    
+    async def send_message(self, message: Dict[str, Any]) -> bool:
+        """Send message to server"""
+        if not self._connected or not self._websocket:
+            # Queue message for later
+            self._message_queue.append(message)
+            self._logger.debug("Message queued (not connected)")
+            return False
+            
+        try:
+            await self._websocket.send(json.dumps(message))
+            return True
+        except Exception as e:
+            self._logger.error(f"Failed to send message: {e}")
+            self._connected = False
+            self._message_queue.append(message)  # Re-queue
+            asyncio.create_task(self._reconnect())
+            return False
+    
+    async def send_request(self, message: Dict[str, Any], timeout: float = 30.0) -> Optional[Dict[str, Any]]:
+        """Send request and wait for response"""
+        import uuid
+        
+        request_id = str(uuid.uuid4())
+        message["request_id"] = request_id
+        
+        # Create future for response
+        future = asyncio.Future()
+        self._pending_requests[request_id] = future
+        
+        try:
+            # Send request
+            success = await self.send_message(message)
+            if not success:
+                return None
+                
+            # Wait for response
+            response = await asyncio.wait_for(future, timeout=timeout)
+            return response
+            
+        except asyncio.TimeoutError:
+            self._logger.error(f"Request timeout: {request_id}")
+            return None
+        except Exception as e:
+            self._logger.error(f"Request failed: {e}")
+            return None
+        finally:
+            # Clean up
+            self._pending_requests.pop(request_id, None)
+    
+    def add_message_handler(self, message_type: str, handler: Callable[[Dict[str, Any]], Awaitable[None]]):
+        """Add handler for specific message type"""
+        self._message_handlers[message_type] = handler
+    
+    async def _message_listener(self):
+        """Listen for incoming messages"""
+        try:
+            async for message_str in self._websocket:
+                try:
+                    message = json.loads(message_str)
+                    await self._handle_message(message)
+                except json.JSONDecodeError:
+                    self._logger.error(f"Invalid JSON received: {message_str}")
+                except Exception as e:
+                    self._logger.error(f"Error handling message: {e}")
+                    
+        except websockets.exceptions.ConnectionClosed:
+            self._logger.warning("WebSocket connection closed")
+            self._connected = False
+            asyncio.create_task(self._reconnect())
+        except Exception as e:
+            self._logger.error(f"Message listener error: {e}")
+            self._connected = False
+            asyncio.create_task(self._reconnect())
+    
+    async def _handle_message(self, message: Dict[str, Any]):
+        """Handle incoming message"""
+        # Check if it's a response to a pending request
+        request_id = message.get("request_id")
+        if request_id and request_id in self._pending_requests:
+            future = self._pending_requests[request_id]
+            if not future.done():
+                future.set_result(message)
+            return
+        
+        # Handle by message type
+        message_type = message.get("type")
+        if message_type and message_type in self._message_handlers:
+            try:
+                await self._message_handlers[message_type](message)
+            except Exception as e:
+                self._logger.error(f"Handler error for {message_type}: {e}")
+    
+    async def _send_queued_messages(self):
+        """Send all queued messages"""
+        while self._message_queue and self._connected:
+            message = self._message_queue.pop(0)
+            success = await self.send_message(message)
+            if not success:
+                # Re-queue and stop
+                self._message_queue.insert(0, message)
+                break
+    
+    async def _reconnect(self):
+        """Auto-reconnect with exponential backoff"""
+        if self._reconnect_task:
+            return  # Already reconnecting
+            
+        self._reconnect_task = asyncio.create_task(self._reconnect_loop())
+    
+    async def _reconnect_loop(self):
+        """Reconnection loop with backoff"""
+        attempt = 0
+        
+        while attempt < self.config.max_reconnect_attempts and not self._connected:
+            attempt += 1
+            wait_time = min(self.config.reconnect_interval * (2 ** (attempt - 1)), 60)
+            
+            self._logger.info(f"Reconnecting in {wait_time}s (attempt {attempt}/{self.config.max_reconnect_attempts})")
+            await asyncio.sleep(wait_time)
+            
+            if await self.connect():
+                self._logger.info("Reconnected successfully")
+                break
+        
+        if not self._connected:
+            self._logger.error("Max reconnection attempts reached")
+        
+        self._reconnect_task = None
+    
+    @property
+    def connected(self) -> bool:
+        """Check if connected"""
+        return self._connected and self._websocket is not None
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get connection statistics"""
+        return {
+            "connected": self._connected,
+            "url": self.config.url,
+            "queued_messages": len(self._message_queue),
+            "pending_requests": len(self._pending_requests),
+            "handlers": list(self._message_handlers.keys())
+        }

unrealon_driver/websocket/config.py

@@ -0,0 +1,188 @@
+"""
+Global WebSocket Configuration for UnrealOn Driver
+
+Provides automatic WebSocket URL detection and configuration management.
+No need to specify URLs in parser config files - everything is handled automatically.
+
+Strict compliance with CRITICAL_REQUIREMENTS.md:
+- Pydantic v2 models everywhere
+- No Dict[str, Any] usage
+- Complete type annotations
+- Proper error handling
+"""
+
+import os
+import socket
+from typing import Optional, Dict, Any
+from enum import Enum
+from pydantic import BaseModel, Field, ConfigDict, field_validator
+
+
+class Environment(str, Enum):
+    """Environment types for automatic URL detection"""
+    DEVELOPMENT = "development"
+    PRODUCTION = "production"
+    LOCAL = "local"
+
+
+class EnvironmentInfo(BaseModel):
+    """Environment detection information with full typing"""
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+    
+    detected_environment: Environment
+    websocket_url: str = Field(..., description="Detected WebSocket URL")
+    localhost_available: bool = Field(..., description="Whether localhost service is available")
+    environment_variables: Dict[str, Optional[str]] = Field(
+        default_factory=dict, 
+        description="Relevant environment variables"
+    )
+
+
+class GlobalWebSocketConfig(BaseModel):
+    """Global WebSocket configuration with automatic URL detection"""
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+    
+    # Production WebSocket URL
+    production_websocket_url: str = Field(
+        default="wss://ws.unrealon.com/ws",
+        description="Production WebSocket URL"
+    )
+    
+    # Development WebSocket URL
+    development_websocket_url: str = Field(
+        default="ws://localhost:8002/ws",
+        description="Development WebSocket URL"
+    )
+    
+    # Default environment
+    default_environment: Environment = Field(
+        default=Environment.PRODUCTION,
+        description="Default environment when detection fails"
+    )
+    
+    @field_validator('production_websocket_url', 'development_websocket_url')
+    @classmethod
+    def validate_websocket_url(cls, v: str) -> str:
+        """Validate WebSocket URL format"""
+        if not v.startswith(('ws://', 'wss://')):
+            raise ValueError("WebSocket URL must start with ws:// or wss://")
+        return v
+    
+    def get_websocket_url(self, environment: Optional[Environment] = None) -> str:
+        """Get WebSocket URL for specified environment"""
+        env = environment or self._detect_environment()
+        
+        if env == Environment.PRODUCTION:
+            return self.production_websocket_url
+        else:
+            # Development and Local both use localhost
+            return self.development_websocket_url
+    
+    def _detect_environment(self) -> Environment:
+        """Automatically detect environment based on various indicators"""
+        
+        # Check explicit environment variable
+        env_var = os.getenv("UNREALON_ENV", "").lower()
+        if env_var == Environment.PRODUCTION.value:
+            return Environment.PRODUCTION
+        elif env_var == Environment.DEVELOPMENT.value:
+            return Environment.DEVELOPMENT
+        elif env_var == Environment.LOCAL.value:
+            return Environment.LOCAL
+        
+        # Check if we're in development mode
+        if os.getenv("DEBUG", "").lower() in ("true", "1", "yes"):
+            return Environment.DEVELOPMENT
+        
+        # Check if localhost services are available
+        if self._is_localhost_available():
+            return Environment.DEVELOPMENT
+        
+        # Check common development indicators
+        development_indicators = [
+            os.getenv("NODE_ENV") == "development",
+            os.getenv("DJANGO_DEBUG", "").lower() in ("true", "1"),
+            os.getenv("FLASK_ENV") == "development",
+            os.path.exists(".env"),
+            os.path.exists("docker-compose.yml"),
+            os.path.exists("pyproject.toml") and os.getcwd().endswith("unrealon-rpc")
+        ]
+        
+        if any(development_indicators):
+            return Environment.DEVELOPMENT
+        
+        # Default to production for safety
+        return Environment.PRODUCTION
+    
+    def _is_localhost_available(self) -> bool:
+        """Check if localhost WebSocket service is available"""
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                sock.settimeout(1)
+                result = sock.connect_ex(('localhost', 8002))
+                return result == 0
+        except OSError:
+            return False
+        except Exception:
+            return False
+    
+    def get_environment_info(self) -> EnvironmentInfo:
+        """Get current environment information for debugging"""
+        current_env = self._detect_environment()
+        
+        env_vars = {
+            "UNREALON_ENV": os.getenv("UNREALON_ENV"),
+            "DEBUG": os.getenv("DEBUG"),
+            "NODE_ENV": os.getenv("NODE_ENV"),
+            "DJANGO_DEBUG": os.getenv("DJANGO_DEBUG"),
+        }
+        
+        return EnvironmentInfo(
+            detected_environment=current_env,
+            websocket_url=self.get_websocket_url(current_env),
+            localhost_available=self._is_localhost_available(),
+            environment_variables=env_vars
+        )
+
+
+# Global configuration instance
+global_websocket_config = GlobalWebSocketConfig()
+
+
+def get_websocket_url(environment: Optional[Environment] = None) -> str:
+    """Get WebSocket URL for current or specified environment"""
+    return global_websocket_config.get_websocket_url(environment)
+
+
+
+
+
+def get_environment() -> Environment:
+    """Get current detected environment"""
+    return global_websocket_config._detect_environment()
+
+
+def set_environment(environment: Environment) -> None:
+    """Override environment detection (for testing)"""
+    os.environ["UNREALON_ENV"] = environment.value
+
+
+def get_debug_info() -> EnvironmentInfo:
+    """Get debug information about current configuration"""
+    return global_websocket_config.get_environment_info()
+
+
+# Convenience functions for common use cases
+def is_production() -> bool:
+    """Check if running in production environment"""
+    return get_environment() == Environment.PRODUCTION
+
+
+def is_development() -> bool:
+    """Check if running in development environment"""
+    return get_environment() == Environment.DEVELOPMENT
+
+
+def is_local() -> bool:
+    """Check if running in local environment"""
+    return get_environment() == Environment.LOCAL

unrealon_driver/websocket/manager.py

@@ -0,0 +1,90 @@
+"""
+WebSocket Manager for unrealon_driver.
+
+Manages shared WebSocket connection for multiple use cases:
+- Logging transport
+- HTML analysis requests  
+- Other driver-server communication
+"""
+
+import asyncio
+from typing import Optional, Dict, Any, Callable, Awaitable
+from .client import WebSocketClient, WebSocketConfig
+
+
+class WebSocketManager:
+    """
+    Singleton WebSocket manager for the driver.
+    
+    Provides shared WebSocket connection for:
+    - SmartLogger (log batching)
+    - ParserManager (HTML analysis)
+    - Other driver components
+    """
+    
+    _instance: Optional['WebSocketManager'] = None
+    _client: Optional[WebSocketClient] = None
+    
+    def __new__(cls) -> 'WebSocketManager':
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        if not hasattr(self, '_initialized'):
+            self._initialized = True
+            self._client = None
+            self._config = None
+    
+    async def initialize(self, config: WebSocketConfig) -> bool:
+        """Initialize WebSocket connection"""
+        if self._client:
+            await self._client.disconnect()
+        
+        self._config = config
+        self._client = WebSocketClient(config)
+        
+        return await self._client.connect()
+    
+    async def send_message(self, message: Dict[str, Any]) -> bool:
+        """Send message via WebSocket"""
+        if not self._client:
+            return False
+        return await self._client.send_message(message)
+    
+    async def send_request(self, message: Dict[str, Any], timeout: float = 30.0) -> Optional[Dict[str, Any]]:
+        """Send request and wait for response"""
+        if not self._client:
+            return None
+        return await self._client.send_request(message, timeout)
+    
+    def add_message_handler(self, message_type: str, handler: Callable[[Dict[str, Any]], Awaitable[None]]):
+        """Add message handler"""
+        if self._client:
+            self._client.add_message_handler(message_type, handler)
+    
+    async def disconnect(self):
+        """Disconnect WebSocket"""
+        if self._client:
+            await self._client.disconnect()
+            self._client = None
+    
+    @property
+    def connected(self) -> bool:
+        """Check if connected"""
+        return self._client is not None and self._client.connected
+    
+    @property
+    def client(self) -> Optional[WebSocketClient]:
+        """Get underlying client"""
+        return self._client
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get connection statistics"""
+        if self._client:
+            return self._client.get_stats()
+        return {"connected": False}
+
+
+# Global instance
+websocket_manager = WebSocketManager()

unrealon-1.1.0.dist-info/METADATA

@@ -1,164 +0,0 @@
-Metadata-Version: 2.4
-Name: unrealon
-Version: 1.1.0
-Summary: Advanced browser automation framework with WebSocket bridge for distributed web scraping
-Project-URL: Homepage, https://github.com/unrealos/unrealon-rpc
-Project-URL: Documentation, https://unrealon-rpc.readthedocs.io
-Project-URL: Repository, https://github.com/unrealos/unrealon-rpc.git
-Project-URL: Issues, https://github.com/unrealos/unrealon-rpc/issues
-Project-URL: Changelog, https://github.com/unrealos/unrealon-rpc/blob/main/CHANGELOG.md
-Author-email: UnrealOS Team <dev@unrealos.com>
-Maintainer-email: UnrealOS Team <dev@unrealos.com>
-License: MIT
-License-File: LICENSE
-Keywords: async,bridge,browser-automation,html-processing,parsing,playwright,pydantic,stealth,web-scraping,websocket
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Topic :: Communications
-Classifier: Topic :: Internet :: WWW/HTTP
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Topic :: System :: Distributed Computing
-Classifier: Typing :: Typed
-Requires-Python: <4.0,>=3.10
-Requires-Dist: aiohttp>=3.9.0
-Requires-Dist: asyncio-mqtt>=0.16.0
-Requires-Dist: beautifulsoup4>=4.12.0
-Requires-Dist: click>=8.2.0
-Requires-Dist: httpx>=0.26.0
-Requires-Dist: ipfshttpclient>=0.8.0a2
-Requires-Dist: lxml>=5.0.0
-Requires-Dist: playwright-stealth<2.0.0,>=1.0.5
-Requires-Dist: playwright>=1.40.0
-Requires-Dist: pydantic<3.0,>=2.11
-Requires-Dist: python-dateutil>=2.8
-Requires-Dist: python-dotenv>=1.0.0
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: redis>=5.0.0
-Requires-Dist: rich>=13.0.0
-Requires-Dist: tomlkit>=0.13.0
-Requires-Dist: websockets>=12.0
-Provides-Extra: dev
-Requires-Dist: bandit>=1.7.0; extra == 'dev'
-Requires-Dist: black>=23.0.0; extra == 'dev'
-Requires-Dist: build>=1.0.0; extra == 'dev'
-Requires-Dist: flake8>=6.0.0; extra == 'dev'
-Requires-Dist: isort>=5.12.0; extra == 'dev'
-Requires-Dist: mkdocs-material>=9.0.0; extra == 'dev'
-Requires-Dist: mkdocs>=1.5.0; extra == 'dev'
-Requires-Dist: mkdocstrings[python]>=0.22.0; extra == 'dev'
-Requires-Dist: mypy>=1.5.0; extra == 'dev'
-Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
-Requires-Dist: pydocstyle>=6.3.0; extra == 'dev'
-Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
-Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
-Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
-Requires-Dist: pytest-xdist>=3.0.0; extra == 'dev'
-Requires-Dist: pytest>=7.0; extra == 'dev'
-Requires-Dist: questionary>=2.1.0; extra == 'dev'
-Requires-Dist: twine>=4.0.0; extra == 'dev'
-Provides-Extra: docs
-Requires-Dist: mkdocs-material>=9.0.0; extra == 'docs'
-Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
-Requires-Dist: mkdocstrings[python]>=0.22.0; extra == 'docs'
-Requires-Dist: pymdown-extensions>=10.0.0; extra == 'docs'
-Provides-Extra: test
-Requires-Dist: factory-boy>=3.2.0; extra == 'test'
-Requires-Dist: pytest-asyncio>=0.21.0; extra == 'test'
-Requires-Dist: pytest-cov>=4.0.0; extra == 'test'
-Requires-Dist: pytest-mock>=3.10.0; extra == 'test'
-Requires-Dist: pytest-xdist>=3.0.0; extra == 'test'
-Requires-Dist: pytest>=7.0; extra == 'test'
-Description-Content-Type: text/markdown
-
-# unrealon-rpc
-
-Universal async RPC + mesh + storage over Redis and IPFS with FastAPI WebSocket support.
-
-## Features
-
-- **Generic WebSocket Bridge**: Universal WebSocket-to-Redis communication layer
-- **Parser-Specific Bridge**: Domain-specific wrapper for parser orchestration
-- **Redis RPC/PubSub**: Asynchronous messaging primitives
-- **IPFS Storage**: Decentralized file storage integration
-- **Pydantic v2**: Strict type validation and data modeling
-- **Modular Architecture**: Clean separation of concerns with dependency injection
-
-## Installation
-
-```bash
-pip install unrealon-rpc
-```
-
-## Quick Start
-
-### Start WebSocket Server
-
-```bash
-# Production mode
-ws-server --host localhost --port 8000
-
-# Development mode with auto-reload (port 8001)
-ws-dev --host localhost --port 8001
-```
-
-### Start Parser Bridge Server
-
-```bash
-# Production mode
-parser-bridge --redis-url redis://localhost:6379/0
-
-# Development mode with auto-reload (port 8002)
-parser-dev --redis-url redis://localhost:6379/0
-```
-
-### Python Usage
-
-```python
-from bridge_parsers import ParserBridgeServer, ParserBridgeClient
-
-# Server
-server = ParserBridgeServer(redis_url="redis://localhost:6379/0")
-await server.start()
-
-# Client
-client = ParserBridgeClient(
-    websocket_url="ws://localhost:8000/ws",
-    parser_type="my_parser"
-)
-await client.connect()
-```
-
-## Architecture
-
-The system follows a clean architecture with:
-
-- **Generic Bridge** (`unrealon_rpc.bridge`): WebSocket-to-Redis communication
-- **Parser Bridge** (`bridge_parsers`): Domain-specific parser orchestration
-- **RPC Layer** (`unrealon_rpc.rpc`): Redis-based RPC implementation
-- **PubSub Layer** (`unrealon_rpc.pubsub`): Redis-based publish/subscribe
-
-## Testing
-
-```bash
-# Run all tests
-python tests/run_tests.py
-
-# Run specific test types
-python tests/run_tests.py --type unit
-python tests/run_tests.py --type e2e
-```
-
-## Documentation
-
-See the `@docs/` directory for comprehensive documentation.
-
-## License
-
-MIT License