unrealon 1.1.6__py3-none-any.whl → 2.0.5__py3-none-any.whl

unrealon_driver/driver/core/config.py

@@ -0,0 +1,175 @@
+"""
+Clean driver configuration without hardcoded values.
+"""
+
+import os
+from enum import Enum
+from typing import Optional
+from pydantic import BaseModel, Field, computed_field
+
+try:
+    from unrealon_core.config.environment import get_environment_config
+    UNREALON_CORE_AVAILABLE = True
+except ImportError:
+    UNREALON_CORE_AVAILABLE = False
+
+
+class DriverMode(str, Enum):
+    """Driver operation modes."""
+    STANDALONE = "standalone"
+    DAEMON = "daemon"
+
+
+class DriverConfig(BaseModel):
+    """
+    Clean driver configuration.
+    No hardcoded presets - user configures everything explicitly.
+    """
+    
+    # Basic settings
+    name: str = Field(..., description="Driver name")
+    mode: DriverMode = Field(default=DriverMode.STANDALONE, description="Operation mode")
+    
+    # WebSocket connection (auto-detected)
+    websocket_url: Optional[str] = Field(default=None, description="Manual WebSocket URL override")
+    websocket_timeout: int = Field(default=30, description="WebSocket timeout seconds")
+    
+    @computed_field
+    @property
+    def effective_websocket_url(self) -> Optional[str]:
+        """
+        Auto-detect WebSocket URL from multiple sources.
+        
+        Priority:
+        1. Explicit websocket_url field
+        2. Environment variables (UNREALON_WEBSOCKET_URL, UNREALON_WS_URL, WS_URL)
+        3. UnrealOn core environment config (if available)
+        4. No default - return None if nothing configured
+        """
+        # 1. Explicit override
+        if self.websocket_url:
+            return self.websocket_url
+        
+        # 2. Environment variables
+        env_url = (
+            os.getenv('UNREALON_WEBSOCKET_URL') or 
+            os.getenv('UNREALON_WS_URL') or
+            os.getenv('WS_URL')
+        )
+        if env_url:
+            return env_url
+        
+        # 3. Try unrealon_core environment config
+        if UNREALON_CORE_AVAILABLE:
+            try:
+                env_config = get_environment_config()
+                return env_config.websocket_url
+            except Exception:
+                pass  # Fallback gracefully if core config fails
+        
+        # 4. No default - return None if nothing is configured
+        return None
+    
+    # Logging
+    log_level: str = Field(default="INFO", description="Logging level")
+    log_file: Optional[str] = Field(default=None, description="Log file path")
+    
+    # HTTP settings
+    http_timeout: int = Field(default=30, description="HTTP timeout seconds")
+    max_retries: int = Field(default=3, description="Max HTTP retries")
+    
+    # Browser settings
+    browser_headless: bool = Field(default=True, description="Run browser headless")
+    browser_timeout: int = Field(default=30, description="Browser timeout seconds")
+    
+    # Proxy settings
+    proxy_enabled: bool = Field(default=False, description="Enable proxy rotation")
+    proxy_rotation_interval: int = Field(default=300, description="Proxy rotation seconds")
+    
+    # Cache settings
+    cache_enabled: bool = Field(default=True, description="Enable response caching")
+    cache_ttl: int = Field(default=3600, description="Cache TTL seconds")
+    
+    # Threading
+    max_workers: int = Field(default=4, description="Max thread workers")
+    
+    # Performance
+    batch_size: int = Field(default=10, description="Batch processing size")
+    
+    model_config = {"extra": "forbid"}
+    
+    @classmethod
+    def for_development(cls, name: str, **kwargs) -> "DriverConfig":
+        """
+        Create development configuration with sensible defaults.
+        
+        Args:
+            name: Driver name
+            **kwargs: Additional configuration overrides
+        """
+        defaults = {
+            "name": name,
+            "mode": DriverMode.STANDALONE,
+            "log_level": "DEBUG",
+            "browser_headless": False,
+            "proxy_enabled": False,
+            "cache_enabled": True,
+            "max_workers": 2,
+        }
+        defaults.update(kwargs)
+        return cls(**defaults)
+    
+    @classmethod
+    def for_production(cls, name: str, **kwargs) -> "DriverConfig":
+        """
+        Create production configuration with performance and reliability defaults.
+        
+        Args:
+            name: Driver name
+            **kwargs: Additional configuration overrides
+        """
+        defaults = {
+            "name": name,
+            "mode": DriverMode.DAEMON,
+            "log_level": "INFO",
+            "browser_headless": True,
+            "proxy_enabled": True,
+            "cache_enabled": True,
+            "max_workers": 4,
+            "max_retries": 5,
+            "http_timeout": 60,
+            "browser_timeout": 60,
+        }
+        defaults.update(kwargs)
+        return cls(**defaults)
+    
+    @classmethod
+    def auto_detect(cls, name: str, **kwargs) -> "DriverConfig":
+        """
+        Auto-detect environment and create appropriate configuration.
+        
+        Uses UNREALON_ENV environment variable or unrealon_core config to determine environment.
+        
+        Args:
+            name: Driver name
+            **kwargs: Additional configuration overrides
+        """
+        # Try to detect environment
+        env_name = os.getenv("UNREALON_ENV", "development").lower()
+        
+        # Try unrealon_core if available
+        if UNREALON_CORE_AVAILABLE:
+            try:
+                env_config = get_environment_config()
+                if env_config.is_production:
+                    return cls.for_production(name, **kwargs)
+                elif env_config.is_development:
+                    return cls.for_development(name, **kwargs)
+            except Exception:
+                pass  # Fallback to env variable
+        
+        # Fallback to environment variable
+        if env_name in ("prod", "production"):
+            return cls.for_production(name, **kwargs)
+        else:
+            return cls.for_development(name, **kwargs)

unrealon_driver/driver/core/driver.py

@@ -0,0 +1,221 @@
+"""
+UniversalDriver - Clean, modular driver orchestrator.
+
+The main driver class that coordinates all components through a clean,
+organized architecture with separated concerns.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional, List, Dict, Any, Callable
+
+from .config import DriverConfig
+from ..communication.websocket_client import WebSocketClient
+from ..communication.session import DriverSession
+from ..factory.manager_factory import ManagerFactory
+from ..lifecycle.initialization import DriverInitializer
+from ..lifecycle.shutdown import DriverShutdown
+from ..lifecycle.daemon import DaemonManager
+from ..monitoring.health import HealthMonitor
+from ..utilities.logging import LoggingUtility
+from ..utilities.serialization import SerializationUtility
+
+from ...managers import (
+    LoggerManager, HttpManager, BrowserManager, CacheManager,
+    ProxyManager, ThreadManager, UpdateManager, ManagerRegistry
+)
+
+from ...core_module import (
+    EventManager,
+    ModuleRegistry
+)
+
+from ...decorators import task, retry, schedule, timing
+
+logger = logging.getLogger(__name__)
+
+
+class UniversalDriver:
+    """
+    Universal Driver - Clean orchestrator for all parsing operations.
+    
+    Key Features:
+    - Zero-config initialization with sensible defaults
+    - Modular manager system (HTTP, Browser, Cache, etc.)
+    - WebSocket communication for RPC and logging
+    - Graceful shutdown with signal handling
+    - Built-in utilities for common operations
+    - Decorator-based task registration
+    
+    Usage:
+        # Basic usage
+        config = DriverConfig.for_development("my_parser")
+        driver = UniversalDriver(config)
+        await driver.initialize()
+        
+        # Daemon mode with RPC
+        await driver.run_daemon_mode()
+        
+        # Standalone mode
+        await driver.initialize()
+        # ... your parsing logic
+        await driver.shutdown()
+    """
+    
+    def __init__(self, config: DriverConfig):
+        """Initialize UniversalDriver with configuration."""
+        self.config = config
+        self.driver_id = config.name
+        self.is_initialized = False
+        self.capabilities: List[str] = []
+        
+        # Core components
+        self.websocket_client: Optional[WebSocketClient] = None
+        self.session: Optional[DriverSession] = None
+        
+        # Manager system (initialized by factory)
+        self.manager_registry: Optional[ManagerRegistry] = None
+        self.logger_manager: Optional[LoggerManager] = None
+        self.http: Optional[HttpManager] = None
+        self.browser: Optional[BrowserManager] = None
+        self.cache: Optional[CacheManager] = None
+        self.proxy: Optional[ProxyManager] = None
+        self.threading: Optional[ThreadManager] = None
+        self.update: Optional[UpdateManager] = None
+        
+        # Module system
+        self.event_manager = EventManager()
+        self.module_registry = ModuleRegistry(self.event_manager)
+        
+        # Utilities
+        self._logging_util = LoggingUtility(self.driver_id)
+        
+        # Decorators (exposed as instance methods for convenience)
+        self.task = task
+        self.retry = retry
+        self.schedule = schedule
+        self.timing = timing
+        
+        # Setup managers using factory
+        self._setup_components()
+    
+    def _setup_components(self):
+        """Setup all driver components."""
+        # Initialize managers using factory
+        self.manager_registry = ManagerFactory.setup_managers(self)
+        
+        # Update logging utility with manager
+        self._logging_util.logger_manager = self.logger_manager
+        
+        logger.debug(f"UniversalDriver components initialized: {self.driver_id}")
+    
+    # === Lifecycle Management ===
+    
+    async def initialize(self, capabilities: List[str] = []) -> bool:
+        """
+        Initialize driver with all components.
+        
+        Args:
+            capabilities: List of driver capabilities
+            
+        Returns:
+            True if initialization successful
+        """
+        return await DriverInitializer.initialize_driver(self, capabilities)
+    
+    async def shutdown(self):
+        """Shutdown driver cleanly."""
+        await DriverShutdown.shutdown_driver(self)
+    
+    async def run_daemon_mode(self, on_start_message: str = "Driver initialized, waiting for RPC tasks..."):
+        """
+        Run driver in daemon mode with graceful shutdown handling.
+        
+        Args:
+            on_start_message: Message to display when daemon starts
+        """
+        await DaemonManager.run_daemon_mode(self, on_start_message)
+    
+    # === Task Registration ===
+    
+    def register_task_handler(self, task_type: str, handler: Callable):
+        """Register task handler for RPC tasks."""
+        if self.session:
+            self.session.register_task_handler(task_type, handler)
+        else:
+            logger.warning("No session available for task handler registration")
+    
+    # === Health & Monitoring ===
+    
+    async def health_check(self) -> Dict[str, Any]:
+        """Get comprehensive health status."""
+        return await HealthMonitor.get_health_status(self)
+    
+    def get_status(self) -> Dict[str, Any]:
+        """Get basic driver status (synchronous)."""
+        return HealthMonitor.get_basic_status(self)
+    
+    # === Utilities ===
+    
+    def save_results_to_file(self, data: dict, filename: str, results_dir: Optional[str] = None) -> Path:
+        """
+        Save parsing results to JSON file with automatic serialization.
+        
+        Args:
+            data: Data to save (can contain Pydantic models)
+            filename: Base filename (without extension)
+            results_dir: Directory to save to (default: ./data/results)
+            
+        Returns:
+            Path to saved file
+        """
+        return SerializationUtility.save_results_to_file(data, filename, results_dir)
+    
+    # === Logging Methods ===
+    
+    def log(self, level: str, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log message through logger manager."""
+        self._logging_util.log(level, message, context)
+    
+    def debug(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log debug message."""
+        self._logging_util.debug(message, context)
+    
+    def info(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log info message."""
+        self._logging_util.info(message, context)
+    
+    def warning(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log warning message."""
+        self._logging_util.warning(message, context)
+    
+    def error(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log error message."""
+        self._logging_util.error(message, context)
+    
+    def critical(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log critical message."""
+        self._logging_util.critical(message, context)
+    
+    # === Convenience Properties ===
+    
+    @property
+    def is_daemon_mode(self) -> bool:
+        """Check if driver is running in daemon mode."""
+        return self.session is not None
+    
+    @property
+    def is_connected(self) -> bool:
+        """Check if WebSocket is connected."""
+        return self.websocket_client is not None and self.websocket_client.is_connected
+    
+    @property
+    def manager_count(self) -> int:
+        """Get number of active managers."""
+        return len(self.manager_registry.managers) if self.manager_registry else 0
+    
+    def __repr__(self) -> str:
+        """String representation of driver."""
+        status = "initialized" if self.is_initialized else "not initialized"
+        mode = "daemon" if self.is_daemon_mode else "standalone"
+        return f"UniversalDriver(id='{self.driver_id}', status='{status}', mode='{mode}')"

unrealon_driver/driver/factory/__init__.py

@@ -0,0 +1,9 @@
+"""
+Factory pattern for driver components.
+
+Handles setup and configuration of managers and other components.
+"""
+
+from .manager_factory import ManagerFactory
+
+__all__ = ["ManagerFactory"]

unrealon_driver/driver/factory/manager_factory.py

@@ -0,0 +1,130 @@
+"""
+Manager Factory - Setup and configuration of all managers.
+
+Handles the creation and configuration of manager components.
+"""
+
+import logging
+from typing import TYPE_CHECKING
+
+from ...managers import (
+    LoggerManager, HttpManager, BrowserManager, CacheManager,
+    ProxyManager, ThreadManager, UpdateManager, ManagerRegistry
+)
+from ...managers.logger import LoggerManagerConfig
+from ...managers.http import HttpManagerConfig
+from ...managers.browser import BrowserManagerConfig
+from ...managers.cache import CacheManagerConfig
+from ...managers.proxy import ProxyManagerConfig
+from ...managers.threading import ThreadManagerConfig
+from ...managers.update import UpdateManagerConfig
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class ManagerFactory:
+    """Factory for creating and configuring manager components."""
+    
+    @staticmethod
+    def setup_managers(driver: 'UniversalDriver') -> ManagerRegistry:
+        """
+        Setup all managers with configurations.
+        
+        Args:
+            driver: UniversalDriver instance
+            
+        Returns:
+            Configured ManagerRegistry
+        """
+        # Create registry
+        manager_registry = ManagerRegistry()
+        
+        # Setup each manager
+        ManagerFactory._setup_logger_manager(driver, manager_registry)
+        ManagerFactory._setup_http_manager(driver, manager_registry)
+        ManagerFactory._setup_browser_manager(driver, manager_registry)
+        ManagerFactory._setup_cache_manager(driver, manager_registry)
+        ManagerFactory._setup_proxy_manager(driver, manager_registry)
+        ManagerFactory._setup_threading_manager(driver, manager_registry)
+        ManagerFactory._setup_update_manager(driver, manager_registry)
+        
+        logger.debug(f"Managers setup complete for driver: {driver.driver_id}")
+        return manager_registry
+    
+    @staticmethod
+    def _setup_logger_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup logger manager."""
+        logger_config = LoggerManagerConfig(
+            enabled=True,
+            log_file=driver.config.log_file,
+            log_level=driver.config.log_level,
+            driver_id=driver.driver_id,
+            timeout=driver.config.websocket_timeout
+        )
+        driver.logger_manager = LoggerManager(logger_config)
+        registry.register(driver.logger_manager)
+    
+    @staticmethod
+    def _setup_http_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup HTTP manager."""
+        http_config = HttpManagerConfig(
+            enabled=True,
+            timeout=driver.config.http_timeout,
+            max_retries=driver.config.max_retries
+        )
+        driver.http = HttpManager(http_config)
+        registry.register(driver.http)
+    
+    @staticmethod
+    def _setup_browser_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup browser manager."""
+        browser_config = BrowserManagerConfig(
+            enabled=True,
+            headless=driver.config.browser_headless,
+            timeout=driver.config.browser_timeout,
+            parser_name=driver.driver_id
+        )
+        driver.browser = BrowserManager(browser_config)
+        registry.register(driver.browser)
+    
+    @staticmethod
+    def _setup_cache_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup cache manager."""
+        cache_config = CacheManagerConfig(
+            enabled=driver.config.cache_enabled,
+            default_ttl=driver.config.cache_ttl
+        )
+        driver.cache = CacheManager(cache_config)
+        registry.register(driver.cache)
+    
+    @staticmethod
+    def _setup_proxy_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup proxy manager."""
+        proxy_config = ProxyManagerConfig(
+            enabled=driver.config.proxy_enabled,
+            rotation_interval=driver.config.proxy_rotation_interval
+        )
+        driver.proxy = ProxyManager(proxy_config)
+        registry.register(driver.proxy)
+    
+    @staticmethod
+    def _setup_threading_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup threading manager."""
+        threading_config = ThreadManagerConfig(
+            enabled=True,
+            max_workers=driver.config.max_workers
+        )
+        driver.threading = ThreadManager(threading_config)
+        registry.register(driver.threading)
+    
+    @staticmethod
+    def _setup_update_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup update manager."""
+        update_config = UpdateManagerConfig(
+            enabled=True
+        )
+        driver.update = UpdateManager(update_config)
+        registry.register(driver.update)

unrealon_driver/driver/lifecycle/__init__.py

@@ -0,0 +1,11 @@
+"""
+Driver lifecycle management.
+
+Handles initialization, shutdown, and daemon mode operations.
+"""
+
+from .initialization import DriverInitializer
+from .shutdown import DriverShutdown
+from .daemon import DaemonManager
+
+__all__ = ["DriverInitializer", "DriverShutdown", "DaemonManager"]

unrealon_driver/driver/lifecycle/daemon.py

@@ -0,0 +1,76 @@
+"""
+Daemon mode management.
+
+Handles running the driver in daemon mode with graceful shutdown.
+"""
+
+import asyncio
+import logging
+import signal
+import sys
+from typing import TYPE_CHECKING
+
+from .initialization import DriverInitializer
+from .shutdown import DriverShutdown
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class DaemonManager:
+    """Manages daemon mode operations."""
+    
+    @staticmethod
+    async def run_daemon_mode(driver: 'UniversalDriver', on_start_message: str = "Driver initialized, waiting for RPC tasks..."):
+        """
+        Run driver in daemon mode with graceful shutdown handling.
+        
+        Args:
+            driver: UniversalDriver instance
+            on_start_message: Message to display when daemon starts
+        """
+        # Flag for graceful shutdown
+        shutdown_event = asyncio.Event()
+        
+        def signal_handler():
+            """Handle shutdown signals gracefully."""
+            logger.info("Shutdown signal received")
+            shutdown_event.set()
+        
+        # Register signal handlers
+        if sys.platform != "win32":
+            loop = asyncio.get_running_loop()
+            for sig in (signal.SIGTERM, signal.SIGINT):
+                loop.add_signal_handler(sig, signal_handler)
+        
+        try:
+            # Initialize driver with capabilities
+            success = await DriverInitializer.initialize_driver(driver, driver.capabilities)
+            if not success:
+                raise RuntimeError("Driver initialization failed")
+            
+            # Call on_start hook if available
+            if hasattr(driver, 'on_start') and callable(driver.on_start):
+                await driver.on_start()
+            
+            logger.info(on_start_message)
+            
+            # Keep running until shutdown signal
+            await shutdown_event.wait()
+            
+        except Exception as e:
+            logger.error(f"Daemon mode error: {e}")
+            raise
+        finally:
+            # Always shutdown cleanly
+            try:
+                # Call on_shutdown hook if available
+                if hasattr(driver, 'on_shutdown') and callable(driver.on_shutdown):
+                    await driver.on_shutdown()
+                
+                await DriverShutdown.shutdown_driver(driver)
+                logger.info("Driver shutdown complete")
+            except Exception as e:
+                logger.error(f"Error during shutdown: {e}")