unrealon 1.1.6__py3-none-any.whl → 2.0.4__py3-none-any.whl

unrealon_driver/driver/core/driver.py

@@ -0,0 +1,221 @@
+"""
+UniversalDriver - Clean, modular driver orchestrator.
+
+The main driver class that coordinates all components through a clean,
+organized architecture with separated concerns.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional, List, Dict, Any, Callable
+
+from .config import DriverConfig
+from ..communication.websocket_client import WebSocketClient
+from ..communication.session import DriverSession
+from ..factory.manager_factory import ManagerFactory
+from ..lifecycle.initialization import DriverInitializer
+from ..lifecycle.shutdown import DriverShutdown
+from ..lifecycle.daemon import DaemonManager
+from ..monitoring.health import HealthMonitor
+from ..utilities.logging import LoggingUtility
+from ..utilities.serialization import SerializationUtility
+
+from ...managers import (
+    LoggerManager, HttpManager, BrowserManager, CacheManager,
+    ProxyManager, ThreadManager, UpdateManager, ManagerRegistry
+)
+
+from ...core_module import (
+    EventManager,
+    ModuleRegistry
+)
+
+from ...decorators import task, retry, schedule, timing
+
+logger = logging.getLogger(__name__)
+
+
+class UniversalDriver:
+    """
+    Universal Driver - Clean orchestrator for all parsing operations.
+    
+    Key Features:
+    - Zero-config initialization with sensible defaults
+    - Modular manager system (HTTP, Browser, Cache, etc.)
+    - WebSocket communication for RPC and logging
+    - Graceful shutdown with signal handling
+    - Built-in utilities for common operations
+    - Decorator-based task registration
+    
+    Usage:
+        # Basic usage
+        config = DriverConfig.for_development("my_parser")
+        driver = UniversalDriver(config)
+        await driver.initialize()
+        
+        # Daemon mode with RPC
+        await driver.run_daemon_mode()
+        
+        # Standalone mode
+        await driver.initialize()
+        # ... your parsing logic
+        await driver.shutdown()
+    """
+    
+    def __init__(self, config: DriverConfig):
+        """Initialize UniversalDriver with configuration."""
+        self.config = config
+        self.driver_id = config.name
+        self.is_initialized = False
+        self.capabilities: List[str] = []
+        
+        # Core components
+        self.websocket_client: Optional[WebSocketClient] = None
+        self.session: Optional[DriverSession] = None
+        
+        # Manager system (initialized by factory)
+        self.manager_registry: Optional[ManagerRegistry] = None
+        self.logger_manager: Optional[LoggerManager] = None
+        self.http: Optional[HttpManager] = None
+        self.browser: Optional[BrowserManager] = None
+        self.cache: Optional[CacheManager] = None
+        self.proxy: Optional[ProxyManager] = None
+        self.threading: Optional[ThreadManager] = None
+        self.update: Optional[UpdateManager] = None
+        
+        # Module system
+        self.event_manager = EventManager()
+        self.module_registry = ModuleRegistry(self.event_manager)
+        
+        # Utilities
+        self._logging_util = LoggingUtility(self.driver_id)
+        
+        # Decorators (exposed as instance methods for convenience)
+        self.task = task
+        self.retry = retry
+        self.schedule = schedule
+        self.timing = timing
+        
+        # Setup managers using factory
+        self._setup_components()
+    
+    def _setup_components(self):
+        """Setup all driver components."""
+        # Initialize managers using factory
+        self.manager_registry = ManagerFactory.setup_managers(self)
+        
+        # Update logging utility with manager
+        self._logging_util.logger_manager = self.logger_manager
+        
+        logger.debug(f"UniversalDriver components initialized: {self.driver_id}")
+    
+    # === Lifecycle Management ===
+    
+    async def initialize(self, capabilities: List[str] = []) -> bool:
+        """
+        Initialize driver with all components.
+        
+        Args:
+            capabilities: List of driver capabilities
+            
+        Returns:
+            True if initialization successful
+        """
+        return await DriverInitializer.initialize_driver(self, capabilities)
+    
+    async def shutdown(self):
+        """Shutdown driver cleanly."""
+        await DriverShutdown.shutdown_driver(self)
+    
+    async def run_daemon_mode(self, on_start_message: str = "Driver initialized, waiting for RPC tasks..."):
+        """
+        Run driver in daemon mode with graceful shutdown handling.
+        
+        Args:
+            on_start_message: Message to display when daemon starts
+        """
+        await DaemonManager.run_daemon_mode(self, on_start_message)
+    
+    # === Task Registration ===
+    
+    def register_task_handler(self, task_type: str, handler: Callable):
+        """Register task handler for RPC tasks."""
+        if self.session:
+            self.session.register_task_handler(task_type, handler)
+        else:
+            logger.warning("No session available for task handler registration")
+    
+    # === Health & Monitoring ===
+    
+    async def health_check(self) -> Dict[str, Any]:
+        """Get comprehensive health status."""
+        return await HealthMonitor.get_health_status(self)
+    
+    def get_status(self) -> Dict[str, Any]:
+        """Get basic driver status (synchronous)."""
+        return HealthMonitor.get_basic_status(self)
+    
+    # === Utilities ===
+    
+    def save_results_to_file(self, data: dict, filename: str, results_dir: Optional[str] = None) -> Path:
+        """
+        Save parsing results to JSON file with automatic serialization.
+        
+        Args:
+            data: Data to save (can contain Pydantic models)
+            filename: Base filename (without extension)
+            results_dir: Directory to save to (default: ./data/results)
+            
+        Returns:
+            Path to saved file
+        """
+        return SerializationUtility.save_results_to_file(data, filename, results_dir)
+    
+    # === Logging Methods ===
+    
+    def log(self, level: str, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log message through logger manager."""
+        self._logging_util.log(level, message, context)
+    
+    def debug(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log debug message."""
+        self._logging_util.debug(message, context)
+    
+    def info(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log info message."""
+        self._logging_util.info(message, context)
+    
+    def warning(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log warning message."""
+        self._logging_util.warning(message, context)
+    
+    def error(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log error message."""
+        self._logging_util.error(message, context)
+    
+    def critical(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log critical message."""
+        self._logging_util.critical(message, context)
+    
+    # === Convenience Properties ===
+    
+    @property
+    def is_daemon_mode(self) -> bool:
+        """Check if driver is running in daemon mode."""
+        return self.session is not None
+    
+    @property
+    def is_connected(self) -> bool:
+        """Check if WebSocket is connected."""
+        return self.websocket_client is not None and self.websocket_client.is_connected
+    
+    @property
+    def manager_count(self) -> int:
+        """Get number of active managers."""
+        return len(self.manager_registry.managers) if self.manager_registry else 0
+    
+    def __repr__(self) -> str:
+        """String representation of driver."""
+        status = "initialized" if self.is_initialized else "not initialized"
+        mode = "daemon" if self.is_daemon_mode else "standalone"
+        return f"UniversalDriver(id='{self.driver_id}', status='{status}', mode='{mode}')"

unrealon_driver/driver/factory/__init__.py

@@ -0,0 +1,9 @@
+"""
+Factory pattern for driver components.
+
+Handles setup and configuration of managers and other components.
+"""
+
+from .manager_factory import ManagerFactory
+
+__all__ = ["ManagerFactory"]

unrealon_driver/driver/factory/manager_factory.py

@@ -0,0 +1,130 @@
+"""
+Manager Factory - Setup and configuration of all managers.
+
+Handles the creation and configuration of manager components.
+"""
+
+import logging
+from typing import TYPE_CHECKING
+
+from ...managers import (
+    LoggerManager, HttpManager, BrowserManager, CacheManager,
+    ProxyManager, ThreadManager, UpdateManager, ManagerRegistry
+)
+from ...managers.logger import LoggerManagerConfig
+from ...managers.http import HttpManagerConfig
+from ...managers.browser import BrowserManagerConfig
+from ...managers.cache import CacheManagerConfig
+from ...managers.proxy import ProxyManagerConfig
+from ...managers.threading import ThreadManagerConfig
+from ...managers.update import UpdateManagerConfig
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class ManagerFactory:
+    """Factory for creating and configuring manager components."""
+    
+    @staticmethod
+    def setup_managers(driver: 'UniversalDriver') -> ManagerRegistry:
+        """
+        Setup all managers with configurations.
+        
+        Args:
+            driver: UniversalDriver instance
+            
+        Returns:
+            Configured ManagerRegistry
+        """
+        # Create registry
+        manager_registry = ManagerRegistry()
+        
+        # Setup each manager
+        ManagerFactory._setup_logger_manager(driver, manager_registry)
+        ManagerFactory._setup_http_manager(driver, manager_registry)
+        ManagerFactory._setup_browser_manager(driver, manager_registry)
+        ManagerFactory._setup_cache_manager(driver, manager_registry)
+        ManagerFactory._setup_proxy_manager(driver, manager_registry)
+        ManagerFactory._setup_threading_manager(driver, manager_registry)
+        ManagerFactory._setup_update_manager(driver, manager_registry)
+        
+        logger.debug(f"Managers setup complete for driver: {driver.driver_id}")
+        return manager_registry
+    
+    @staticmethod
+    def _setup_logger_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup logger manager."""
+        logger_config = LoggerManagerConfig(
+            enabled=True,
+            log_file=driver.config.log_file,
+            log_level=driver.config.log_level,
+            driver_id=driver.driver_id,
+            timeout=driver.config.websocket_timeout
+        )
+        driver.logger_manager = LoggerManager(logger_config)
+        registry.register(driver.logger_manager)
+    
+    @staticmethod
+    def _setup_http_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup HTTP manager."""
+        http_config = HttpManagerConfig(
+            enabled=True,
+            timeout=driver.config.http_timeout,
+            max_retries=driver.config.max_retries
+        )
+        driver.http = HttpManager(http_config)
+        registry.register(driver.http)
+    
+    @staticmethod
+    def _setup_browser_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup browser manager."""
+        browser_config = BrowserManagerConfig(
+            enabled=True,
+            headless=driver.config.browser_headless,
+            timeout=driver.config.browser_timeout,
+            parser_name=driver.driver_id
+        )
+        driver.browser = BrowserManager(browser_config)
+        registry.register(driver.browser)
+    
+    @staticmethod
+    def _setup_cache_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup cache manager."""
+        cache_config = CacheManagerConfig(
+            enabled=driver.config.cache_enabled,
+            default_ttl=driver.config.cache_ttl
+        )
+        driver.cache = CacheManager(cache_config)
+        registry.register(driver.cache)
+    
+    @staticmethod
+    def _setup_proxy_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup proxy manager."""
+        proxy_config = ProxyManagerConfig(
+            enabled=driver.config.proxy_enabled,
+            rotation_interval=driver.config.proxy_rotation_interval
+        )
+        driver.proxy = ProxyManager(proxy_config)
+        registry.register(driver.proxy)
+    
+    @staticmethod
+    def _setup_threading_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup threading manager."""
+        threading_config = ThreadManagerConfig(
+            enabled=True,
+            max_workers=driver.config.max_workers
+        )
+        driver.threading = ThreadManager(threading_config)
+        registry.register(driver.threading)
+    
+    @staticmethod
+    def _setup_update_manager(driver: 'UniversalDriver', registry: ManagerRegistry):
+        """Setup update manager."""
+        update_config = UpdateManagerConfig(
+            enabled=True
+        )
+        driver.update = UpdateManager(update_config)
+        registry.register(driver.update)

unrealon_driver/driver/lifecycle/__init__.py

@@ -0,0 +1,11 @@
+"""
+Driver lifecycle management.
+
+Handles initialization, shutdown, and daemon mode operations.
+"""
+
+from .initialization import DriverInitializer
+from .shutdown import DriverShutdown
+from .daemon import DaemonManager
+
+__all__ = ["DriverInitializer", "DriverShutdown", "DaemonManager"]

unrealon_driver/driver/lifecycle/daemon.py

@@ -0,0 +1,76 @@
+"""
+Daemon mode management.
+
+Handles running the driver in daemon mode with graceful shutdown.
+"""
+
+import asyncio
+import logging
+import signal
+import sys
+from typing import TYPE_CHECKING
+
+from .initialization import DriverInitializer
+from .shutdown import DriverShutdown
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class DaemonManager:
+    """Manages daemon mode operations."""
+    
+    @staticmethod
+    async def run_daemon_mode(driver: 'UniversalDriver', on_start_message: str = "Driver initialized, waiting for RPC tasks..."):
+        """
+        Run driver in daemon mode with graceful shutdown handling.
+        
+        Args:
+            driver: UniversalDriver instance
+            on_start_message: Message to display when daemon starts
+        """
+        # Flag for graceful shutdown
+        shutdown_event = asyncio.Event()
+        
+        def signal_handler():
+            """Handle shutdown signals gracefully."""
+            logger.info("Shutdown signal received")
+            shutdown_event.set()
+        
+        # Register signal handlers
+        if sys.platform != "win32":
+            loop = asyncio.get_running_loop()
+            for sig in (signal.SIGTERM, signal.SIGINT):
+                loop.add_signal_handler(sig, signal_handler)
+        
+        try:
+            # Initialize driver with capabilities
+            success = await DriverInitializer.initialize_driver(driver, driver.capabilities)
+            if not success:
+                raise RuntimeError("Driver initialization failed")
+            
+            # Call on_start hook if available
+            if hasattr(driver, 'on_start') and callable(driver.on_start):
+                await driver.on_start()
+            
+            logger.info(on_start_message)
+            
+            # Keep running until shutdown signal
+            await shutdown_event.wait()
+            
+        except Exception as e:
+            logger.error(f"Daemon mode error: {e}")
+            raise
+        finally:
+            # Always shutdown cleanly
+            try:
+                # Call on_shutdown hook if available
+                if hasattr(driver, 'on_shutdown') and callable(driver.on_shutdown):
+                    await driver.on_shutdown()
+                
+                await DriverShutdown.shutdown_driver(driver)
+                logger.info("Driver shutdown complete")
+            except Exception as e:
+                logger.error(f"Error during shutdown: {e}")

unrealon_driver/driver/lifecycle/initialization.py

@@ -0,0 +1,97 @@
+"""
+Driver initialization logic.
+
+Handles the setup and initialization of all driver components.
+"""
+
+import logging
+from typing import List, TYPE_CHECKING
+
+from ..core.config import DriverMode
+from ..communication.websocket_client import WebSocketClient
+from ..communication.session import DriverSession
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class DriverInitializer:
+    """Handles driver initialization process."""
+    
+    @staticmethod
+    async def initialize_driver(driver: 'UniversalDriver', capabilities: List[str] = []) -> bool:
+        """
+        Initialize driver with all components.
+        
+        Args:
+            driver: UniversalDriver instance
+            capabilities: List of driver capabilities
+            
+        Returns:
+            True if initialization successful
+        """
+        # Store capabilities
+        driver.capabilities = capabilities
+        
+        try:
+            logger.info(f"Initializing UniversalDriver: {driver.driver_id}")
+            
+            # Initialize manager system
+            if not await driver.manager_registry.initialize_all():
+                logger.error("Manager initialization failed")
+                return False
+            
+            # Initialize module system
+            await driver.module_registry.initialize_all()
+            
+            # Setup WebSocket and session
+            await DriverInitializer._setup_communication(driver, capabilities)
+            
+            logger.info(f"UniversalDriver {driver.driver_id} initialized successfully")
+            driver.is_initialized = True
+            return True
+            
+        except Exception as e:
+            logger.error(f"Driver initialization failed: {e}")
+            return False
+    
+    @staticmethod
+    async def _setup_communication(driver: 'UniversalDriver', capabilities: List[str]):
+        """Setup WebSocket client and session based on driver mode."""
+        # Get WebSocket URL from config (auto-detected)
+        websocket_url = driver.config.effective_websocket_url
+        
+        # Setup WebSocket client if URL available
+        if websocket_url:
+            driver.websocket_client = WebSocketClient(
+                websocket_url,
+                driver.driver_id
+            )
+            
+            # Set WebSocket client for logger (always for logging)
+            if driver.logger_manager:
+                driver.logger_manager.websocket_client = driver.websocket_client
+            
+            # Only create RPC session in DAEMON mode
+            if driver.config.mode == DriverMode.DAEMON:
+                # Create session
+                driver.session = DriverSession(
+                    driver.driver_id,
+                    driver.websocket_client
+                )
+                
+                # Start session with capabilities
+                if capabilities is not None:
+                    if not await driver.session.start_session(capabilities):
+                        logger.error("Session start failed")
+                        raise RuntimeError("Session start failed")
+            else:
+                # In STANDALONE mode - try to connect WebSocket for logging (optional)
+                try:
+                    await driver.websocket_client.connect()
+                    logger.info("WebSocket connected for logging only (standalone mode)")
+                except Exception as e:
+                    logger.warning(f"WebSocket connection failed in standalone mode (will use local logging): {e}")
+                    # Continue without WebSocket - not critical in standalone mode

unrealon_driver/driver/lifecycle/shutdown.py

@@ -0,0 +1,48 @@
+"""
+Driver shutdown logic.
+
+Handles clean shutdown of all driver components.
+"""
+
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class DriverShutdown:
+    """Handles driver shutdown process."""
+    
+    @staticmethod
+    async def shutdown_driver(driver: 'UniversalDriver'):
+        """Shutdown driver cleanly."""
+        try:
+            logger.info(f"Shutting down UniversalDriver: {driver.driver_id}")
+            
+            # Stop session
+            if driver.session:
+                await driver.session.stop_session()
+            
+            # Disconnect WebSocket client
+            if driver.websocket_client:
+                await driver.websocket_client.disconnect()
+            
+            # Stop event manager
+            await driver.event_manager.stop()
+            
+            # Stop module system
+            await driver.module_registry.stop_all()
+            
+            # Shutdown managers
+            await driver.manager_registry.shutdown_all()
+            
+            logger.info(f"UniversalDriver {driver.driver_id} shutdown complete")
+            driver.is_initialized = False
+            
+        except Exception as e:
+            logger.error(f"Driver shutdown error: {e}")
+            # Still mark as not initialized even if shutdown failed
+            driver.is_initialized = False

unrealon_driver/driver/monitoring/__init__.py

@@ -0,0 +1,9 @@
+"""
+Driver monitoring and health checks.
+
+Provides health status and monitoring capabilities.
+"""
+
+from .health import HealthMonitor
+
+__all__ = ["HealthMonitor"]

unrealon_driver/driver/monitoring/health.py

@@ -0,0 +1,63 @@
+"""
+Health monitoring for driver components.
+
+Provides health checks and status monitoring for the driver.
+"""
+
+import logging
+from typing import Dict, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..core.driver import UniversalDriver
+
+logger = logging.getLogger(__name__)
+
+
+class HealthMonitor:
+    """Monitors driver health and provides status information."""
+    
+    @staticmethod
+    async def get_health_status(driver: 'UniversalDriver') -> Dict[str, Any]:
+        """Get comprehensive health status."""
+        try:
+            # Get manager health
+            manager_health = await driver.manager_registry.health_check_all()
+            
+            # Get module health
+            module_health = await driver.module_registry.health_check_all()
+            
+            # Get session status
+            session_status = None
+            if driver.session:
+                session_status = driver.session.get_status()
+            
+            return {
+                "driver_id": driver.driver_id,
+                "config": {
+                    "mode": driver.config.mode.value,
+                    "websocket_enabled": driver.config.websocket_url is not None
+                },
+                "managers": manager_health,
+                "modules": module_health,
+                "session": session_status
+            }
+            
+        except Exception as e:
+            logger.error(f"Health check failed: {e}")
+            return {
+                "driver_id": driver.driver_id,
+                "status": "error",
+                "error": str(e)
+            }
+    
+    @staticmethod
+    def get_basic_status(driver: 'UniversalDriver') -> Dict[str, Any]:
+        """Get basic driver status (synchronous)."""
+        return {
+            "driver_id": driver.driver_id,
+            "is_initialized": driver.is_initialized,
+            "is_daemon_mode": driver.session is not None,
+            "is_connected": driver.websocket_client is not None and driver.websocket_client.is_connected,
+            "manager_count": len(driver.manager_registry.managers) if driver.manager_registry else 0,
+            "capabilities": driver.capabilities
+        }

unrealon_driver/driver/utilities/__init__.py

@@ -0,0 +1,10 @@
+"""
+Driver utilities.
+
+Common utilities for logging, serialization, and result handling.
+"""
+
+from .logging import LoggingUtility
+from .serialization import SerializationUtility
+
+__all__ = ["LoggingUtility", "SerializationUtility"]