unrealon 1.1.5__py3-none-any.whl → 2.0.4__py3-none-any.whl

unrealon_driver/driver/utilities/logging.py

@@ -0,0 +1,51 @@
+"""
+Logging utilities for driver components.
+
+Provides convenient logging methods and utilities.
+"""
+
+import logging
+from typing import Dict, Any, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from unrealon_driver.managers.logger import LoggerManager
+
+logger = logging.getLogger(__name__)
+
+
+class LoggingUtility:
+    """Utility class for driver logging operations."""
+    
+    def __init__(self, driver_id: str):
+        """Initialize logging utility."""
+        self.driver_id = driver_id
+        self.logger_manager: Optional['LoggerManager'] = None
+    
+    def log(self, level: str, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log message through logger manager."""
+        if self.logger_manager:
+            self.logger_manager.log(level, message, context)
+        else:
+            # Fallback to standard logging
+            log_func = getattr(logger, level.lower(), logger.info)
+            log_func(f"{message} | Context: {context}" if context else message)
+    
+    def debug(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log debug message."""
+        self.log("DEBUG", message, context)
+    
+    def info(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log info message."""
+        self.log("INFO", message, context)
+    
+    def warning(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log warning message."""
+        self.log("WARNING", message, context)
+    
+    def error(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log error message."""
+        self.log("ERROR", message, context)
+    
+    def critical(self, message: str, context: Optional[Dict[str, Any]] = None):
+        """Log critical message."""
+        self.log("CRITICAL", message, context)

unrealon_driver/driver/utilities/serialization.py

@@ -0,0 +1,61 @@
+"""
+Serialization utilities for driver components.
+
+Handles result saving and JSON serialization with Pydantic support.
+"""
+
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SerializationUtility:
+    """Utility class for serialization and result saving."""
+    
+    @staticmethod
+    def save_results_to_file(data: dict, filename: str, results_dir: Optional[str] = None) -> Path:
+        """
+        Save parsing results to JSON file with automatic serialization.
+        
+        Args:
+            data: Data to save (can contain Pydantic models)
+            filename: Base filename (without extension)
+            results_dir: Directory to save to (default: ./data/results)
+            
+        Returns:
+            Path to saved file
+        """
+        if results_dir is None:
+            results_dir = "./data/results"
+        
+        results_path = Path(results_dir)
+        results_path.mkdir(parents=True, exist_ok=True)
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filepath = results_path / f"{filename}_{timestamp}.json"
+        
+        # Convert Pydantic models to dict for JSON serialization
+        serializable_data = SerializationUtility._serialize_for_json(data)
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(serializable_data, f, indent=2, ensure_ascii=False, default=str)
+        
+        logger.info(f"Results saved to: {filepath}")
+        return filepath
+    
+    @staticmethod
+    def _serialize_for_json(data: Any) -> Any:
+        """Recursively serialize data for JSON, handling Pydantic models."""
+        if hasattr(data, "model_dump"):
+            # Pydantic v2 model
+            return data.model_dump()
+        elif isinstance(data, dict):
+            return {key: SerializationUtility._serialize_for_json(value) for key, value in data.items()}
+        elif isinstance(data, list):
+            return [SerializationUtility._serialize_for_json(item) for item in data]
+        else:
+            return data

unrealon_driver/managers/__init__.py

@@ -0,0 +1,32 @@
+"""
+Clean manager system for UnrealOn Driver.
+"""
+
+from .base import BaseManager, ManagerConfig, ManagerStatus
+from .logger import LoggerManager, LoggerManagerConfig
+from .http import HttpManager, HttpManagerConfig
+from .browser import BrowserManager, BrowserManagerConfig
+from .cache import CacheManager, CacheManagerConfig
+from .proxy import ProxyManager, ProxyManagerConfig
+from .threading import ThreadManager, ThreadManagerConfig
+from .update import UpdateManager, UpdateManagerConfig
+from .registry import ManagerRegistry
+
+__all__ = [
+    # Base
+    "BaseManager",
+    "ManagerConfig", 
+    "ManagerStatus",
+    
+    # Managers
+    "LoggerManager", "LoggerManagerConfig",
+    "HttpManager", "HttpManagerConfig",
+    "BrowserManager", "BrowserManagerConfig",
+    "CacheManager", "CacheManagerConfig",
+    "ProxyManager", "ProxyManagerConfig",
+    "ThreadManager", "ThreadManagerConfig",
+    "UpdateManager", "UpdateManagerConfig",
+    
+    # Registry
+    "ManagerRegistry",
+]

unrealon_driver/managers/base.py

@@ -0,0 +1,174 @@
+"""
+Clean base manager system.
+"""
+
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Optional, Dict, Any
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+from ..utils.time import utc_now
+
+logger = logging.getLogger(__name__)
+
+
+class ManagerStatus(str, Enum):
+    """Manager lifecycle status."""
+    UNINITIALIZED = "uninitialized"
+    INITIALIZING = "initializing"
+    READY = "ready"
+    ERROR = "error"
+    SHUTTING_DOWN = "shutting_down"
+    SHUTDOWN = "shutdown"
+
+
+class ManagerConfig(BaseModel):
+    """Base configuration for all managers."""
+    enabled: bool = Field(default=True, description="Whether manager is enabled")
+    timeout: int = Field(default=30, description="Operation timeout seconds")
+    max_retries: int = Field(default=3, description="Max retry attempts")
+    log_level: str = Field(default="INFO", description="Logging level")
+    
+    model_config = {"extra": "forbid"}
+
+
+class ManagerStats(BaseModel):
+    """Manager operation statistics."""
+    operations_total: int = 0
+    operations_successful: int = 0
+    operations_failed: int = 0
+    last_operation: Optional[datetime] = None
+    average_duration: float = 0.0
+    
+    def record_operation(self, success: bool, duration: float):
+        """Record operation result."""
+        self.operations_total += 1
+        if success:
+            self.operations_successful += 1
+        else:
+            self.operations_failed += 1
+        
+        self.last_operation = utc_now()
+        
+        # Simple moving average
+        if self.operations_total == 1:
+            self.average_duration = duration
+        else:
+            self.average_duration = (
+                (self.average_duration * (self.operations_total - 1) + duration) 
+                / self.operations_total
+            )
+    
+    def get_success_rate(self) -> float:
+        """Get success rate percentage."""
+        if self.operations_total == 0:
+            return 0.0
+        return (self.operations_successful / self.operations_total) * 100.0
+
+
+class BaseManager(ABC):
+    """
+    Clean base manager class.
+    
+    Provides common functionality for all managers:
+    - Lifecycle management
+    - Statistics tracking
+    - Error handling
+    - Health checks
+    """
+    
+    def __init__(self, config: ManagerConfig, name: str):
+        self.config = config
+        self.name = name
+        self.status = ManagerStatus.UNINITIALIZED
+        self.stats = ManagerStats()
+        self.logger = logging.getLogger(f"{__name__}.{name}")
+        
+        # Set log level
+        if hasattr(logging, config.log_level):
+            self.logger.setLevel(getattr(logging, config.log_level))
+    
+    async def initialize(self) -> bool:
+        """Initialize manager."""
+        if not self.config.enabled:
+            self.logger.info(f"Manager {self.name} is disabled")
+            self.status = ManagerStatus.SHUTDOWN
+            return True
+        
+        try:
+            self.status = ManagerStatus.INITIALIZING
+            self.logger.info(f"Initializing manager: {self.name}")
+            
+            success = await self._initialize()
+            
+            if success:
+                self.status = ManagerStatus.READY
+                self.logger.info(f"Manager {self.name} initialized successfully")
+            else:
+                self.status = ManagerStatus.ERROR
+                self.logger.error(f"Manager {self.name} initialization failed")
+            
+            return success
+            
+        except Exception as e:
+            self.status = ManagerStatus.ERROR
+            self.logger.error(f"Manager {self.name} initialization error: {e}")
+            return False
+    
+    async def shutdown(self):
+        """Shutdown manager."""
+        try:
+            self.status = ManagerStatus.SHUTTING_DOWN
+            self.logger.info(f"Shutting down manager: {self.name}")
+            
+            await self._shutdown()
+            
+            self.status = ManagerStatus.SHUTDOWN
+            self.logger.info(f"Manager {self.name} shutdown complete")
+            
+        except Exception as e:
+            self.logger.error(f"Manager {self.name} shutdown error: {e}")
+            self.status = ManagerStatus.ERROR
+    
+    @abstractmethod
+    async def _initialize(self) -> bool:
+        """Manager-specific initialization."""
+        pass
+    
+    @abstractmethod
+    async def _shutdown(self):
+        """Manager-specific shutdown."""
+        pass
+    
+    async def health_check(self) -> Dict[str, Any]:
+        """Perform health check."""
+        try:
+            health_data = await self._health_check()
+            
+            return {
+                "name": self.name,
+                "status": self.status.value,
+                "enabled": self.config.enabled,
+                "stats": self.stats.model_dump(),
+                "health": health_data
+            }
+            
+        except Exception as e:
+            self.logger.error(f"Health check failed for {self.name}: {e}")
+            return {
+                "name": self.name,
+                "status": "error",
+                "error": str(e)
+            }
+    
+    async def _health_check(self) -> Dict[str, Any]:
+        """Manager-specific health check."""
+        return {"status": "ok"}
+    
+    def is_ready(self) -> bool:
+        """Check if manager is ready."""
+        return self.status == ManagerStatus.READY

unrealon_driver/managers/browser.py

@@ -0,0 +1,98 @@
+"""
+Clean browser manager using unrealon_browser.
+"""
+
+from typing import Optional
+from pydantic import Field
+
+from unrealon_browser import BrowserManager as CoreBrowserManager, BrowserConfig
+from .base import BaseManager, ManagerConfig
+
+
+class BrowserManagerConfig(ManagerConfig):
+    """Browser manager configuration."""
+
+    headless: bool = Field(default=True, description="Run headless")
+    parser_name: str = Field(..., description="Parser name for browser")
+
+
+class BrowserManager(BaseManager):
+    """Clean browser manager wrapper."""
+
+    def __init__(self, config: BrowserManagerConfig):
+        super().__init__(config, "browser")
+        self.config: BrowserManagerConfig = config
+        self.browser: Optional[CoreBrowserManager] = None
+
+    async def _initialize(self) -> bool:
+        """Initialize browser manager (but not the actual browser yet)."""
+        try:
+            # Don't initialize the actual browser here - do it lazily on first use
+            # This prevents browser from starting in daemon mode until needed
+            self.logger.info("Browser manager ready (browser will start on first use)")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Browser manager initialization failed: {e}")
+            return False
+
+    async def _shutdown(self):
+        """Shutdown browser."""
+        if self.browser:
+            await self.browser.close_async()
+            self.browser = None
+
+    async def _ensure_browser_initialized(self) -> bool:
+        """Ensure browser is initialized (lazy initialization)."""
+        if self.browser is not None:
+            return True
+            
+        try:
+            self.logger.info("🚀 Starting browser (lazy initialization)...")
+            
+            # Create browser config
+            browser_config = BrowserConfig(parser_name=self.config.parser_name)
+
+            # Create browser manager
+            self.browser = CoreBrowserManager(browser_config)
+
+            # Initialize browser
+            await self.browser.initialize_async()
+            
+            self.logger.info("✅ Browser started successfully")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"❌ Browser initialization failed: {e}")
+            self.browser = None
+            return False
+
+    async def navigate(self, url: str) -> bool:
+        """Navigate to URL (with lazy browser initialization)."""
+        # Ensure browser is initialized
+        if not await self._ensure_browser_initialized():
+            raise RuntimeError("Failed to initialize browser")
+
+        try:
+            await self.browser.navigate_async(url)
+            self.stats.record_operation(True, 0.0)
+            return True
+        except Exception as e:
+            self.logger.error(f"Navigation failed: {e}")
+            self.stats.record_operation(False, 0.0)
+            return False
+
+    async def get_html(self) -> Optional[str]:
+        """Get page HTML (with lazy browser initialization)."""
+        # Ensure browser is initialized
+        if not await self._ensure_browser_initialized():
+            raise RuntimeError("Failed to initialize browser")
+
+        try:
+            html = await self.browser.get_page_content_async()
+            self.stats.record_operation(True, 0.0)
+            return html
+        except Exception as e:
+            self.logger.error(f"Get HTML failed: {e}")
+            self.stats.record_operation(False, 0.0)
+            return None

unrealon_driver/managers/cache.py

@@ -0,0 +1,116 @@
+"""
+Clean cache manager.
+"""
+
+import asyncio
+from typing import Any, Optional, Dict
+from datetime import datetime, timedelta
+from pydantic import Field
+
+from ..utils.time import utc_now
+from .base import BaseManager, ManagerConfig
+
+
+class CacheManagerConfig(ManagerConfig):
+    """Cache manager configuration."""
+    default_ttl: int = Field(default=3600, description="Default TTL seconds")
+    max_size: int = Field(default=1000, description="Max cache entries")
+
+
+class CacheEntry:
+    """Cache entry with TTL."""
+    
+    def __init__(self, value: Any, ttl: int):
+        self.value = value
+        self.expires_at = utc_now() + timedelta(seconds=ttl)
+    
+    def is_expired(self) -> bool:
+        """Check if entry is expired."""
+        return utc_now() > self.expires_at
+
+
+class CacheManager(BaseManager):
+    """Simple in-memory cache manager."""
+    
+    def __init__(self, config: CacheManagerConfig):
+        super().__init__(config, "cache")
+        self.config: CacheManagerConfig = config
+        self._cache: Dict[str, CacheEntry] = {}
+        self._cleanup_task: Optional[asyncio.Task] = None
+    
+    async def _initialize(self) -> bool:
+        """Initialize cache."""
+        # Start cleanup task
+        self._cleanup_task = asyncio.create_task(self._cleanup_expired())
+        return True
+    
+    async def _shutdown(self):
+        """Shutdown cache."""
+        if self._cleanup_task:
+            self._cleanup_task.cancel()
+            try:
+                await self._cleanup_task
+            except asyncio.CancelledError:
+                pass
+        
+        self._cache.clear()
+    
+    async def _cleanup_expired(self):
+        """Background task to clean expired entries."""
+        while True:
+            try:
+                await asyncio.sleep(60)  # Cleanup every minute
+                
+                expired_keys = []
+                for key, entry in self._cache.items():
+                    if entry.is_expired():
+                        expired_keys.append(key)
+                
+                for key in expired_keys:
+                    del self._cache[key]
+                
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Cache cleanup error: {e}")
+    
+    def get(self, key: str) -> Optional[Any]:
+        """Get value from cache."""
+        entry = self._cache.get(key)
+        if not entry:
+            return None
+        
+        if entry.is_expired():
+            del self._cache[key]
+            return None
+        
+        return entry.value
+    
+    def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
+        """Set value in cache."""
+        if len(self._cache) >= self.config.max_size:
+            # Remove oldest entry
+            oldest_key = next(iter(self._cache))
+            del self._cache[oldest_key]
+        
+        ttl = ttl or self.config.default_ttl
+        self._cache[key] = CacheEntry(value, ttl)
+    
+    def delete(self, key: str) -> bool:
+        """Delete key from cache."""
+        if key in self._cache:
+            del self._cache[key]
+            return True
+        return False
+    
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        self._cache.clear()
+    
+    async def _health_check(self) -> Dict[str, Any]:
+        """Cache health check."""
+        return {
+            "status": "ok",
+            "entries": len(self._cache),
+            "max_size": self.config.max_size
+        }

unrealon_driver/managers/http.py

@@ -0,0 +1,107 @@
+"""
+Clean HTTP manager for requests.
+"""
+
+import asyncio
+import aiohttp
+from typing import Dict, Any, Optional
+from pydantic import Field
+
+from .base import BaseManager, ManagerConfig
+
+
+class HttpManagerConfig(ManagerConfig):
+    """HTTP manager configuration."""
+    user_agent: str = Field(default="UnrealOn-Driver/1.0", description="User agent string")
+    max_connections: int = Field(default=100, description="Max concurrent connections")
+    connector_limit: int = Field(default=30, description="Connector limit per host")
+
+
+class HttpManager(BaseManager):
+    """Clean HTTP manager with aiohttp."""
+    
+    def __init__(self, config: HttpManagerConfig):
+        super().__init__(config, "http")
+        self.config: HttpManagerConfig = config
+        self.session: Optional[aiohttp.ClientSession] = None
+    
+    async def _initialize(self) -> bool:
+        """Initialize HTTP session."""
+        try:
+            # Create connector
+            connector = aiohttp.TCPConnector(
+                limit=self.config.max_connections,
+                limit_per_host=self.config.connector_limit,
+                ttl_dns_cache=300,
+                use_dns_cache=True
+            )
+            
+            # Create session
+            timeout = aiohttp.ClientTimeout(total=self.config.timeout)
+            headers = {"User-Agent": self.config.user_agent}
+            
+            self.session = aiohttp.ClientSession(
+                connector=connector,
+                timeout=timeout,
+                headers=headers
+            )
+            
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"HTTP manager initialization failed: {e}")
+            return False
+    
+    async def _shutdown(self):
+        """Shutdown HTTP session."""
+        if self.session:
+            await self.session.close()
+            self.session = None
+    
+    async def get(self, url: str, **kwargs) -> aiohttp.ClientResponse:
+        """Make GET request."""
+        if not self.session:
+            raise RuntimeError("HTTP manager not initialized")
+        
+        start_time = asyncio.get_event_loop().time()
+        success = False
+        
+        try:
+            response = await self.session.get(url, **kwargs)
+            success = True
+            return response
+        finally:
+            duration = asyncio.get_event_loop().time() - start_time
+            self.stats.record_operation(success, duration)
+    
+    async def post(self, url: str, **kwargs) -> aiohttp.ClientResponse:
+        """Make POST request."""
+        if not self.session:
+            raise RuntimeError("HTTP manager not initialized")
+        
+        start_time = asyncio.get_event_loop().time()
+        success = False
+        
+        try:
+            response = await self.session.post(url, **kwargs)
+            success = True
+            return response
+        finally:
+            duration = asyncio.get_event_loop().time() - start_time
+            self.stats.record_operation(success, duration)
+    
+    async def request(self, method: str, url: str, **kwargs) -> aiohttp.ClientResponse:
+        """Make generic request."""
+        if not self.session:
+            raise RuntimeError("HTTP manager not initialized")
+        
+        start_time = asyncio.get_event_loop().time()
+        success = False
+        
+        try:
+            response = await self.session.request(method, url, **kwargs)
+            success = True
+            return response
+        finally:
+            duration = asyncio.get_event_loop().time() - start_time
+            self.stats.record_operation(success, duration)