unrealon 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

unrealon_driver/parser/daemon_manager.py

@@ -0,0 +1,227 @@
+"""
+Daemon Manager - Base class for parser daemons
+
+Strict Pydantic v2 compliance and type safety
+"""
+
+import asyncio
+import signal
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional, Dict, Any
+from pydantic import BaseModel, Field
+
+from .parser_manager import ParserManager, ParserManagerConfig
+from .managers import ParserConfig, LoggingConfig, HTMLCleaningConfig, BrowserConfig
+
+# RPC removed - all commands go through WebSocket bridge
+
+
+class DaemonStatus(BaseModel):
+    """Daemon status information."""
+    running: bool = Field(..., description="Whether daemon is running")
+    parser_id: str = Field(..., description="Parser identifier")
+    started_at: datetime = Field(..., description="Daemon start time")
+    uptime_seconds: float = Field(..., description="Uptime in seconds")
+    schedule_enabled: bool = Field(default=False, description="Whether scheduling is active")
+    next_run_at: Optional[datetime] = Field(default=None, description="Next scheduled run")
+    total_runs: int = Field(default=0, description="Total completed runs")
+    successful_runs: int = Field(default=0, description="Successful runs")
+    failed_runs: int = Field(default=0, description="Failed runs")
+
+
+class DaemonManager(ParserManager):
+    """Base daemon manager with scheduling and status display."""
+    
+    def __init__(self, parser_name: str, parser_type: str, system_dir: str, 
+                 bridge_enabled: bool = False, websocket_url: str = "ws://localhost:8000/ws"):
+        # Create parser config
+        parser_config = ParserConfig(
+            parser_name=parser_name,
+            parser_type=parser_type,
+            system_dir=Path(system_dir)
+        )
+        
+        # Create logging config
+        logging_config = LoggingConfig(parser_name=parser_name)
+        
+        # Create other configs
+        html_config = HTMLCleaningConfig()
+        browser_config = BrowserConfig()
+        
+        # Create manager config
+        manager_config = ParserManagerConfig(
+            parser_config=parser_config,
+            logging_config=logging_config,
+            html_config=html_config,
+            browser_config=browser_config,
+            bridge_enabled=bridge_enabled
+        )
+        
+        super().__init__(manager_config)
+        
+        # Daemon state
+        self.running = False
+        self.started_at: Optional[datetime] = None
+        self.next_run_at: Optional[datetime] = None
+        
+        # Statistics
+        self.total_runs = 0
+        self.successful_runs = 0
+        self.failed_runs = 0
+        
+        # Setup signal handlers
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGTERM, self._signal_handler)
+        
+        # RPC removed - commands come through WebSocket bridge
+    
+    def _signal_handler(self, signum: int, frame) -> None:
+        """Handle shutdown signals."""
+        self.logger.info(f"🛑 Received signal {signum}, shutting down...")
+        self.running = False
+    
+    # RPC methods removed - commands handled through WebSocket bridge
+    
+    async def start_daemon(self, schedule_enabled: bool = False, interval_minutes: Optional[int] = None) -> bool:
+        """Start the daemon."""
+        try:
+            self.logger.info("🚀 Starting daemon...")
+            self.running = True
+            self.started_at = datetime.now()
+            
+            # Initialize parser
+            await self.initialize()
+            
+            # RPC server removed - using WebSocket bridge
+            
+            # Calculate next run if scheduling enabled
+            if schedule_enabled and interval_minutes:
+                self._calculate_next_run(interval_minutes)
+            
+            # Start main loop
+            await self._daemon_loop(schedule_enabled, interval_minutes)
+            
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"❌ Daemon startup failed: {e}")
+            return False
+        finally:
+            await self.cleanup()
+    
+    def _calculate_next_run(self, interval_minutes: int) -> None:
+        """Calculate next scheduled run time."""
+        now = datetime.now()
+        self.next_run_at = now + timedelta(minutes=interval_minutes)
+    
+    async def _daemon_loop(self, schedule_enabled: bool, interval_minutes: Optional[int]) -> None:
+        """Main daemon loop."""
+        self.logger.info("🔄 Daemon loop started")
+        
+        if schedule_enabled and self.next_run_at:
+            self.logger.info(f"⏰ Next run: {self.next_run_at.strftime('%Y-%m-%d %H:%M:%S')}")
+        else:
+            self.logger.info("📋 Manual mode")
+        
+        last_status_update = time.time()
+        
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                # Update status every second
+                if current_time - last_status_update >= 1.0:
+                    self._display_status(schedule_enabled)
+                    last_status_update = current_time
+                
+                # Check for scheduled run
+                if self._should_run_now():
+                    await self._execute_run()
+                    if interval_minutes:
+                        self._calculate_next_run(interval_minutes)
+                
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                self.logger.error(f"❌ Daemon loop error: {e}")
+                await asyncio.sleep(1)
+    
+    def _display_status(self, schedule_enabled: bool) -> None:
+        """Display live status."""
+        if not self.running:
+            return
+        
+        # Clear previous lines
+        print("\033[2K\033[1A" * 3, end="")
+        
+        now = datetime.now()
+        uptime = (now - self.started_at).total_seconds() if self.started_at else 0
+        
+        print(f"🕐 {now.strftime('%H:%M:%S')} | ⏱️  Uptime: {int(uptime//3600):02d}:{int((uptime%3600)//60):02d}:{int(uptime%60):02d}")
+        
+        # Schedule status
+        if self.next_run_at and schedule_enabled:
+            seconds_until = (self.next_run_at - now).total_seconds()
+            if seconds_until > 0:
+                hours = int(seconds_until // 3600)
+                minutes = int((seconds_until % 3600) // 60)
+                seconds = int(seconds_until % 60)
+                print(f"⏰ Next run in: {hours:02d}:{minutes:02d}:{seconds:02d} | 📊 Runs: {self.successful_runs}✅ {self.failed_runs}❌")
+            else:
+                print(f"🚀 Running now... | 📊 Runs: {self.successful_runs}✅ {self.failed_runs}❌")
+        else:
+            print(f"📋 Manual mode | 📊 Runs: {self.successful_runs}✅ {self.failed_runs}❌")
+        
+        status = "🟢 RUNNING" if self.running else "🔴 STOPPED"
+        print(f"{status} | 💾 System: {self.config.system_dir}")
+    
+    def _should_run_now(self) -> bool:
+        """Check if should run now."""
+        if not self.next_run_at:
+            return False
+        return datetime.now() >= self.next_run_at
+    
+    async def _execute_run(self) -> None:
+        """Execute a parsing run - override in subclass."""
+        self.logger.info("🚀 Starting parsing run...")
+        
+        try:
+            # Default implementation - override in subclass
+            result = await self.parse_url("https://example.com")
+            
+            self.total_runs += 1
+            
+            if result.get("success") == "true":
+                self.successful_runs += 1
+                self.logger.info("✅ Run completed successfully")
+            else:
+                self.failed_runs += 1
+                self.logger.error("❌ Run failed")
+            
+        except Exception as e:
+            self.failed_runs += 1
+            self.logger.error(f"❌ Run exception: {e}")
+    
+    def get_status(self) -> DaemonStatus:
+        """Get daemon status."""
+        now = datetime.now()
+        uptime = (now - self.started_at).total_seconds() if self.started_at else 0
+        
+        return DaemonStatus(
+            running=self.running,
+            parser_id=self.config.parser_config.parser_name,
+            started_at=self.started_at or now,
+            uptime_seconds=uptime,
+            schedule_enabled=bool(self.next_run_at),
+            next_run_at=self.next_run_at,
+            total_runs=self.total_runs,
+            successful_runs=self.successful_runs,
+            failed_runs=self.failed_runs
+        )
+    
+    async def cleanup(self):
+        """Cleanup daemon resources."""
+        # RPC server removed - only parent cleanup needed
+        await super().cleanup()

unrealon_driver/parser/managers/__init__.py

@@ -0,0 +1,46 @@
+"""
+Parser Managers - Specialized management components
+
+All managers follow strict Pydantic v2 compliance and CRITICAL_REQUIREMENTS.md
+"""
+
+from .config import ConfigManager, ParserConfig
+from .result import ResultManager, ParseResult, ParseMetrics, OperationStatus
+from .error import ErrorManager, RetryConfig, ErrorInfo, ErrorSeverity
+from .logging import LoggingManager, LoggingConfig, LogLevel, LogContext
+from .html import HTMLManager, HTMLCleaningConfig, HTMLCleaningStats
+from .browser import BrowserManager, BrowserConfig, BrowserStats
+
+__all__ = [
+    # Config Manager
+    "ConfigManager",
+    "ParserConfig",
+    
+    # Result Manager
+    "ResultManager",
+    "ParseResult",
+    "ParseMetrics",
+    "OperationStatus",
+    
+    # Error Manager
+    "ErrorManager",
+    "RetryConfig",
+    "ErrorInfo",
+    "ErrorSeverity",
+    
+    # Logging Manager
+    "LoggingManager",
+    "LoggingConfig",
+    "LogLevel",
+    "LogContext",
+    
+    # HTML Manager
+    "HTMLManager",
+    "HTMLCleaningConfig",
+    "HTMLCleaningStats",
+    
+    # Browser Manager
+    "BrowserManager",
+    "BrowserConfig",
+    "BrowserStats"
+]

unrealon_driver/parser/managers/browser.py

@@ -0,0 +1,51 @@
+"""
+Browser Manager - Wrapper over unrealon_driver.browser
+
+Simple wrapper that inherits from the main BrowserManager
+"""
+
+from typing import Optional, Dict, Any
+from pydantic import BaseModel, Field, ConfigDict
+
+from unrealon_driver.browser import BrowserManager as BaseBrowserManager, BrowserConfig as BaseBrowserConfig
+
+
+class BrowserConfig(BaseBrowserConfig):
+    """Extended browser configuration for parser manager"""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+
+class BrowserStats(BaseModel):
+    """Browser usage statistics"""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    pages_visited: int = Field(default=0, ge=0)
+    total_load_time: float = Field(default=0.0, ge=0.0)
+    average_load_time: float = Field(default=0.0, ge=0.0)
+    screenshots_taken: int = Field(default=0, ge=0)
+    cookies_saved: int = Field(default=0, ge=0)
+    errors_count: int = Field(default=0, ge=0)
+    session_duration: float = Field(default=0.0, ge=0.0)
+
+
+class BrowserManager(BaseBrowserManager):
+    """
+    🌐 Browser Manager - Wrapper over base browser manager
+
+    Simple wrapper that extends the base BrowserManager with parser-specific functionality
+    """
+
+    def __init__(self, config: BrowserConfig):
+        super().__init__(config)
+        self._stats = BrowserStats()
+
+    def get_stats(self) -> BrowserStats:
+        """Get browser usage statistics"""
+        return self._stats
+
+    async def health_check(self) -> Dict[str, Any]:
+        """Browser health check"""
+        base_health = await super().health_check()
+        return {**base_health, "parser_manager": True}

unrealon_driver/parser/managers/config.py

@@ -0,0 +1,281 @@
+"""
+Config Manager - Type-safe configuration management with Pydantic v2
+
+Strict compliance with CRITICAL_REQUIREMENTS.md:
+- No Dict[str, Any] usage
+- Complete type annotations
+- Pydantic v2 models everywhere
+- No mutable defaults
+"""
+
+from typing import Optional, List
+from pathlib import Path
+from pydantic import BaseModel, Field, ConfigDict, field_validator
+import uuid
+
+
+class ParserConfig(BaseModel):
+    """
+    Parser configuration with smart defaults and strict typing
+    
+    Zero configuration approach - everything has sensible defaults
+    """
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid",
+        str_strip_whitespace=True
+    )
+    
+    # Parser identity
+    parser_id: str = Field(
+        default_factory=lambda: f"parser_{uuid.uuid4().hex[:8]}",
+        description="Unique parser identifier"
+    )
+    parser_name: str = Field(
+        default="UnrealOn Parser",
+        description="Human-readable parser name"
+    )
+    parser_type: str = Field(
+        default="generic",
+        description="Parser type for classification"
+    )
+    
+    # Connection settings
+    websocket_url: str = Field(
+        default="ws://localhost:8002/ws",
+        description="WebSocket bridge URL"
+    )
+    api_key: Optional[str] = Field(
+        default=None,
+        description="API key for authentication"
+    )
+    
+    # Browser settings
+    headless: bool = Field(
+        default=True,
+        description="Run browser in headless mode"
+    )
+    stealth_mode: bool = Field(
+        default=True,
+        description="Enable stealth mode"
+    )
+    user_agent: Optional[str] = Field(
+        default=None,
+        description="Custom user agent"
+    )
+    
+    # HTML cleaning settings
+    aggressive_cleaning: bool = Field(
+        default=True,
+        description="Enable aggressive HTML cleaning"
+    )
+    preserve_js_data: bool = Field(
+        default=True,
+        description="Preserve JavaScript data during cleaning"
+    )
+    
+    # Timeouts (in milliseconds)
+    page_timeout: int = Field(
+        default=30000,
+        ge=1000,
+        le=300000,
+        description="Page load timeout in milliseconds"
+    )
+    navigation_timeout: int = Field(
+        default=30000,
+        ge=1000,
+        le=300000,
+        description="Navigation timeout in milliseconds"
+    )
+    
+    # Directories
+    system_dir: Optional[Path] = Field(
+        default=None,
+        description="System directory for logs and data"
+    )
+    screenshots_dir: Optional[Path] = Field(
+        default=None,
+        description="Screenshots directory"
+    )
+    
+    # Development settings
+    debug: bool = Field(
+        default=False,
+        description="Enable debug mode"
+    )
+    save_html: bool = Field(
+        default=False,
+        description="Save HTML files for debugging"
+    )
+    save_screenshots: bool = Field(
+        default=False,
+        description="Save screenshots for debugging"
+    )
+    
+    @field_validator('parser_name')
+    @classmethod
+    def validate_parser_name(cls, v: str) -> str:
+        """Validate parser name is not empty"""
+        if not v.strip():
+            raise ValueError("Parser name cannot be empty")
+        return v.strip()
+    
+    @field_validator('parser_type')
+    @classmethod
+    def validate_parser_type(cls, v: str) -> str:
+        """Validate parser type"""
+        allowed_types = {
+            "generic", "ecommerce", "news", "jobs", 
+            "real_estate", "social_media", "reviews", 
+            "events", "directory"
+        }
+        if v not in allowed_types:
+            raise ValueError(f"Parser type must be one of: {', '.join(allowed_types)}")
+        return v
+    
+    @field_validator('websocket_url')
+    @classmethod
+    def validate_websocket_url(cls, v: str) -> str:
+        """Validate WebSocket URL format"""
+        if not v.startswith(('ws://', 'wss://')):
+            raise ValueError("WebSocket URL must start with ws:// or wss://")
+        return v
+    
+    def model_post_init(self, __context) -> None:
+        """Post-initialization setup"""
+        # Setup system directory if not provided
+        if self.system_dir is None:
+            self.system_dir = Path.cwd() / "system"
+        
+        # Setup screenshots directory if not provided
+        if self.screenshots_dir is None:
+            self.screenshots_dir = self.system_dir / "screenshots"
+        
+        # Create directories
+        self.system_dir.mkdir(parents=True, exist_ok=True)
+        self.screenshots_dir.mkdir(parents=True, exist_ok=True)
+
+
+class ConfigManager:
+    """
+    🔧 Config Manager - Type-safe configuration management
+    
+    Features:
+    - Pydantic v2 validation
+    - Environment variable integration
+    - Configuration profiles
+    - Hot reloading
+    - Type safety enforcement
+    """
+    
+    def __init__(self, config: Optional[ParserConfig] = None):
+        self._config: ParserConfig = config or ParserConfig()
+        self._profiles: dict[str, ParserConfig] = {}
+        self._current_profile: Optional[str] = None
+    
+    @property
+    def config(self) -> ParserConfig:
+        """Get current configuration"""
+        return self._config
+    
+    def update_config(self, **kwargs) -> None:
+        """Update configuration with new values"""
+        # Create new config with updated values
+        current_data = self._config.model_dump()
+        current_data.update(kwargs)
+        self._config = ParserConfig.model_validate(current_data)
+    
+    def load_from_dict(self, config_dict: dict[str, str]) -> None:
+        """Load configuration from dictionary"""
+        self._config = ParserConfig.model_validate(config_dict)
+    
+    def load_from_env(self, prefix: str = "PARSER_") -> None:
+        """Load configuration from environment variables"""
+        import os
+        
+        env_config = {}
+        for key, value in os.environ.items():
+            if key.startswith(prefix):
+                config_key = key[len(prefix):].lower()
+                
+                # Convert string values to appropriate types
+                if config_key in ['headless', 'stealth_mode', 'aggressive_cleaning', 
+                                 'preserve_js_data', 'debug', 'save_html', 'save_screenshots']:
+                    env_config[config_key] = value.lower() in ('true', '1', 'yes', 'on')
+                elif config_key in ['page_timeout', 'navigation_timeout']:
+                    env_config[config_key] = int(value)
+                elif config_key in ['system_dir', 'screenshots_dir']:
+                    env_config[config_key] = Path(value)
+                else:
+                    env_config[config_key] = value
+        
+        if env_config:
+            current_data = self._config.model_dump()
+            current_data.update(env_config)
+            self._config = ParserConfig.model_validate(current_data)
+    
+    def save_profile(self, name: str) -> None:
+        """Save current configuration as a profile"""
+        if not name.strip():
+            raise ValueError("Profile name cannot be empty")
+        self._profiles[name] = ParserConfig.model_validate(self._config.model_dump())
+    
+    def load_profile(self, name: str) -> None:
+        """Load configuration from a saved profile"""
+        if name not in self._profiles:
+            raise ValueError(f"Profile '{name}' not found")
+        self._config = ParserConfig.model_validate(self._profiles[name].model_dump())
+        self._current_profile = name
+    
+    def get_profiles(self) -> List[str]:
+        """Get list of available profiles"""
+        return list(self._profiles.keys())
+    
+    def delete_profile(self, name: str) -> None:
+        """Delete a saved profile"""
+        if name not in self._profiles:
+            raise ValueError(f"Profile '{name}' not found")
+        del self._profiles[name]
+        if self._current_profile == name:
+            self._current_profile = None
+    
+    def get_current_profile(self) -> Optional[str]:
+        """Get current profile name"""
+        return self._current_profile
+    
+    def validate_config(self) -> List[str]:
+        """Validate current configuration and return any issues"""
+        issues = []
+        
+        # Check directory permissions
+        try:
+            test_file = self._config.system_dir / ".test"
+            test_file.touch()
+            test_file.unlink()
+        except PermissionError:
+            issues.append(f"No write permission for system directory: {self._config.system_dir}")
+        except Exception as e:
+            issues.append(f"System directory issue: {e}")
+        
+        # Check timeouts are reasonable
+        if self._config.page_timeout < 5000:
+            issues.append("Page timeout is very low (< 5 seconds)")
+        if self._config.navigation_timeout < 5000:
+            issues.append("Navigation timeout is very low (< 5 seconds)")
+        
+        return issues
+    
+    def to_dict(self) -> dict[str, str]:
+        """Export configuration as dictionary"""
+        return self._config.model_dump(mode='json')
+    
+    def to_env_format(self, prefix: str = "PARSER_") -> List[str]:
+        """Export configuration as environment variable format"""
+        config_dict = self.to_dict()
+        env_vars = []
+        
+        for key, value in config_dict.items():
+            env_key = f"{prefix}{key.upper()}"
+            env_vars.append(f"{env_key}={value}")
+        
+        return env_vars