unrealon 1.1.1__py3-none-any.whl → 1.1.5__py3-none-any.whl

unrealon_driver/html_analyzer/config.py

@@ -0,0 +1,64 @@
+"""
+Configuration models for HTML Analyzer.
+"""
+
+from typing import Optional, List
+from pathlib import Path
+from pydantic import BaseModel, Field, ConfigDict
+
+from unrealon_driver.websocket import get_websocket_url
+
+
+class HTMLCleaningConfig(BaseModel):
+    """HTML cleaning configuration with strict typing"""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    # Cleaning modes
+    aggressive_cleaning: bool = Field(default=True, description="Enable aggressive cleaning")
+    preserve_js_data: bool = Field(default=True, description="Preserve JavaScript data during cleaning")
+
+    # Content preservation
+    preserve_images: bool = Field(default=False, description="Preserve image tags")
+    preserve_links: bool = Field(default=True, description="Preserve link tags")
+    preserve_forms: bool = Field(default=False, description="Preserve form elements")
+
+    # Size limits
+    max_html_size: int = Field(default=1000000, ge=1000, le=10000000, description="Maximum HTML size in characters")
+    max_text_length: int = Field(default=300, ge=50, le=1000, description="Maximum text content length per element")
+    max_url_length: int = Field(default=500, ge=100, le=2000, description="Maximum URL length")
+
+    # Noise removal
+    remove_comments: bool = Field(default=True, description="Remove HTML comments")
+    remove_scripts: bool = Field(default=True, description="Remove script tags")
+    remove_styles: bool = Field(default=True, description="Remove style tags")
+    remove_tracking: bool = Field(default=True, description="Remove tracking URLs and attributes")
+
+    # Whitespace handling
+    normalize_whitespace: bool = Field(default=True, description="Normalize whitespace")
+    remove_empty_elements: bool = Field(default=True, description="Remove empty elements")
+
+    # Custom selectors
+    noise_selectors: List[str] = Field(
+        default_factory=lambda: ['[class*="nav"]', '[class*="menu"]', '[class*="sidebar"]', '[class*="footer"]', '[class*="header"]', '[class*="ads"]', '[class*="popup"]', '[class*="modal"]', '[class*="cookie"]'], description="CSS selectors for noise elements to remove"
+    )
+
+
+class HTMLAnalyzerConfig(BaseModel):
+    """Configuration for HTML Analyzer"""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    # Parser identity
+    parser_id: str = Field(..., min_length=1)
+
+    # Cleaning configuration
+    cleaning_config: HTMLCleaningConfig = Field(default_factory=HTMLCleaningConfig)
+
+    # WebSocket configuration (auto-detected)
+    websocket_url: Optional[str] = Field(default_factory=lambda: get_websocket_url(), description="WebSocket URL for analysis requests (auto-detected based on environment)")
+    api_key: Optional[str] = Field(default=None, description="API key for authentication")
+
+    # Analysis settings
+    default_timeout: float = Field(default=60.0, gt=0.0, description="Default analysis timeout")
+    enable_websocket_analysis: bool = Field(default=True, description="Enable WebSocket-based analysis")

unrealon_driver/html_analyzer/manager.py

@@ -0,0 +1,247 @@
+"""
+HTML Analyzer Manager - Main interface for HTML analysis operations.
+"""
+
+from typing import Optional, Tuple
+from unrealon_driver.smart_logging import create_smart_logger
+
+from .config import HTMLAnalyzerConfig, HTMLCleaningConfig
+from .cleaner import HTMLCleaner, HTMLCleaningStats
+from .websocket_analyzer import WebSocketHTMLAnalyzer
+from .models import HTMLAnalysisResult, HTMLParseResult, HTMLAnalyzerStats, HTMLAnalysisRequest, HTMLParseRequest, HTMLAnalyzerError, HTMLCleaningError, HTMLAnalysisError
+
+
+class HTMLAnalyzer:
+    """
+    🧠 HTML Analyzer - Complete HTML processing and analysis system
+
+    Features:
+    - Smart HTML cleaning with noise removal
+    - JavaScript data extraction
+    - WebSocket-based LLM analysis
+    - Token-optimized output
+    - Configurable cleaning strategies
+    """
+
+    def __init__(self, config: HTMLAnalyzerConfig):
+        self.config = config
+        self.logger = create_smart_logger(parser_id=config.parser_id)
+
+        # Initialize components
+        self.cleaner = HTMLCleaner(parser_id=config.parser_id, config=config.cleaning_config)
+
+        # Initialize WebSocket analyzer if enabled
+        if config.enable_websocket_analysis and config.websocket_url:
+            self.websocket_analyzer = WebSocketHTMLAnalyzer(config)
+        else:
+            self.websocket_analyzer = None
+
+    async def analyze_html(self, html: str, instructions: Optional[str] = None, session_id: Optional[str] = None, clean_first: bool = True, preserve_js_data: bool = True, aggressive_cleaning: bool = False, **kwargs) -> HTMLAnalysisResult:
+        """
+        Complete HTML analysis workflow
+
+        Args:
+            html: Raw HTML content
+            instructions: Analysis instructions for LLM
+            session_id: Session identifier
+            clean_first: Whether to clean HTML before analysis
+            preserve_js_data: Whether to extract JavaScript data
+            aggressive_cleaning: Whether to apply aggressive cleaning
+            **kwargs: Additional parameters
+
+        Returns:
+            Analysis result with cleaned HTML and extracted data
+        """
+        try:
+            self.logger.info("🧠 Starting HTML analysis workflow")
+
+            # Initialize result with proper typing
+            result_data = {"success": True, "original_html_size": len(html), "cleaned_html": html, "extracted_data": {}, "analysis_result": {}, "cleaning_stats": {}, "error_message": ""}
+
+            # Step 1: Clean HTML if requested
+            if clean_first:
+                cleaned_html, extracted_data = await self.cleaner.clean_html(html, preserve_js_data=preserve_js_data, aggressive_cleaning=aggressive_cleaning)
+
+                result_data["cleaned_html"] = cleaned_html
+                result_data["extracted_data"] = extracted_data
+                result_data["cleaned_html_size"] = len(cleaned_html)
+
+                # Get cleaning statistics
+                stats = self.cleaner.get_cleaning_stats(html, cleaned_html)
+                result_data["cleaning_stats"] = stats.model_dump()
+
+                self.logger.info(f"✅ HTML cleaned: {len(html)} → {len(cleaned_html)} chars")
+            else:
+                result_data["cleaned_html_size"] = len(html)
+
+            # Step 2: Perform LLM analysis via WebSocket if available
+            if self.websocket_analyzer and instructions:
+                analysis_result = await self.websocket_analyzer.analyze_html(result_data["cleaned_html"], instructions=instructions, session_id=session_id, **kwargs)
+                result_data["analysis_result"] = analysis_result
+
+                if analysis_result.get("success") == "true":
+                    self.logger.info("✅ LLM analysis completed successfully")
+                else:
+                    self.logger.warning(f"⚠️ LLM analysis failed: {analysis_result.get('error_message')}")
+            else:
+                if not self.websocket_analyzer:
+                    self.logger.info("ℹ️ WebSocket analyzer not configured - skipping LLM analysis")
+                else:
+                    self.logger.info("ℹ️ No instructions provided - skipping LLM analysis")
+
+            return HTMLAnalysisResult.model_validate(result_data)
+
+        except Exception as e:
+            self.logger.error(f"❌ HTML analysis failed: {str(e)}")
+            error_result = {"success": False, "original_html_size": len(html), "cleaned_html": "", "cleaned_html_size": 0, "extracted_data": {}, "analysis_result": {}, "cleaning_stats": {}, "error_message": str(e)}
+            return HTMLAnalysisResult.model_validate(error_result)
+
+    async def clean_html_only(self, html: str, preserve_js_data: bool = True, aggressive_cleaning: bool = False) -> Tuple[str, dict[str, str]]:
+        """
+        Clean HTML without LLM analysis
+
+        Args:
+            html: Raw HTML content
+            preserve_js_data: Whether to extract JavaScript data
+            aggressive_cleaning: Whether to apply aggressive cleaning
+
+        Returns:
+            Tuple of (cleaned_html, extracted_data)
+        """
+        return await self.cleaner.clean_html(html, preserve_js_data=preserve_js_data, aggressive_cleaning=aggressive_cleaning)
+
+    async def analyze_with_llm_only(self, html: str, instructions: str, session_id: Optional[str] = None, **kwargs) -> dict[str, str]:
+        """
+        Perform LLM analysis without cleaning
+
+        Args:
+            html: HTML content (should be pre-cleaned)
+            instructions: Analysis instructions
+            session_id: Session identifier
+            **kwargs: Additional parameters
+
+        Returns:
+            LLM analysis result
+        """
+        if not self.websocket_analyzer:
+            return {"success": "false", "parsed_data": "", "markdown": "", "error_message": "WebSocket analyzer not configured"}
+
+        return await self.websocket_analyzer.analyze_html(html, instructions=instructions, session_id=session_id, **kwargs)
+
+    async def parse_html(self, html: str, url: Optional[str] = None, instructions: Optional[str] = None, session_id: Optional[str] = None, **kwargs) -> HTMLParseResult:
+        """
+        Complete HTML parsing workflow: clean → analyze.
+
+        This is the main method that should be used by ParserManager.
+        Returns standardized string-based result format.
+
+        Args:
+            html: Raw HTML content
+            url: Source URL (for logging)
+            instructions: Optional analysis instructions
+            session_id: Optional session ID
+            **kwargs: Additional parameters
+
+        Returns:
+            Standardized parsing result dictionary with string values
+        """
+        try:
+            if url:
+                self.logger.info(f"🔄 Processing HTML from {url}: {len(html)} characters")
+            else:
+                self.logger.info(f"🔄 Processing HTML: {len(html)} characters")
+
+            # Use existing analyze_html method
+            result = await self.analyze_html(html=html, instructions=instructions, session_id=session_id, **kwargs)
+
+            # Convert to standardized string format for ParserManager
+            if result.success:
+                analysis_result = result.analysis_result
+                return HTMLParseResult(success="true", parsed_data=str(analysis_result.get("parsed_data", "")), markdown=str(analysis_result.get("markdown", "")), error_message="")
+            else:
+                return HTMLParseResult(success="false", parsed_data="", markdown="", error_message=result.error_message or "Analysis failed")
+
+        except Exception as e:
+            self.logger.error(f"❌ HTML parsing failed: {str(e)}")
+            return HTMLParseResult(success="false", parsed_data="", markdown="", error_message=str(e))
+
+    def get_cleaning_stats(self, original_html: str, cleaned_html: str) -> HTMLCleaningStats:
+        """Get cleaning statistics"""
+        return self.cleaner.get_cleaning_stats(original_html, cleaned_html)
+
+    def get_stats(self) -> HTMLAnalyzerStats:
+        """Get HTML analyzer statistics"""
+        return HTMLAnalyzerStats(cleaned_count=getattr(self.cleaner, "_cleaned_count", 0), total_reduction=getattr(self.cleaner, "_total_reduction", 0.0), websocket_enabled=self.websocket_analyzer is not None)
+
+    async def close(self):
+        """Close all resources"""
+        if self.websocket_analyzer:
+            await self.websocket_analyzer.close()
+        self.logger.info("🔌 HTML Analyzer closed")
+
+
+def create_html_analyzer(parser_id: str, websocket_url: Optional[str] = None, api_key: Optional[str] = None, cleaning_config: Optional[HTMLCleaningConfig] = None, **kwargs) -> HTMLAnalyzer:
+    """
+    Create HTML analyzer with configuration
+
+    Args:
+        parser_id: Parser identifier
+        websocket_url: WebSocket URL for LLM analysis (optional, auto-detected if not provided)
+        api_key: API key for authentication
+        cleaning_config: HTML cleaning configuration
+        **kwargs: Additional configuration options
+
+    Returns:
+        Configured HTMLAnalyzer instance
+    """
+    # Only pass websocket_url if explicitly provided, otherwise use auto-detection
+    config_kwargs = {"parser_id": parser_id, "api_key": api_key, "cleaning_config": cleaning_config or HTMLCleaningConfig(), **kwargs}
+    if websocket_url is not None:
+        config_kwargs["websocket_url"] = websocket_url
+    
+    config = HTMLAnalyzerConfig(**config_kwargs)
+
+    return HTMLAnalyzer(config)
+
+
+# Convenience functions
+async def quick_analyze_html(html: str, parser_id: str, instructions: Optional[str] = None, websocket_url: Optional[str] = None, **kwargs) -> HTMLAnalysisResult:
+    """
+    Quick HTML analysis convenience function
+
+    Args:
+        html: Raw HTML content
+        instructions: Analysis instructions
+        parser_id: Parser identifier
+        websocket_url: WebSocket URL for analysis (optional, auto-detected if not provided)
+        **kwargs: Additional options
+
+    Returns:
+        Analysis result
+    """
+    analyzer = create_html_analyzer(parser_id=parser_id, websocket_url=websocket_url, **kwargs)
+
+    try:
+        return await analyzer.analyze_html(html, instructions=instructions, **kwargs)
+    finally:
+        await analyzer.close()
+
+
+async def quick_clean_html(html: str, parser_id: str, **kwargs) -> Tuple[str, dict[str, str]]:
+    """
+    Quick HTML cleaning convenience function
+
+    Args:
+        html: Raw HTML content
+        parser_id: Parser identifier
+        **kwargs: Cleaning options
+
+    Returns:
+        Tuple of (cleaned_html, extracted_data)
+    """
+    analyzer = create_html_analyzer(parser_id=parser_id)
+
+    try:
+        return await analyzer.clean_html_only(html, **kwargs)
+    finally:
+        await analyzer.close()

unrealon_driver/html_analyzer/models.py

@@ -0,0 +1,115 @@
+"""
+HTML Analyzer Models - Pydantic v2 models for HTML analysis operations.
+
+Strict compliance with CRITICAL_REQUIREMENTS.md:
+- No Dict[str, Any] usage
+- Complete type annotations
+- Pydantic v2 models everywhere
+- Custom exception hierarchy
+- No try blocks in imports
+"""
+
+from typing import Optional
+from pydantic import BaseModel, Field, ConfigDict
+
+
+class HTMLAnalysisResult(BaseModel):
+    """Complete HTML analysis result with proper typing."""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    success: bool = Field(..., description="Analysis success status")
+    original_html_size: int = Field(..., ge=0, description="Original HTML size in characters")
+    cleaned_html: str = Field(..., description="Cleaned HTML content")
+    cleaned_html_size: int = Field(..., ge=0, description="Cleaned HTML size in characters")
+    extracted_data: dict[str, str] = Field(default_factory=dict, description="Extracted JavaScript data")
+    analysis_result: dict[str, str] = Field(default_factory=dict, description="LLM analysis result")
+    cleaning_stats: dict[str, float] = Field(default_factory=dict, description="Cleaning statistics")
+    error_message: str = Field(default="", description="Error message if failed")
+
+
+class HTMLParseResult(BaseModel):
+    """Standardized HTML parsing result for ParserManager."""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    success: str = Field(..., description="Success status as string (true/false)")
+    parsed_data: str = Field(..., description="Parsed data as string")
+    markdown: str = Field(..., description="Markdown representation")
+    error_message: str = Field(..., description="Error message if failed")
+
+
+class HTMLAnalyzerStats(BaseModel):
+    """HTML analyzer statistics."""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    cleaned_count: int = Field(default=0, ge=0, description="Number of HTML documents cleaned")
+    total_reduction: float = Field(default=0.0, ge=0.0, description="Total size reduction percentage")
+    websocket_enabled: bool = Field(..., description="Whether WebSocket analyzer is enabled")
+
+
+class HTMLCleaningRequest(BaseModel):
+    """Request model for HTML cleaning operations."""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    html: str = Field(..., min_length=1, description="HTML content to clean")
+    preserve_js_data: bool = Field(default=True, description="Whether to extract JavaScript data")
+    aggressive_cleaning: bool = Field(default=False, description="Whether to apply aggressive cleaning")
+
+
+class HTMLAnalysisRequest(BaseModel):
+    """Request model for HTML analysis operations."""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    html: str = Field(..., min_length=1, description="HTML content to analyze")
+    instructions: Optional[str] = Field(default=None, description="Analysis instructions for LLM")
+    session_id: Optional[str] = Field(default=None, description="Session identifier")
+    url: Optional[str] = Field(default=None, description="Source URL for logging")
+    clean_first: bool = Field(default=True, description="Whether to clean HTML before analysis")
+    preserve_js_data: bool = Field(default=True, description="Whether to extract JavaScript data")
+    aggressive_cleaning: bool = Field(default=False, description="Whether to apply aggressive cleaning")
+
+
+class HTMLParseRequest(BaseModel):
+    """Complete HTML parsing request model."""
+
+    model_config = ConfigDict(validate_assignment=True, extra="forbid")
+
+    html: str = Field(..., min_length=1, description="HTML content to parse")
+    url: Optional[str] = Field(default=None, description="Source URL for logging")
+    instructions: Optional[str] = Field(default=None, description="Analysis instructions")
+    session_id: Optional[str] = Field(default=None, description="Session identifier")
+    clean_first: bool = Field(default=True, description="Whether to clean HTML before analysis")
+    preserve_js_data: bool = Field(default=True, description="Whether to extract JavaScript data")
+    aggressive_cleaning: bool = Field(default=False, description="Whether to apply aggressive cleaning")
+
+
+class HTMLAnalyzerError(Exception):
+    """Base exception for HTML analyzer operations."""
+
+    def __init__(self, message: str, operation: str, details: Optional[dict[str, str]] = None):
+        self.message = message
+        self.operation = operation
+        self.details = details or {}
+        super().__init__(message)
+
+
+class HTMLCleaningError(HTMLAnalyzerError):
+    """Raised when HTML cleaning fails."""
+
+    pass
+
+
+class HTMLAnalysisError(HTMLAnalyzerError):
+    """Raised when HTML analysis fails."""
+
+    pass
+
+
+class WebSocketAnalysisError(HTMLAnalyzerError):
+    """Raised when WebSocket analysis fails."""
+
+    pass

unrealon_driver/html_analyzer/websocket_analyzer.py

@@ -0,0 +1,157 @@
+"""
+WebSocket HTML Analyzer - Handles HTML analysis via WebSocket communication.
+"""
+
+from typing import Optional, Dict, Any
+from unrealon_driver.smart_logging import create_smart_logger
+from unrealon_driver.websocket import websocket_manager, WebSocketConfig
+
+from .config import HTMLAnalyzerConfig
+
+
+class WebSocketHTMLAnalyzer:
+    """
+    WebSocket-based HTML analyzer that sends HTML to server for LLM analysis.
+    """
+    
+    def __init__(self, config: HTMLAnalyzerConfig):
+        self.config = config
+        self.logger = create_smart_logger(parser_id=config.parser_id)
+        
+        # Initialize WebSocket if configured
+        if config.websocket_url and config.enable_websocket_analysis:
+            self._websocket_config = WebSocketConfig(
+                url=config.websocket_url,
+                api_key=config.api_key,
+                parser_id=config.parser_id
+            )
+            self._websocket_initialized = False
+        else:
+            self._websocket_config = None
+            self._websocket_initialized = False
+    
+    async def analyze_html(
+        self, 
+        html: str, 
+        instructions: Optional[str] = None, 
+        session_id: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, str]:
+        """
+        Analyze HTML content via WebSocket
+        
+        Args:
+            html: HTML content to analyze
+            instructions: Analysis instructions
+            session_id: Session identifier
+            **kwargs: Additional parameters
+            
+        Returns:
+            Analysis result dictionary
+        """
+        if not self._websocket_config:
+            self.logger.warning("🔌 WebSocket not configured for HTML analysis")
+            return {
+                "success": "false",
+                "parsed_data": "",
+                "markdown": "",
+                "error_message": "WebSocket not configured"
+            }
+        
+        try:
+            # Ensure WebSocket connection
+            if not self._websocket_initialized:
+                await self._initialize_websocket()
+            
+            self.logger.info("🤖 Analyzing HTML with LLM via WebSocket...")
+            
+            # Prepare analysis request
+            analysis_request = {
+                "type": "html_analysis_request",
+                "parser_id": self.config.parser_id,
+                "session_id": session_id,
+                "html_content": html,
+                "instructions": instructions or "Extract and structure the data from this HTML",
+                "parse_type": "general",
+                "timeout": kwargs.get("timeout", self.config.default_timeout),
+                "metadata": kwargs.get("metadata", {})
+            }
+            
+            # Send request via WebSocket
+            if websocket_manager.connected:
+                response = await websocket_manager.send_request(
+                    analysis_request, 
+                    timeout=kwargs.get("timeout", self.config.default_timeout)
+                )
+                
+                if response and response.get("success"):
+                    self.logger.info("✅ HTML analysis completed successfully")
+                    return {
+                        "success": "true",
+                        "parsed_data": str(response.get("parsed_data", "")),
+                        "markdown": response.get("markdown", ""),
+                        "error_message": ""
+                    }
+                else:
+                    error_msg = response.get("error_message", "Analysis failed") if response else "No response"
+                    self.logger.error(f"❌ HTML analysis failed: {error_msg}")
+                    return {
+                        "success": "false", 
+                        "parsed_data": "",
+                        "markdown": "",
+                        "error_message": error_msg
+                    }
+            else:
+                self.logger.warning("🔌 WebSocket not connected for HTML analysis")
+                return {
+                    "success": "false",
+                    "parsed_data": "",
+                    "markdown": "",
+                    "error_message": "WebSocket not connected"
+                }
+                
+        except Exception as e:
+            self.logger.error(f"❌ HTML analysis failed: {str(e)}")
+            return {
+                "success": "false",
+                "parsed_data": "",
+                "markdown": "",
+                "error_message": str(e)
+            }
+    
+    async def _initialize_websocket(self) -> bool:
+        """Initialize WebSocket connection"""
+        if not self._websocket_config:
+            return False
+        
+        try:
+            success = await websocket_manager.initialize(self._websocket_config)
+            if success:
+                self._websocket_initialized = True
+                self.logger.info("🔌 WebSocket initialized for HTML analysis")
+            else:
+                self.logger.warning("🔌 WebSocket initialization failed")
+            return success
+        except Exception as e:
+            self.logger.error(f"❌ WebSocket initialization error: {e}")
+            return False
+    
+    async def close(self):
+        """Close WebSocket connection"""
+        if self._websocket_initialized:
+            await websocket_manager.disconnect()
+            self._websocket_initialized = False
+            self.logger.info("🔌 WebSocket connection closed")
+
+
+def create_websocket_analyzer(config: HTMLAnalyzerConfig) -> WebSocketHTMLAnalyzer:
+    """
+    Create WebSocket HTML analyzer
+    
+    Args:
+        config: HTML analyzer configuration
+        
+    Returns:
+        Configured WebSocketHTMLAnalyzer instance
+    """
+    return WebSocketHTMLAnalyzer(config)

unrealon_driver/models/__init__.py

@@ -0,0 +1,31 @@
+"""
+Models for unrealon_driver.
+
+Pydantic v2 models for type safety and validation.
+"""
+
+from .websocket import (
+    MessageType,
+    BridgeMessageType,
+    RegistrationMessage,
+    CommandMessage,
+    CommandResponseMessage,
+    StatusMessage,
+    HeartbeatMessage,
+    BridgeRegistrationPayload,
+    BridgeMessage,
+    BridgeRegistrationMessage
+)
+
+__all__ = [
+    "MessageType",
+    "BridgeMessageType",
+    "RegistrationMessage", 
+    "CommandMessage",
+    "CommandResponseMessage",
+    "StatusMessage",
+    "HeartbeatMessage",
+    "BridgeRegistrationPayload",
+    "BridgeMessage",
+    "BridgeRegistrationMessage"
+]