unrealon 1.1.6__py3-none-any.whl → 2.0.5__py3-none-any.whl

unrealon_driver/html_analyzer/config.py

@@ -1,64 +0,0 @@
-"""
-Configuration models for HTML Analyzer.
-"""
-
-from typing import Optional, List
-from pathlib import Path
-from pydantic import BaseModel, Field, ConfigDict
-
-from unrealon_driver.websocket import get_websocket_url
-
-
-class HTMLCleaningConfig(BaseModel):
-    """HTML cleaning configuration with strict typing"""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    # Cleaning modes
-    aggressive_cleaning: bool = Field(default=True, description="Enable aggressive cleaning")
-    preserve_js_data: bool = Field(default=True, description="Preserve JavaScript data during cleaning")
-
-    # Content preservation
-    preserve_images: bool = Field(default=False, description="Preserve image tags")
-    preserve_links: bool = Field(default=True, description="Preserve link tags")
-    preserve_forms: bool = Field(default=False, description="Preserve form elements")
-
-    # Size limits
-    max_html_size: int = Field(default=1000000, ge=1000, le=10000000, description="Maximum HTML size in characters")
-    max_text_length: int = Field(default=300, ge=50, le=1000, description="Maximum text content length per element")
-    max_url_length: int = Field(default=500, ge=100, le=2000, description="Maximum URL length")
-
-    # Noise removal
-    remove_comments: bool = Field(default=True, description="Remove HTML comments")
-    remove_scripts: bool = Field(default=True, description="Remove script tags")
-    remove_styles: bool = Field(default=True, description="Remove style tags")
-    remove_tracking: bool = Field(default=True, description="Remove tracking URLs and attributes")
-
-    # Whitespace handling
-    normalize_whitespace: bool = Field(default=True, description="Normalize whitespace")
-    remove_empty_elements: bool = Field(default=True, description="Remove empty elements")
-
-    # Custom selectors
-    noise_selectors: List[str] = Field(
-        default_factory=lambda: ['[class*="nav"]', '[class*="menu"]', '[class*="sidebar"]', '[class*="footer"]', '[class*="header"]', '[class*="ads"]', '[class*="popup"]', '[class*="modal"]', '[class*="cookie"]'], description="CSS selectors for noise elements to remove"
-    )
-
-
-class HTMLAnalyzerConfig(BaseModel):
-    """Configuration for HTML Analyzer"""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    # Parser identity
-    parser_id: str = Field(..., min_length=1)
-
-    # Cleaning configuration
-    cleaning_config: HTMLCleaningConfig = Field(default_factory=HTMLCleaningConfig)
-
-    # WebSocket configuration (auto-detected)
-    websocket_url: Optional[str] = Field(default_factory=lambda: get_websocket_url(), description="WebSocket URL for analysis requests (auto-detected based on environment)")
-    api_key: Optional[str] = Field(default=None, description="API key for authentication")
-
-    # Analysis settings
-    default_timeout: float = Field(default=60.0, gt=0.0, description="Default analysis timeout")
-    enable_websocket_analysis: bool = Field(default=True, description="Enable WebSocket-based analysis")

unrealon_driver/html_analyzer/manager.py

@@ -1,247 +0,0 @@
-"""
-HTML Analyzer Manager - Main interface for HTML analysis operations.
-"""
-
-from typing import Optional, Tuple
-from unrealon_driver.smart_logging import create_smart_logger
-
-from .config import HTMLAnalyzerConfig, HTMLCleaningConfig
-from .cleaner import HTMLCleaner, HTMLCleaningStats
-from .websocket_analyzer import WebSocketHTMLAnalyzer
-from .models import HTMLAnalysisResult, HTMLParseResult, HTMLAnalyzerStats, HTMLAnalysisRequest, HTMLParseRequest, HTMLAnalyzerError, HTMLCleaningError, HTMLAnalysisError
-
-
-class HTMLAnalyzer:
-    """
-    🧠 HTML Analyzer - Complete HTML processing and analysis system
-
-    Features:
-    - Smart HTML cleaning with noise removal
-    - JavaScript data extraction
-    - WebSocket-based LLM analysis
-    - Token-optimized output
-    - Configurable cleaning strategies
-    """
-
-    def __init__(self, config: HTMLAnalyzerConfig):
-        self.config = config
-        self.logger = create_smart_logger(parser_id=config.parser_id)
-
-        # Initialize components
-        self.cleaner = HTMLCleaner(parser_id=config.parser_id, config=config.cleaning_config)
-
-        # Initialize WebSocket analyzer if enabled
-        if config.enable_websocket_analysis and config.websocket_url:
-            self.websocket_analyzer = WebSocketHTMLAnalyzer(config)
-        else:
-            self.websocket_analyzer = None
-
-    async def analyze_html(self, html: str, instructions: Optional[str] = None, session_id: Optional[str] = None, clean_first: bool = True, preserve_js_data: bool = True, aggressive_cleaning: bool = False, **kwargs) -> HTMLAnalysisResult:
-        """
-        Complete HTML analysis workflow
-
-        Args:
-            html: Raw HTML content
-            instructions: Analysis instructions for LLM
-            session_id: Session identifier
-            clean_first: Whether to clean HTML before analysis
-            preserve_js_data: Whether to extract JavaScript data
-            aggressive_cleaning: Whether to apply aggressive cleaning
-            **kwargs: Additional parameters
-
-        Returns:
-            Analysis result with cleaned HTML and extracted data
-        """
-        try:
-            self.logger.info("🧠 Starting HTML analysis workflow")
-
-            # Initialize result with proper typing
-            result_data = {"success": True, "original_html_size": len(html), "cleaned_html": html, "extracted_data": {}, "analysis_result": {}, "cleaning_stats": {}, "error_message": ""}
-
-            # Step 1: Clean HTML if requested
-            if clean_first:
-                cleaned_html, extracted_data = await self.cleaner.clean_html(html, preserve_js_data=preserve_js_data, aggressive_cleaning=aggressive_cleaning)
-
-                result_data["cleaned_html"] = cleaned_html
-                result_data["extracted_data"] = extracted_data
-                result_data["cleaned_html_size"] = len(cleaned_html)
-
-                # Get cleaning statistics
-                stats = self.cleaner.get_cleaning_stats(html, cleaned_html)
-                result_data["cleaning_stats"] = stats.model_dump()
-
-                self.logger.info(f"✅ HTML cleaned: {len(html)} → {len(cleaned_html)} chars")
-            else:
-                result_data["cleaned_html_size"] = len(html)
-
-            # Step 2: Perform LLM analysis via WebSocket if available
-            if self.websocket_analyzer and instructions:
-                analysis_result = await self.websocket_analyzer.analyze_html(result_data["cleaned_html"], instructions=instructions, session_id=session_id, **kwargs)
-                result_data["analysis_result"] = analysis_result
-
-                if analysis_result.get("success") == "true":
-                    self.logger.info("✅ LLM analysis completed successfully")
-                else:
-                    self.logger.warning(f"⚠️ LLM analysis failed: {analysis_result.get('error_message')}")
-            else:
-                if not self.websocket_analyzer:
-                    self.logger.info("ℹ️ WebSocket analyzer not configured - skipping LLM analysis")
-                else:
-                    self.logger.info("ℹ️ No instructions provided - skipping LLM analysis")
-
-            return HTMLAnalysisResult.model_validate(result_data)
-
-        except Exception as e:
-            self.logger.error(f"❌ HTML analysis failed: {str(e)}")
-            error_result = {"success": False, "original_html_size": len(html), "cleaned_html": "", "cleaned_html_size": 0, "extracted_data": {}, "analysis_result": {}, "cleaning_stats": {}, "error_message": str(e)}
-            return HTMLAnalysisResult.model_validate(error_result)
-
-    async def clean_html_only(self, html: str, preserve_js_data: bool = True, aggressive_cleaning: bool = False) -> Tuple[str, dict[str, str]]:
-        """
-        Clean HTML without LLM analysis
-
-        Args:
-            html: Raw HTML content
-            preserve_js_data: Whether to extract JavaScript data
-            aggressive_cleaning: Whether to apply aggressive cleaning
-
-        Returns:
-            Tuple of (cleaned_html, extracted_data)
-        """
-        return await self.cleaner.clean_html(html, preserve_js_data=preserve_js_data, aggressive_cleaning=aggressive_cleaning)
-
-    async def analyze_with_llm_only(self, html: str, instructions: str, session_id: Optional[str] = None, **kwargs) -> dict[str, str]:
-        """
-        Perform LLM analysis without cleaning
-
-        Args:
-            html: HTML content (should be pre-cleaned)
-            instructions: Analysis instructions
-            session_id: Session identifier
-            **kwargs: Additional parameters
-
-        Returns:
-            LLM analysis result
-        """
-        if not self.websocket_analyzer:
-            return {"success": "false", "parsed_data": "", "markdown": "", "error_message": "WebSocket analyzer not configured"}
-
-        return await self.websocket_analyzer.analyze_html(html, instructions=instructions, session_id=session_id, **kwargs)
-
-    async def parse_html(self, html: str, url: Optional[str] = None, instructions: Optional[str] = None, session_id: Optional[str] = None, **kwargs) -> HTMLParseResult:
-        """
-        Complete HTML parsing workflow: clean → analyze.
-
-        This is the main method that should be used by ParserManager.
-        Returns standardized string-based result format.
-
-        Args:
-            html: Raw HTML content
-            url: Source URL (for logging)
-            instructions: Optional analysis instructions
-            session_id: Optional session ID
-            **kwargs: Additional parameters
-
-        Returns:
-            Standardized parsing result dictionary with string values
-        """
-        try:
-            if url:
-                self.logger.info(f"🔄 Processing HTML from {url}: {len(html)} characters")
-            else:
-                self.logger.info(f"🔄 Processing HTML: {len(html)} characters")
-
-            # Use existing analyze_html method
-            result = await self.analyze_html(html=html, instructions=instructions, session_id=session_id, **kwargs)
-
-            # Convert to standardized string format for ParserManager
-            if result.success:
-                analysis_result = result.analysis_result
-                return HTMLParseResult(success="true", parsed_data=str(analysis_result.get("parsed_data", "")), markdown=str(analysis_result.get("markdown", "")), error_message="")
-            else:
-                return HTMLParseResult(success="false", parsed_data="", markdown="", error_message=result.error_message or "Analysis failed")
-
-        except Exception as e:
-            self.logger.error(f"❌ HTML parsing failed: {str(e)}")
-            return HTMLParseResult(success="false", parsed_data="", markdown="", error_message=str(e))
-
-    def get_cleaning_stats(self, original_html: str, cleaned_html: str) -> HTMLCleaningStats:
-        """Get cleaning statistics"""
-        return self.cleaner.get_cleaning_stats(original_html, cleaned_html)
-
-    def get_stats(self) -> HTMLAnalyzerStats:
-        """Get HTML analyzer statistics"""
-        return HTMLAnalyzerStats(cleaned_count=getattr(self.cleaner, "_cleaned_count", 0), total_reduction=getattr(self.cleaner, "_total_reduction", 0.0), websocket_enabled=self.websocket_analyzer is not None)
-
-    async def close(self):
-        """Close all resources"""
-        if self.websocket_analyzer:
-            await self.websocket_analyzer.close()
-        self.logger.info("🔌 HTML Analyzer closed")
-
-
-def create_html_analyzer(parser_id: str, websocket_url: Optional[str] = None, api_key: Optional[str] = None, cleaning_config: Optional[HTMLCleaningConfig] = None, **kwargs) -> HTMLAnalyzer:
-    """
-    Create HTML analyzer with configuration
-
-    Args:
-        parser_id: Parser identifier
-        websocket_url: WebSocket URL for LLM analysis (optional, auto-detected if not provided)
-        api_key: API key for authentication
-        cleaning_config: HTML cleaning configuration
-        **kwargs: Additional configuration options
-
-    Returns:
-        Configured HTMLAnalyzer instance
-    """
-    # Only pass websocket_url if explicitly provided, otherwise use auto-detection
-    config_kwargs = {"parser_id": parser_id, "api_key": api_key, "cleaning_config": cleaning_config or HTMLCleaningConfig(), **kwargs}
-    if websocket_url is not None:
-        config_kwargs["websocket_url"] = websocket_url
-    
-    config = HTMLAnalyzerConfig(**config_kwargs)
-
-    return HTMLAnalyzer(config)
-
-
-# Convenience functions
-async def quick_analyze_html(html: str, parser_id: str, instructions: Optional[str] = None, websocket_url: Optional[str] = None, **kwargs) -> HTMLAnalysisResult:
-    """
-    Quick HTML analysis convenience function
-
-    Args:
-        html: Raw HTML content
-        instructions: Analysis instructions
-        parser_id: Parser identifier
-        websocket_url: WebSocket URL for analysis (optional, auto-detected if not provided)
-        **kwargs: Additional options
-
-    Returns:
-        Analysis result
-    """
-    analyzer = create_html_analyzer(parser_id=parser_id, websocket_url=websocket_url, **kwargs)
-
-    try:
-        return await analyzer.analyze_html(html, instructions=instructions, **kwargs)
-    finally:
-        await analyzer.close()
-
-
-async def quick_clean_html(html: str, parser_id: str, **kwargs) -> Tuple[str, dict[str, str]]:
-    """
-    Quick HTML cleaning convenience function
-
-    Args:
-        html: Raw HTML content
-        parser_id: Parser identifier
-        **kwargs: Cleaning options
-
-    Returns:
-        Tuple of (cleaned_html, extracted_data)
-    """
-    analyzer = create_html_analyzer(parser_id=parser_id)
-
-    try:
-        return await analyzer.clean_html_only(html, **kwargs)
-    finally:
-        await analyzer.close()

unrealon_driver/html_analyzer/models.py

@@ -1,115 +0,0 @@
-"""
-HTML Analyzer Models - Pydantic v2 models for HTML analysis operations.
-
-Strict compliance with CRITICAL_REQUIREMENTS.md:
-- No Dict[str, Any] usage
-- Complete type annotations
-- Pydantic v2 models everywhere
-- Custom exception hierarchy
-- No try blocks in imports
-"""
-
-from typing import Optional
-from pydantic import BaseModel, Field, ConfigDict
-
-
-class HTMLAnalysisResult(BaseModel):
-    """Complete HTML analysis result with proper typing."""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    success: bool = Field(..., description="Analysis success status")
-    original_html_size: int = Field(..., ge=0, description="Original HTML size in characters")
-    cleaned_html: str = Field(..., description="Cleaned HTML content")
-    cleaned_html_size: int = Field(..., ge=0, description="Cleaned HTML size in characters")
-    extracted_data: dict[str, str] = Field(default_factory=dict, description="Extracted JavaScript data")
-    analysis_result: dict[str, str] = Field(default_factory=dict, description="LLM analysis result")
-    cleaning_stats: dict[str, float] = Field(default_factory=dict, description="Cleaning statistics")
-    error_message: str = Field(default="", description="Error message if failed")
-
-
-class HTMLParseResult(BaseModel):
-    """Standardized HTML parsing result for ParserManager."""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    success: str = Field(..., description="Success status as string (true/false)")
-    parsed_data: str = Field(..., description="Parsed data as string")
-    markdown: str = Field(..., description="Markdown representation")
-    error_message: str = Field(..., description="Error message if failed")
-
-
-class HTMLAnalyzerStats(BaseModel):
-    """HTML analyzer statistics."""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    cleaned_count: int = Field(default=0, ge=0, description="Number of HTML documents cleaned")
-    total_reduction: float = Field(default=0.0, ge=0.0, description="Total size reduction percentage")
-    websocket_enabled: bool = Field(..., description="Whether WebSocket analyzer is enabled")
-
-
-class HTMLCleaningRequest(BaseModel):
-    """Request model for HTML cleaning operations."""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    html: str = Field(..., min_length=1, description="HTML content to clean")
-    preserve_js_data: bool = Field(default=True, description="Whether to extract JavaScript data")
-    aggressive_cleaning: bool = Field(default=False, description="Whether to apply aggressive cleaning")
-
-
-class HTMLAnalysisRequest(BaseModel):
-    """Request model for HTML analysis operations."""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    html: str = Field(..., min_length=1, description="HTML content to analyze")
-    instructions: Optional[str] = Field(default=None, description="Analysis instructions for LLM")
-    session_id: Optional[str] = Field(default=None, description="Session identifier")
-    url: Optional[str] = Field(default=None, description="Source URL for logging")
-    clean_first: bool = Field(default=True, description="Whether to clean HTML before analysis")
-    preserve_js_data: bool = Field(default=True, description="Whether to extract JavaScript data")
-    aggressive_cleaning: bool = Field(default=False, description="Whether to apply aggressive cleaning")
-
-
-class HTMLParseRequest(BaseModel):
-    """Complete HTML parsing request model."""
-
-    model_config = ConfigDict(validate_assignment=True, extra="forbid")
-
-    html: str = Field(..., min_length=1, description="HTML content to parse")
-    url: Optional[str] = Field(default=None, description="Source URL for logging")
-    instructions: Optional[str] = Field(default=None, description="Analysis instructions")
-    session_id: Optional[str] = Field(default=None, description="Session identifier")
-    clean_first: bool = Field(default=True, description="Whether to clean HTML before analysis")
-    preserve_js_data: bool = Field(default=True, description="Whether to extract JavaScript data")
-    aggressive_cleaning: bool = Field(default=False, description="Whether to apply aggressive cleaning")
-
-
-class HTMLAnalyzerError(Exception):
-    """Base exception for HTML analyzer operations."""
-
-    def __init__(self, message: str, operation: str, details: Optional[dict[str, str]] = None):
-        self.message = message
-        self.operation = operation
-        self.details = details or {}
-        super().__init__(message)
-
-
-class HTMLCleaningError(HTMLAnalyzerError):
-    """Raised when HTML cleaning fails."""
-
-    pass
-
-
-class HTMLAnalysisError(HTMLAnalyzerError):
-    """Raised when HTML analysis fails."""
-
-    pass
-
-
-class WebSocketAnalysisError(HTMLAnalyzerError):
-    """Raised when WebSocket analysis fails."""
-
-    pass

unrealon_driver/html_analyzer/websocket_analyzer.py

@@ -1,157 +0,0 @@
-"""
-WebSocket HTML Analyzer - Handles HTML analysis via WebSocket communication.
-"""
-
-from typing import Optional, Dict, Any
-from unrealon_driver.smart_logging import create_smart_logger
-from unrealon_driver.websocket import websocket_manager, WebSocketConfig
-
-from .config import HTMLAnalyzerConfig
-
-
-class WebSocketHTMLAnalyzer:
-    """
-    WebSocket-based HTML analyzer that sends HTML to server for LLM analysis.
-    """
-    
-    def __init__(self, config: HTMLAnalyzerConfig):
-        self.config = config
-        self.logger = create_smart_logger(parser_id=config.parser_id)
-        
-        # Initialize WebSocket if configured
-        if config.websocket_url and config.enable_websocket_analysis:
-            self._websocket_config = WebSocketConfig(
-                url=config.websocket_url,
-                api_key=config.api_key,
-                parser_id=config.parser_id
-            )
-            self._websocket_initialized = False
-        else:
-            self._websocket_config = None
-            self._websocket_initialized = False
-    
-    async def analyze_html(
-        self, 
-        html: str, 
-        instructions: Optional[str] = None, 
-        session_id: Optional[str] = None,
-        **kwargs
-    ) -> Dict[str, str]:
-        """
-        Analyze HTML content via WebSocket
-        
-        Args:
-            html: HTML content to analyze
-            instructions: Analysis instructions
-            session_id: Session identifier
-            **kwargs: Additional parameters
-            
-        Returns:
-            Analysis result dictionary
-        """
-        if not self._websocket_config:
-            self.logger.warning("🔌 WebSocket not configured for HTML analysis")
-            return {
-                "success": "false",
-                "parsed_data": "",
-                "markdown": "",
-                "error_message": "WebSocket not configured"
-            }
-        
-        try:
-            # Ensure WebSocket connection
-            if not self._websocket_initialized:
-                await self._initialize_websocket()
-            
-            self.logger.info("🤖 Analyzing HTML with LLM via WebSocket...")
-            
-            # Prepare analysis request
-            analysis_request = {
-                "type": "html_analysis_request",
-                "parser_id": self.config.parser_id,
-                "session_id": session_id,
-                "html_content": html,
-                "instructions": instructions or "Extract and structure the data from this HTML",
-                "parse_type": "general",
-                "timeout": kwargs.get("timeout", self.config.default_timeout),
-                "metadata": kwargs.get("metadata", {})
-            }
-            
-            # Send request via WebSocket
-            if websocket_manager.connected:
-                response = await websocket_manager.send_request(
-                    analysis_request, 
-                    timeout=kwargs.get("timeout", self.config.default_timeout)
-                )
-                
-                if response and response.get("success"):
-                    self.logger.info("✅ HTML analysis completed successfully")
-                    return {
-                        "success": "true",
-                        "parsed_data": str(response.get("parsed_data", "")),
-                        "markdown": response.get("markdown", ""),
-                        "error_message": ""
-                    }
-                else:
-                    error_msg = response.get("error_message", "Analysis failed") if response else "No response"
-                    self.logger.error(f"❌ HTML analysis failed: {error_msg}")
-                    return {
-                        "success": "false", 
-                        "parsed_data": "",
-                        "markdown": "",
-                        "error_message": error_msg
-                    }
-            else:
-                self.logger.warning("🔌 WebSocket not connected for HTML analysis")
-                return {
-                    "success": "false",
-                    "parsed_data": "",
-                    "markdown": "",
-                    "error_message": "WebSocket not connected"
-                }
-                
-        except Exception as e:
-            self.logger.error(f"❌ HTML analysis failed: {str(e)}")
-            return {
-                "success": "false",
-                "parsed_data": "",
-                "markdown": "",
-                "error_message": str(e)
-            }
-    
-    async def _initialize_websocket(self) -> bool:
-        """Initialize WebSocket connection"""
-        if not self._websocket_config:
-            return False
-        
-        try:
-            success = await websocket_manager.initialize(self._websocket_config)
-            if success:
-                self._websocket_initialized = True
-                self.logger.info("🔌 WebSocket initialized for HTML analysis")
-            else:
-                self.logger.warning("🔌 WebSocket initialization failed")
-            return success
-        except Exception as e:
-            self.logger.error(f"❌ WebSocket initialization error: {e}")
-            return False
-    
-    async def close(self):
-        """Close WebSocket connection"""
-        if self._websocket_initialized:
-            await websocket_manager.disconnect()
-            self._websocket_initialized = False
-            self.logger.info("🔌 WebSocket connection closed")
-
-
-def create_websocket_analyzer(config: HTMLAnalyzerConfig) -> WebSocketHTMLAnalyzer:
-    """
-    Create WebSocket HTML analyzer
-    
-    Args:
-        config: HTML analyzer configuration
-        
-    Returns:
-        Configured WebSocketHTMLAnalyzer instance
-    """
-    return WebSocketHTMLAnalyzer(config)

unrealon_driver/models/__init__.py

@@ -1,31 +0,0 @@
-"""
-Models for unrealon_driver.
-
-Pydantic v2 models for type safety and validation.
-"""
-
-from .websocket import (
-    MessageType,
-    BridgeMessageType,
-    RegistrationMessage,
-    CommandMessage,
-    CommandResponseMessage,
-    StatusMessage,
-    HeartbeatMessage,
-    BridgeRegistrationPayload,
-    BridgeMessage,
-    BridgeRegistrationMessage
-)
-
-__all__ = [
-    "MessageType",
-    "BridgeMessageType",
-    "RegistrationMessage", 
-    "CommandMessage",
-    "CommandResponseMessage",
-    "StatusMessage",
-    "HeartbeatMessage",
-    "BridgeRegistrationPayload",
-    "BridgeMessage",
-    "BridgeRegistrationMessage"
-]