unrealon 1.0.8__py3-none-any.whl → 1.1.0__py3-none-any.whl

unrealon_bridge/client/events.py

@@ -0,0 +1,65 @@
+"""
+Event logging and heartbeat for Parser Bridge Client.
+"""
+
+import uuid
+from typing import Optional
+from datetime import datetime
+from unrealon_rpc.logging import get_logger
+
+from ..models import ParserEvent, ParserStats
+
+logger = get_logger(__name__)
+
+
+class EventsMixin:
+    """Mixin for event logging and heartbeat functionality."""
+
+    async def log_event(self, event_type: str, message: str, level: str = "info", data: Optional[dict[str, str]] = None, command_id: Optional[str] = None) -> None:
+        """
+        Log parser event.
+
+        Args:
+            event_type: Type of event
+            message: Event message
+            level: Log level
+            data: Additional event data (string values only)
+            command_id: Associated command ID
+        """
+        if not self.registered:
+            return
+
+        event = ParserEvent(
+            event_id=str(uuid.uuid4()),
+            parser_id=self.parser_id,
+            event_type=event_type,
+            level=level,
+            message=message,
+            data=data or {},
+            session_id=self.session_id,
+            command_id=command_id
+        )
+
+        # Send event via PubSub
+        await self.bridge_client.publish("parser_events", event.model_dump())
+
+    async def send_heartbeat(self, stats: Optional[ParserStats] = None) -> None:
+        """
+        Send parser heartbeat with optional stats.
+
+        Args:
+            stats: Parser statistics
+        """
+        if not self.registered:
+            return
+
+        heartbeat_data = {
+            "parser_id": self.parser_id, 
+            "timestamp": datetime.now().isoformat(), 
+            "session_id": self.session_id or ""
+        }
+
+        if stats:
+            heartbeat_data["stats"] = stats.model_dump()
+
+        await self.bridge_client.send_heartbeat("alive", heartbeat_data)

unrealon_bridge/client/health.py

@@ -0,0 +1,38 @@
+"""
+Health monitoring for Parser Bridge Client.
+"""
+
+from unrealon_rpc.logging import get_logger
+
+from ..models import (
+    ParserHealth, ParserHealthRequest, ParserHealthResponse
+)
+
+logger = get_logger(__name__)
+
+
+class HealthMixin:
+    """Mixin for health monitoring functionality."""
+
+    async def get_health(self) -> ParserHealth:
+        """
+        Get parser health information.
+
+        Returns:
+            Parser health status
+        """
+        self._ensure_registered()
+
+        request = ParserHealthRequest(parser_id=self.parser_id)
+        
+        result = await self.bridge_client.call_rpc(
+            method="parser.get_health", 
+            params=request.model_dump()
+        )
+
+        response = ParserHealthResponse.model_validate(result)
+        
+        if response.success and response.health:
+            return response.health
+        else:
+            raise RuntimeError(f"Health check failed: {response.error}")

unrealon_bridge/client/html_parser.py

@@ -0,0 +1,146 @@
+"""
+HTML Parser functionality for Parser Bridge Client.
+
+Provides methods for sending HTML content to Django for AI-powered parsing.
+"""
+
+from typing import Optional, Dict
+from unrealon_rpc.logging import get_logger
+
+from ..models import HTMLParseResult, HTMLParseRPCResponse, HTMLParseRPCRequest
+
+logger = get_logger(__name__)
+
+
+class HTMLParserMixin:
+    """Mixin for HTML parsing functionality."""
+
+    async def parse_html(
+        self,
+        html_content: str,
+        url: Optional[str] = None,
+        parse_type: str = "general",
+        instructions: Optional[str] = None,
+        timeout: int = 60,
+        metadata: Optional[Dict[str, str]] = None
+    ) -> HTMLParseResult:
+        """
+        Parse HTML content using AI/LLM via Django backend.
+
+        Args:
+            html_content: Raw HTML content to parse
+            url: Source URL of the HTML (for context)
+            parse_type: Type of parsing (product, listing, article, etc.)
+            instructions: Additional parsing instructions for the LLM
+            timeout: Timeout in seconds (default 60s for LLM processing)
+            metadata: Additional metadata
+
+        Returns:
+            HTMLParseResult with parsed data or error information
+
+        Example:
+            ```python
+            # Parse product page HTML
+            result = await client.parse_html(
+                html_content="<html>...</html>",
+                url="https://encar.com/car/123456",
+                parse_type="car_product",
+                instructions="Extract car details, price, and specifications"
+            )
+            
+            if result.success:
+                print(f"Parsed data: {result.parsed_data}")
+                print(f"Instructions: {result.markdown}")
+            else:
+                print(f"Parse failed: {result.error_message}")
+            ```
+        """
+        if not self.registered:
+            logger.warning("Cannot parse HTML - parser not registered")
+            return HTMLParseResult(
+                success=False,
+                error_message="Parser not registered"
+            )
+
+        try:
+            logger.info(f"Parsing HTML content: {len(html_content)} chars, type: {parse_type}")
+
+            request = HTMLParseRPCRequest(
+                html_content=html_content,
+                parser_id=self.parser_id,
+                url=url,
+                parse_type=parse_type,
+                instructions=instructions,
+                timeout=timeout,
+                metadata=metadata or {}
+            )
+
+            response = await self.bridge_client.call_rpc(
+                method="html_parser.parse",
+                params=request.model_dump(),
+                timeout=timeout + 5  # Add buffer for network/processing
+            )
+
+            rpc_response = HTMLParseRPCResponse.model_validate(response)
+            
+            if rpc_response.success and rpc_response.result:
+                logger.info(f"HTML parsing completed successfully")
+                return rpc_response.result
+            else:
+                error_msg = rpc_response.error or "Unknown parsing error"
+                logger.error(f"HTML parsing failed: {error_msg}")
+                return HTMLParseResult(
+                    success=False,
+                    error_message=error_msg
+                )
+
+        except Exception as e:
+            error_msg = f"HTML parsing request failed: {e}"
+            logger.error(error_msg)
+            return HTMLParseResult(
+                success=False,
+                error_message=error_msg
+            )
+
+    async def parse_html_with_retry(
+        self,
+        html_content: str,
+        max_retries: int = 3,
+        **kwargs
+    ) -> HTMLParseResult:
+        """
+        Parse HTML with automatic retry on failure.
+
+        Args:
+            html_content: Raw HTML content to parse
+            max_retries: Maximum number of retry attempts
+            **kwargs: Additional arguments passed to parse_html
+
+        Returns:
+            HTMLParseResult with parsed data or error information
+        """
+        last_error = None
+        
+        for attempt in range(max_retries):
+            try:
+                result = await self.parse_html(html_content, **kwargs)
+                
+                if result.success:
+                    if attempt > 0:
+                        logger.info(f"HTML parsing succeeded on attempt {attempt + 1}")
+                    return result
+                else:
+                    last_error = result.error_message
+                    if attempt < max_retries - 1:
+                        logger.warning(f"HTML parsing failed on attempt {attempt + 1}, retrying...")
+                    
+            except Exception as e:
+                last_error = str(e)
+                if attempt < max_retries - 1:
+                    logger.warning(f"HTML parsing error on attempt {attempt + 1}, retrying: {e}")
+
+        logger.error(f"HTML parsing failed after {max_retries} attempts")
+        return HTMLParseResult(
+            success=False,
+            error_message=f"Failed after {max_retries} attempts. Last error: {last_error}"
+        )

unrealon_bridge/client/logging.py

@@ -0,0 +1,139 @@
+"""
+Parser logging functionality for Parser Bridge Client.
+
+Provides methods for sending parser logs to Django via WebSocket/Redis.
+"""
+
+from typing import Optional, Dict
+from datetime import datetime
+from unrealon_rpc.logging import get_logger
+
+from ..models import ParserLogEntry, ParserLogResponse
+
+logger = get_logger(__name__)
+
+
+class LoggingMixin:
+    """Mixin for parser logging functionality."""
+
+    async def send_log(
+        self,
+        level: str,
+        message: str,
+        session_id: Optional[str] = None,
+        command_id: Optional[str] = None,
+        url: Optional[str] = None,
+        operation: Optional[str] = None,
+        data: Optional[Dict[str, str]] = None,
+        error_details: Optional[str] = None
+    ) -> bool:
+        """
+        Send log entry to Django via WebSocket/Redis.
+
+        Args:
+            level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+            message: Log message
+            session_id: Parser session ID (optional)
+            command_id: Command ID if related to command (optional)
+            url: URL being processed (optional)
+            operation: Operation being performed (optional)
+            data: Additional log data (optional)
+            error_details: Error details if error log (optional)
+
+        Returns:
+            True if log was sent successfully, False otherwise
+
+        Example:
+            ```python
+            # Send info log
+            await client.send_log("INFO", "Started parsing product page")
+            
+            # Send error log with details
+            await client.send_log(
+                "ERROR", 
+                "Failed to parse product price",
+                url="https://example.com/product/123",
+                operation="price_extraction",
+                error_details="Price element not found in DOM"
+            )
+            
+            # Send log with session context
+            await client.send_log(
+                "DEBUG",
+                "Processing page 5 of search results", 
+                session_id=self.session_id,
+                operation="pagination"
+            )
+            ```
+        """
+        if not self.registered:
+            logger.warning("Cannot send log - parser not registered")
+            return False
+
+        try:
+            log_entry = ParserLogEntry(
+                parser_id=self.parser_id,
+                level=level.upper(),
+                message=message,
+                session_id=session_id,
+                command_id=command_id,
+                url=url,
+                operation=operation,
+                data=data or {},
+                error_details=error_details
+            )
+            
+            response_dict = await self.bridge_client.call_rpc(
+                method="parser.log",
+                params=log_entry.model_dump()
+            )
+
+            response = ParserLogResponse.model_validate(response_dict)
+            
+            if response.success:
+                logger.debug(f"Log sent to Django: {level} - {message}")
+                return True
+            else:
+                logger.error(f"Failed to send log to Django: {response.error}")
+                return False
+
+        except Exception as e:
+            logger.error(f"Log sending failed: {e}")
+            return False
+
+    async def log_debug(self, message: str, **kwargs) -> bool:
+        """Send DEBUG level log."""
+        return await self.send_log("DEBUG", message, **kwargs)
+
+    async def log_info(self, message: str, **kwargs) -> bool:
+        """Send INFO level log."""
+        return await self.send_log("INFO", message, **kwargs)
+
+    async def log_warning(self, message: str, **kwargs) -> bool:
+        """Send WARNING level log."""
+        return await self.send_log("WARNING", message, **kwargs)
+
+    async def log_error(self, message: str, **kwargs) -> bool:
+        """Send ERROR level log."""
+        return await self.send_log("ERROR", message, **kwargs)
+
+    async def log_critical(self, message: str, **kwargs) -> bool:
+        """Send CRITICAL level log."""
+        return await self.send_log("CRITICAL", message, **kwargs)
+
+    async def log_operation_start(self, operation: str, **kwargs) -> bool:
+        """Log the start of an operation."""
+        return await self.log_info(f"Started {operation}", operation=operation, **kwargs)
+
+    async def log_operation_end(self, operation: str, **kwargs) -> bool:
+        """Log the end of an operation."""
+        return await self.log_info(f"Completed {operation}", operation=operation, **kwargs)
+
+    async def log_operation_error(self, operation: str, error: str, **kwargs) -> bool:
+        """Log an operation error."""
+        return await self.log_error(
+            f"Failed {operation}: {error}", 
+            operation=operation, 
+            error_details=error,
+            **kwargs
+        )

unrealon_bridge/client/proxy.py

@@ -0,0 +1,70 @@
+"""
+Proxy management for Parser Bridge Client.
+"""
+
+from typing import Optional
+from unrealon_rpc.logging import get_logger
+
+from ..models import (
+    ProxyInfo, ProxyAllocateRequest, ProxyAllocateResponse,
+    ProxyReleaseRequest, ProxyReleaseResponse
+)
+
+logger = get_logger(__name__)
+
+
+class ProxyMixin:
+    """Mixin for proxy management functionality."""
+
+    async def request_proxy(self, proxy_type: str = "http", location: Optional[str] = None) -> ProxyInfo:
+        """
+        Request proxy allocation.
+
+        Args:
+            proxy_type: Type of proxy needed
+            location: Preferred proxy location
+
+        Returns:
+            Allocated proxy information
+        """
+        self._ensure_registered()
+
+        request = ProxyAllocateRequest(
+            parser_id=self.parser_id,
+            proxy_type=proxy_type,
+            location=location
+        )
+
+        result = await self.bridge_client.call_rpc(
+            method="proxy.allocate", 
+            params=request.model_dump()
+        )
+
+        response = ProxyAllocateResponse.model_validate(result)
+        
+        if response.success and response.proxy:
+            logger.info(f"Proxy allocated: {response.proxy.proxy_id} for {self.parser_id}")
+            return response.proxy
+        else:
+            raise RuntimeError(f"Proxy allocation failed: {response.error}")
+
+    async def release_proxy(self, proxy_id: str) -> None:
+        """
+        Release proxy allocation.
+
+        Args:
+            proxy_id: Proxy ID to release
+        """
+        request = ProxyReleaseRequest(proxy_id=proxy_id)
+        
+        result = await self.bridge_client.call_rpc(
+            method="proxy.release", 
+            params=request.model_dump()
+        )
+
+        response = ProxyReleaseResponse.model_validate(result)
+        
+        if response.success:
+            logger.info(f"Proxy released: {proxy_id}")
+        else:
+            logger.error(f"Proxy release failed: {response.error}")