unrealon 1.1.1__py3-none-any.whl → 1.1.4__py3-none-any.whl

unrealon_bridge/server/base.py

@@ -1,171 +0,0 @@
-"""
-Base Parser Bridge Server.
-
-Core server functionality and state management.
-"""
-
-import asyncio
-from typing import Dict, Callable, Optional, List
-from unrealon_rpc.bridge import WebSocketBridge
-from unrealon_rpc.rpc import RPCServer
-from unrealon_rpc.pubsub import PubSubSubscriber
-from unrealon_rpc.logging import get_logger
-
-from ..models import (
-    ParserInfo, ParserCommand, ParserSession, ParserEvent, ParserSystemStats
-)
-
-logger = get_logger(__name__)
-
-
-class ParserBridgeServerBase:
-    """
-    Base parser bridge server with core functionality.
-    
-    Manages server state and provides foundation for specialized handlers.
-    """
-
-    def __init__(self, redis_url: str = "redis://localhost:6379/0", rpc_channel: str = "parser_rpc", pubsub_prefix: str = "parser", **kwargs):
-        """
-        Initialize parser bridge server.
-
-        Args:
-            redis_url: Redis connection URL
-            rpc_channel: RPC channel name
-            pubsub_prefix: PubSub channel prefix
-            **kwargs: Additional arguments for WebSocketBridge
-        """
-        self.redis_url = redis_url
-        self.rpc_channel = rpc_channel
-        self.pubsub_prefix = pubsub_prefix
-
-        # Initialize bridge components
-        self.bridge = WebSocketBridge(
-            redis_url=redis_url,
-            rpc_channel=rpc_channel,
-            pubsub_prefix=pubsub_prefix,
-            **kwargs
-        )
-
-        # Initialize RPC and PubSub
-        self.parser_rpc = RPCServer(channel=rpc_channel, redis_url=redis_url)
-        self.parser_pubsub = PubSubSubscriber(channel_prefix=pubsub_prefix, redis_url=redis_url)
-
-        # Server state
-        self.parsers: Dict[str, ParserInfo] = {}
-        self.sessions: Dict[str, ParserSession] = {}
-        self.commands: Dict[str, ParserCommand] = {}
-        self.proxies: Dict[str, any] = {}  # Will be typed properly in proxy handler
-        
-        # Mapping between parser_id and client_id for WebSocket forwarding
-        self.parser_to_client: Dict[str, str] = {}
-
-        # Custom command handlers
-        self.command_handlers: Dict[str, Callable] = {}
-
-        # Background tasks
-        self._tasks: List[asyncio.Task] = []
-        self._running = False
-    
-    def get_client_by_parser_id(self, parser_id: str):
-        """Get WebSocket client by parser_id."""
-        client_id = self.parser_to_client.get(parser_id)
-        if client_id and client_id in self.bridge.connections:
-            return self.bridge.connections[client_id]
-        return None
-
-    async def start(self) -> None:
-        """Start the parser bridge server."""
-        if self._running:
-            return
-
-        logger.info("Starting Parser Bridge Server...")
-
-        # Start bridge components
-        await self.bridge.start()
-        await self.parser_rpc.start()
-        await self.parser_pubsub.start()
-
-        # Start background tasks
-        self._tasks.append(asyncio.create_task(self._pubsub_listener()))
-
-        self._running = True
-        logger.info("Parser Bridge Server started")
-
-    async def stop(self) -> None:
-        """Stop the parser bridge server."""
-        if not self._running:
-            return
-
-        logger.info("Stopping Parser Bridge Server...")
-
-        # Cancel background tasks
-        for task in self._tasks:
-            task.cancel()
-        
-        # Wait for tasks to complete
-        if self._tasks:
-            await asyncio.gather(*self._tasks, return_exceptions=True)
-        self._tasks.clear()
-
-        # Stop bridge components
-        await self.parser_pubsub.stop()
-        await self.parser_rpc.stop()
-        await self.bridge.stop()
-
-        self._running = False
-        logger.info("Parser Bridge Server stopped")
-
-    async def _pubsub_listener(self) -> None:
-        """Listen to parser events via PubSub."""
-        try:
-            # Register handler for parser events
-            @self.parser_pubsub.subscribe("parser_events")
-            async def event_handler(payload: dict):
-                try:
-                    event = ParserEvent.model_validate(payload)
-                    await self._handle_parser_event(event)
-                except Exception as e:
-                    logger.error(f"Error processing parser event: {e}")
-            
-            # Start the subscriber (this will run indefinitely)
-            await self.parser_pubsub.start()
-            
-        except asyncio.CancelledError:
-            logger.info("PubSub listener cancelled")
-        except Exception as e:
-            logger.error(f"PubSub listener error: {e}")
-
-    async def _handle_parser_event(self, event: ParserEvent) -> None:
-        """
-        Handle parser event from PubSub.
-
-        Args:
-            event: Parser event to handle
-        """
-        logger.debug(f"Parser event: {event.event_type} from {event.parser_id}")
-
-    def register_command_handler(self, command_type: str, handler: Callable) -> None:
-        """
-        Register custom command handler.
-
-        Args:
-            command_type: Type of command to handle
-            handler: Async handler function
-        """
-        self.command_handlers[command_type] = handler
-        logger.info(f"Registered command handler for: {command_type}")
-
-    def get_parser_stats(self) -> ParserSystemStats:
-        """Get parser statistics."""
-        parser_types = {}
-        for parser in self.parsers.values():
-            parser_types[parser.parser_type] = parser_types.get(parser.parser_type, 0) + 1
-        
-        return ParserSystemStats(
-            total_parsers=len(self.parsers),
-            active_sessions=len([s for s in self.sessions.values() if s.status == "active"]),
-            total_commands=len(self.commands),
-            allocated_proxies=len(self.proxies),
-            parser_types=parser_types
-        )

unrealon_bridge/server/handlers/__init__.py

@@ -1,23 +0,0 @@
-"""
-RPC Handlers for Parser Bridge Server.
-
-Modular handlers for different types of RPC operations.
-"""
-
-from .parser import ParserHandlers
-from .session import SessionHandlers
-from .command import CommandHandlers
-from .proxy import ProxyHandlers
-from .html_parser import HTMLParserHandlers
-from .logging import LoggingHandlers
-from .scheduler import SchedulerHandlers
-
-__all__ = [
-    "ParserHandlers",
-    "SessionHandlers", 
-    "CommandHandlers",
-    "ProxyHandlers",
-    "HTMLParserHandlers",
-    "LoggingHandlers",
-    "SchedulerHandlers"
-]

unrealon_bridge/server/handlers/command.py

@@ -1,110 +0,0 @@
-"""
-Command-related RPC handlers.
-"""
-
-import uuid
-from unrealon_rpc.logging import get_logger
-
-from ...models import (
-    ParserCommand, CommandResult,
-    CommandExecuteRequest, CommandExecuteResponse,
-    CommandCreateRequest, CommandCreateResponse,
-    CommandStatusRequest, CommandStatusResponse
-)
-
-logger = get_logger(__name__)
-
-
-class CommandHandlers:
-    """Handlers for command-related RPC operations."""
-
-    async def handle_command_execute(self, parser_id: str, command_type: str, parameters: dict, timeout: int = 30) -> dict:
-        """Handle command execution."""
-        try:
-            # Create request object for validation
-            request = CommandExecuteRequest(
-                parser_id=parser_id,
-                command_type=command_type,
-                parameters=parameters,
-                timeout=timeout
-            )
-            
-            command = ParserCommand(
-                command_id=str(uuid.uuid4()),
-                command_type=request.command_type,
-                parser_id=request.parser_id,
-                parameters=request.parameters,
-                timeout=request.timeout
-            )
-            self.commands[command.command_id] = command
-
-            # Forward command to daemon via WebSocket
-            daemon_client = self.get_client_by_parser_id(parser_id)
-            if daemon_client:
-                logger.info(f"📤 Forwarding command {command.command_type} to daemon {parser_id}")
-                # Send command via WebSocket
-                command_message = {
-                    "message_type": "command",
-                    "command_id": command.command_id,
-                    "command_type": command.command_type,
-                    "parameters": command.parameters,
-                    "parser_id": parser_id
-                }
-                await daemon_client.send_message(command_message)
-                # For now, return mock response - daemon should respond via WebSocket later
-                result_data = {
-                    "command_type": command.command_type,
-                    "status": "forwarded_to_daemon",
-                    "parser_id": parser_id
-                }
-            else:
-                logger.warning(f"⚠️ No daemon found for parser {parser_id}")
-                # Fallback to local handlers
-                handler = self.command_handlers.get(command.command_type)
-                if handler:
-                    logger.info(f"🔧 Using local handler for {command.command_type}")
-                    result_data = await handler(command)
-                else:
-                    logger.warning(f"⚠️ No handler found for {command.command_type}")
-                    result_data = {"error": f"No daemon connected for parser {parser_id}"}
-
-            result = CommandResult(
-                command_id=command.command_id,
-                success=True,
-                result_data=result_data,
-                execution_time=0.5
-            )
-
-            logger.info(f"Command executed: {command.command_id} ({command.command_type})")
-
-            response = CommandExecuteResponse(success=True, result=result)
-            return response.model_dump(mode='json')
-
-        except Exception as e:
-            logger.error(f"Command execution failed: {e}")
-            response = CommandExecuteResponse(success=False, error=str(e))
-            return response.model_dump(mode='json')
-
-    async def handle_command_create(self, request: CommandCreateRequest) -> CommandCreateResponse:
-        """Handle command creation."""
-        try:
-            command = ParserCommand(
-                command_id=str(uuid.uuid4()),
-                command_type=request.command_type,
-                parser_id=request.parser_id,
-                parameters=request.parameters
-            )
-            self.commands[command.command_id] = command
-
-            return CommandCreateResponse(success=True, command=command)
-        except Exception as e:
-            return CommandCreateResponse(success=False, error=str(e))
-
-    async def handle_command_get_status(self, request: CommandStatusRequest) -> CommandStatusResponse:
-        """Handle command status request."""
-        command = self.commands.get(request.command_id)
-
-        if not command:
-            return CommandStatusResponse(success=False, error=f"Command {request.command_id} not found")
-
-        return CommandStatusResponse(success=True, command=command)

unrealon_bridge/server/handlers/html_parser.py

@@ -1,139 +0,0 @@
-"""
-HTML Parser RPC handlers.
-
-Clean implementation following CRITICAL_REQUIREMENTS.md:
-- No inline imports
-- Strict Pydantic v2 usage
-- Complete type annotations
-- No Dict[str, Any] usage
-"""
-
-import asyncio
-import random
-import uuid
-from datetime import datetime
-from typing import Optional
-
-from unrealon_rpc.logging import get_logger
-
-from ...models import HTMLParseRPCRequest, HTMLParseRPCResponse, HTMLParseResult
-
-logger = get_logger(__name__)
-
-
-class HTMLParserHandlers:
-    """Handlers for HTML parser RPC operations."""
-
-    def __init__(self) -> None:
-        """Initialize HTML parser handlers."""
-        pass
-
-    async def handle_html_parse(self, html_content: str, parser_id: str, url: Optional[str] = None, parse_type: str = "general", instructions: Optional[str] = None, timeout: int = 60, metadata: Optional[dict] = None) -> dict:
-        """
-        Handle HTML parsing request.
-
-        Forwards HTML content to Django backend for AI/LLM processing.
-        Django will parse HTML and return JSON + markdown instructions.
-
-        Args:
-            html_content: Raw HTML content to parse
-            parser_id: ID of the parser making the request
-            url: Source URL of the HTML (optional)
-            parse_type: Type of parsing (product, listing, article, etc.)
-            instructions: Additional parsing instructions (optional)
-            timeout: Timeout in seconds (default 60s for LLM processing)
-            metadata: Additional metadata (optional)
-
-        Returns:
-            HTMLParseRPCResponse as dict with success, result, request_id
-        """
-        try:
-            # Create and validate request object
-            request = HTMLParseRPCRequest(html_content=html_content, parser_id=parser_id, url=url, parse_type=parse_type, instructions=instructions, timeout=timeout, metadata=metadata or {})
-
-            request_id = str(uuid.uuid4())
-
-            logger.info(f"HTML parse request from parser {parser_id}: " f"{len(html_content)} chars, type: {parse_type}")
-
-            # TODO: In production, make RPC call to Django backend
-            # For now, simulate the response
-            result = await self._simulate_html_parsing(request)
-
-            response = HTMLParseRPCResponse(success=True, result=result, request_id=request_id, message="HTML parsed successfully" if result.success else "HTML parsing failed")
-
-            return response.model_dump(mode="json")
-
-        except Exception as e:
-            logger.error(f"HTML parsing failed for parser {parser_id}: {e}")
-
-            response = HTMLParseRPCResponse(success=False, error=str(e), message="HTML parsing request failed")
-
-            return response.model_dump(mode="json")
-
-    async def _simulate_html_parsing(self, request: HTMLParseRPCRequest) -> HTMLParseResult:
-        """
-        Simulate HTML parsing for demo purposes.
-
-        In production, this would make an RPC call to Django backend which would:
-        1. Receive the HTML content
-        2. Use LLM (GPT-4, Claude, etc.) to parse the HTML
-        3. Return structured JSON data + markdown instructions
-
-        Args:
-            request: Validated HTML parse request
-
-        Returns:
-            HTMLParseResult with success/failure and data/markdown
-        """
-        # Simulate brief processing delay
-        await asyncio.sleep(0.1)
-
-        # Simulate success/failure (85% success rate)
-        success_rate = 0.85
-        is_successful = random.random() < success_rate
-
-        if is_successful:
-            return self._create_success_result(request)
-        else:
-            return HTMLParseResult(success=False, error_message="Failed to extract structured data from HTML")
-
-    def _create_success_result(self, request: HTMLParseRPCRequest) -> HTMLParseResult:
-        """Create successful parsing result with sample data."""
-        # Sample parsed data
-        parsed_data = {
-            "title": "Sample Product Title",
-            "price": "29,900,000",
-            "description": "Sample product description extracted from HTML",
-            "specifications": {"year": "2020", "mileage": "45,000 km", "fuel": "Gasoline"},
-            "images": ["https://example.com/image1.jpg", "https://example.com/image2.jpg"],
-        }
-        # Make parsed_data with pydantic model
-
-        # Generate markdown instructions
-        markdown_instructions = f"""# HTML Parsing Results
-
-## Extracted Data
-Successfully parsed {request.parse_type} content from the provided HTML.
-
-### Key Findings:
-- **Title**: {parsed_data.get('title', 'N/A')}
-- **Price**: {parsed_data.get('price', 'N/A')}
-- **Content Size**: {len(request.html_content)} characters
-
-### Parsing Notes:
-- Applied {request.parse_type} parsing rules
-- Processed HTML structure successfully
-- Extracted all required fields
-
-### Recommendations:
-- Data quality appears good
-- Consider validating price format
-- Check for additional product images
-
-### Next Steps:
-1. Validate extracted data against business rules
-2. Store in appropriate database tables
-3. Process for further analysis
-"""
-
-        return HTMLParseResult(success=True, parsed_data=parsed_data, markdown=markdown_instructions)

unrealon_bridge/server/handlers/logging.py

@@ -1,95 +0,0 @@
-"""
-Parser logging RPC handlers.
-
-Handles parser log entries sent from parsers to Django.
-"""
-
-from typing import Optional
-from unrealon_rpc.logging import get_logger
-
-from ...models import ParserLogEntry, ParserLogRequest, ParserLogResponse
-
-logger = get_logger(__name__)
-
-
-class LoggingHandlers:
-    """Handlers for parser logging RPC operations."""
-
-    def __init__(self) -> None:
-        """Initialize logging handlers."""
-        pass
-
-    async def handle_parser_log(
-        self,
-        parser_id: str,
-        level: str,
-        message: str,
-        session_id: Optional[str] = None,
-        command_id: Optional[str] = None,
-        url: Optional[str] = None,
-        operation: Optional[str] = None,
-        data: Optional[dict] = None,
-        error_details: Optional[str] = None
-    ) -> dict:
-        """
-        Handle parser log entry.
-        
-        Receives log from parser and forwards to Django for storage/processing.
-        
-        Args:
-            parser_id: ID of the parser sending the log
-            level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
-            message: Log message
-            session_id: Parser session ID (optional)
-            command_id: Command ID if related to command (optional)
-            url: URL being processed (optional)
-            operation: Operation being performed (optional)
-            data: Additional log data (optional)
-            error_details: Error details if error log (optional)
-            
-        Returns:
-            ParserLogResponse as dict with success status
-        """
-        try:
-            # Create and validate log entry
-            log_entry = ParserLogEntry(
-                parser_id=parser_id,
-                level=level,
-                message=message,
-                session_id=session_id,
-                command_id=command_id,
-                url=url,
-                operation=operation,
-                data=data or {},
-                error_details=error_details
-            )
-            
-            # Log locally for debugging
-            local_logger_method = getattr(logger, level.lower(), logger.info)
-            local_logger_method(
-                f"Parser {parser_id} log: {message}",
-                component="parser_log",
-                operation=operation
-            )
-            
-            # Django will receive this RPC call via Redis and handle the log
-            
-            response = ParserLogResponse(
-                success=True,
-                message="Log entry received and forwarded to Django"
-            )
-            
-            return response.model_dump(mode='json')
-            
-        except Exception as e:
-            logger.error(f"Failed to handle parser log from {parser_id}: {e}")
-            
-            response = ParserLogResponse(
-                success=False,
-                error=str(e),
-                message="Failed to process log entry"
-            )
-            
-            return response.model_dump(mode='json')
-
-

unrealon_bridge/server/handlers/parser.py

@@ -1,95 +0,0 @@
-"""
-Parser-related RPC handlers.
-"""
-
-from unrealon_rpc.logging import get_logger
-from unrealon_bridge.configs import load_bridge_config
-from unrealon_bridge.models import ParserInfo, ParserHealth, ParserSystemStats, ParserRegisterRequest, ParserRegisterResponse, ParserStatusRequest, ParserStatusResponse, ParserListRequest, ParserListResponse, ParserHealthRequest, ParserHealthResponse
-
-logger = get_logger(__name__)
-
-
-class ParserHandlers:
-    """Handlers for parser-related RPC operations."""
-
-    async def handle_parser_register(self, parser_id: str, parser_type: str, version: str, capabilities: list, metadata: dict = None, api_key: str = None) -> dict:
-        """Handle parser registration."""
-        try:
-            # Load bridge configuration
-            config = load_bridge_config()
-
-            # Check if API key is required
-            if config.security.require_api_key:
-                if not api_key:
-                    response = ParserRegisterResponse(success=False, error="API key is required")
-                    return response.model_dump(mode="json")
-
-                # Validate API key
-                if not config.is_valid_api_key(api_key):
-                    logger.warning(f"Invalid API key attempted: {api_key[:8] if api_key else 'None'}...")
-                    response = ParserRegisterResponse(success=False, error="Invalid API key")
-                    return response.model_dump(mode="json")
-
-            # Create request object for validation
-            request = ParserRegisterRequest(parser_id=parser_id, parser_type=parser_type, version=version, capabilities=capabilities, metadata=metadata)
-
-            parser_info = ParserInfo(parser_id=request.parser_id, parser_type=request.parser_type, version=request.version, capabilities=request.capabilities, metadata=request.metadata or {})
-            self.parsers[parser_info.parser_id] = parser_info
-            
-            # Find and map the most recent WebSocket client (daemon usually connects then registers immediately)
-            if self.bridge.connections:
-                # Get the most recently connected client
-                latest_client_id = max(self.bridge.connections.keys(), 
-                                     key=lambda cid: self.bridge.connections[cid].client_info.connected_at)
-                self.parser_to_client[parser_info.parser_id] = latest_client_id
-                logger.info(f"🔗 Mapped parser {parser_info.parser_id} to client {latest_client_id}")
-            else:
-                logger.warning(f"⚠️ No WebSocket clients connected during parser registration")
-
-            # Log successful registration
-            api_key_display = api_key[:8] + "..." if api_key else "None"
-            logger.info(f"Parser registered: {parser_info.parser_id} ({parser_info.parser_type}) with API key: {api_key_display}")
-
-            # Log test key usage in development
-            if config.is_development() and api_key in config.security.test_api_keys:
-                logger.info(f"🧪 Using test API key for development: {api_key}")
-
-            response = ParserRegisterResponse(success=True, parser_id=parser_info.parser_id, message="Parser registered successfully")
-            return response.model_dump(mode="json")
-        except Exception as e:
-            logger.error(f"Parser registration failed: {e}")
-            response = ParserRegisterResponse(success=False, error=str(e))
-            return response.model_dump(mode="json")
-
-    async def handle_parser_get_status(self, parser_id: str) -> dict:
-        """Handle parser status request."""
-        parser_info = self.parsers.get(parser_id)
-
-        if not parser_info:
-            response = ParserStatusResponse(success=False, error=f"Parser {parser_id} not found")
-            return response.model_dump(mode="json")
-
-        response = ParserStatusResponse(success=True, parser=parser_info)
-        return response.model_dump(mode="json")
-
-    async def handle_parser_list(self, parser_type: str = None) -> dict:
-        """Handle parser list request."""
-        parsers = list(self.parsers.values())
-
-        if parser_type:
-            parsers = [p for p in parsers if p.parser_type == parser_type]
-
-        response = ParserListResponse(success=True, parsers=parsers, total=len(parsers))
-        return response.model_dump(mode="json")
-
-    async def handle_parser_get_health(self, parser_id: str) -> dict:
-        """Handle parser health check."""
-        if parser_id not in self.parsers:
-            response = ParserHealthResponse(success=False, error="Parser not found")
-            return response.model_dump(mode="json")
-
-        # Health check implementation should be provided by external service
-        health = ParserHealth(parser_id=parser_id, status="healthy", response_time=0.1, memory_usage=50.0, cpu_usage=25.0, active_connections=1, queue_size=0)
-
-        response = ParserHealthResponse(success=True, health=health)
-        return response.model_dump(mode="json")