unrealon 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

unrealon_bridge/configs/bridge_config.py

@@ -0,0 +1,212 @@
+"""
+Bridge Server Configuration
+
+Pydantic2-based configuration management for the bridge server.
+Includes default test API keys for debugging.
+"""
+
+import os
+import yaml
+from pathlib import Path
+from typing import Optional, Dict, List, Any
+from pydantic import BaseModel, Field, ConfigDict, field_validator
+
+
+class RedisConfig(BaseModel):
+    """Redis connection configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True)
+
+    url: str = Field(default="redis://localhost:6379/0", description="Redis connection URL")
+    max_connections: int = Field(default=10, description="Maximum Redis connections")
+    connection_timeout: int = Field(default=5, description="Redis connection timeout in seconds")
+
+
+class WebSocketConfig(BaseModel):
+    """WebSocket server configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True)
+
+    host: str = Field(default="localhost", description="WebSocket server host")
+    port: int = Field(default=8002, description="WebSocket server port")
+    max_connections: int = Field(default=100, description="Maximum WebSocket connections")
+    ping_interval: int = Field(default=20, description="WebSocket ping interval in seconds")
+    ping_timeout: int = Field(default=10, description="WebSocket ping timeout in seconds")
+
+
+class SecurityConfig(BaseModel):
+    """Security and authentication configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True)
+
+    # Default test API keys for debugging
+    test_api_keys: List[str] = Field(default=["amazon_parser_key_123", "test_parser_key_456", "debug_key_789", "development_key_000"], description="Test API keys for debugging (DO NOT USE IN PRODUCTION)")
+
+    # Production API keys (loaded from environment or config)
+    api_keys: List[str] = Field(default_factory=list, description="Production API keys")
+
+    require_api_key: bool = Field(default=True, description="Whether API key is required for parser registration")
+
+    allow_test_keys: bool = Field(default=True, description="Allow test API keys (disable in production)")
+
+
+class RPCConfig(BaseModel):
+    """RPC configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True)
+
+    channel: str = Field(default="amazon_parser_rpc", description="RPC channel name")
+    timeout: int = Field(default=30, description="RPC call timeout in seconds")
+
+
+class PubSubConfig(BaseModel):
+    """PubSub configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True)
+
+    prefix: str = Field(default="amazon_parser", description="PubSub channel prefix")
+
+
+class LoggingConfig(BaseModel):
+    """Logging configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True)
+
+    level: str = Field(default="INFO", description="Logging level")
+    format: str = Field(default="%(asctime)s - %(name)s - %(levelname)s - %(message)s", description="Log message format")
+    enable_debug: bool = Field(default=False, description="Enable debug logging")
+
+
+class BridgeConfig(BaseModel):
+    """Main bridge server configuration."""
+
+    model_config = ConfigDict(str_strip_whitespace=True, validate_assignment=True, extra="forbid")
+
+    # Sub-configurations
+    redis: RedisConfig = Field(default_factory=RedisConfig)
+    websocket: WebSocketConfig = Field(default_factory=WebSocketConfig)
+    security: SecurityConfig = Field(default_factory=SecurityConfig)
+    rpc: RPCConfig = Field(default_factory=RPCConfig)
+    pubsub: PubSubConfig = Field(default_factory=PubSubConfig)
+    logging: LoggingConfig = Field(default_factory=LoggingConfig)
+
+    # Environment
+    environment: str = Field(default="development", description="Environment (development, testing, production)")
+
+    def get_all_api_keys(self) -> List[str]:
+        """Get all valid API keys (test + production)."""
+        all_keys = []
+
+        # Add production keys
+        all_keys.extend(self.security.api_keys)
+
+        # Add test keys if allowed
+        if self.security.allow_test_keys:
+            all_keys.extend(self.security.test_api_keys)
+
+        return all_keys
+
+    def is_valid_api_key(self, api_key: str) -> bool:
+        """Check if API key is valid."""
+        if not self.security.require_api_key:
+            return True
+
+        return api_key in self.get_all_api_keys()
+
+    def is_production(self) -> bool:
+        """Check if running in production."""
+        return self.environment.lower() == "production"
+
+    def is_development(self) -> bool:
+        """Check if running in development."""
+        return self.environment.lower() in ("development", "dev")
+
+    @field_validator("environment")
+    @classmethod
+    def validate_environment(cls, v):
+        """Validate environment value."""
+        valid_envs = ["development", "dev", "testing", "test", "production", "prod"]
+        if v.lower() not in valid_envs:
+            raise ValueError(f"Environment must be one of: {valid_envs}")
+        return v.lower()
+
+
+# Global config cache
+_config_cache: Optional[BridgeConfig] = None
+
+
+def get_config_path() -> Path:
+    """Get configuration file path."""
+    # Check environment variable first
+    config_path = os.getenv("BRIDGE_CONFIG_PATH")
+    if config_path:
+        return Path(config_path)
+
+    # Default path
+    return Path(__file__).parent / "bridge_config.yaml"
+
+
+def load_bridge_config(config_path: Optional[Path] = None) -> BridgeConfig:
+    """
+    Load bridge configuration from YAML file.
+
+    Args:
+        config_path: Optional path to config file
+
+    Returns:
+        BridgeConfig instance
+    """
+    global _config_cache
+
+    if _config_cache is not None:
+        return _config_cache
+
+    if config_path is None:
+        config_path = get_config_path()
+
+    if config_path.exists():
+        with open(config_path, "r", encoding="utf-8") as f:
+            config_data = yaml.safe_load(f)
+            _config_cache = BridgeConfig(**config_data)
+    else:
+        # Create default config
+        _config_cache = BridgeConfig()
+        save_bridge_config(_config_cache, config_path)
+
+    return _config_cache
+
+
+def save_bridge_config(config: BridgeConfig, config_path: Optional[Path] = None) -> Path:
+    """
+    Save bridge configuration to YAML file.
+
+    Args:
+        config: BridgeConfig instance
+        config_path: Optional path to save config
+
+    Returns:
+        Path where config was saved
+    """
+    if config_path is None:
+        config_path = get_config_path()
+
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(config_path, "w", encoding="utf-8") as f:
+        yaml.dump(config.model_dump(), f, default_flow_style=False, indent=2, allow_unicode=True)
+
+    return config_path
+
+
+def reload_config() -> BridgeConfig:
+    """Reload configuration from file, clearing cache."""
+    global _config_cache
+    _config_cache = None
+    return load_bridge_config()
+
+
+def create_sample_config() -> BridgeConfig:
+    """Create a sample configuration for testing."""
+    return BridgeConfig(
+        environment="development", websocket=WebSocketConfig(host="localhost", port=8002), security=SecurityConfig(test_api_keys=["amazon_parser_key_123", "test_parser_key_456", "debug_key_789"], allow_test_keys=True, require_api_key=True), logging=LoggingConfig(level="DEBUG", enable_debug=True)
+    )

unrealon_bridge/configs/bridge_config.yaml

@@ -0,0 +1,39 @@
+environment: development
+
+redis:
+  url: redis://localhost:6379/0
+  max_connections: 10
+  connection_timeout: 5
+
+websocket:
+  host: localhost
+  port: 8002
+  max_connections: 100
+  ping_interval: 20
+  ping_timeout: 10
+
+security:
+  # Test API keys for debugging (DO NOT USE IN PRODUCTION)
+  test_api_keys:
+    - amazon_parser_key_123
+    - test_parser_key_456
+    - debug_key_789
+    - development_key_000
+  
+  # Production API keys (add your keys here)
+  api_keys: []
+  
+  require_api_key: true
+  allow_test_keys: true
+
+rpc:
+  channel: amazon_parser_rpc
+  timeout: 30
+
+pubsub:
+  prefix: amazon_parser
+
+logging:
+  level: INFO
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  enable_debug: false

unrealon_bridge/models/__init__.py

@@ -0,0 +1,138 @@
+"""
+Bridge Parsers Models - Decomposed and strictly typed.
+
+All models follow CRITICAL_REQUIREMENTS.md:
+- NO Dict[str, Any] usage
+- Strict Pydantic v2 typing
+- NO raw JSON handling
+- Complete type annotations
+"""
+
+# Base models
+from .base import BaseParserModel, BaseRPCRequest, BaseRPCResponse
+
+# Core domain models
+from .parser import ParserInfo, ParserStats, ParserHealth, ParserSystemStats
+from .command import ParserCommand, CommandResult
+from .proxy import ProxyRequest, ProxyInfo
+from .session import ParserSession
+from .events import ParserEvent, WebhookConfig
+from .html_parser import (
+    HTMLParseRequest, HTMLParseResult, HTMLParseResponse,
+    HTMLParseRPCRequest, HTMLParseRPCResponse
+)
+from .logging import (
+    ParserLogEntry, ParserLogRequest, ParserLogResponse
+)
+from .scheduler import (
+    TaskStatus, TaskPriority, ScheduleType, TaskExecutionMode,
+    CronExpression, TaskParameters, TaskRetryConfig, ScheduledTask,
+    TaskQueue, TaskExecutionResult, ParserStatus,
+    SchedulerError, TaskValidationError, TaskExecutionError,
+    ParserUnavailableError, QueueError
+)
+
+# RPC Request models (strict typing)
+from .requests import (
+    ParserRegisterRequest,
+    ParserStatusRequest,
+    ParserListRequest,
+    ParserHealthRequest,
+    SessionStartRequest,
+    SessionEndRequest,
+    CommandExecuteRequest,
+    CommandCreateRequest,
+    CommandStatusRequest,
+    ProxyAllocateRequest,
+    ProxyReleaseRequest,
+    ProxyCheckRequest,
+)
+
+# RPC Response models (strict typing)
+from .responses import (
+    ParserRegisterResponse,
+    ParserStatusResponse,
+    ParserListResponse,
+    ParserHealthResponse,
+    SessionStartResponse,
+    SessionEndResponse,
+    CommandExecuteResponse,
+    CommandCreateResponse,
+    CommandStatusResponse,
+    ProxyAllocateResponse,
+    ProxyReleaseResponse,
+    ProxyCheckResponse,
+)
+
+__all__ = [
+    # Base models
+    "BaseParserModel",
+    "BaseRPCRequest",
+    "BaseRPCResponse",
+    # Core domain models
+    "ParserInfo",
+    "ParserStats",
+    "ParserHealth",
+    "ParserSystemStats",
+    "ParserCommand",
+    "CommandResult",
+    "ProxyRequest",
+    "ProxyInfo",
+    "ParserSession",
+    "ParserEvent",
+    "WebhookConfig",
+    # HTML Parser models
+    "HTMLParseRequest",
+    "HTMLParseResult", 
+    "HTMLParseResponse",
+    "HTMLParseRPCRequest",
+    "HTMLParseRPCResponse",
+    # Parser Logging models
+    "ParserLogEntry",
+    "ParserLogRequest", 
+    "ParserLogResponse",
+    # Scheduler models
+    "TaskStatus",
+    "TaskPriority", 
+    "ScheduleType",
+    "TaskExecutionMode",
+    "CronExpression",
+    "TaskParameters",
+    "TaskRetryConfig",
+    "ScheduledTask",
+    "TaskQueue", 
+    "TaskExecutionResult",
+    "ParserStatus",
+    # Scheduler exceptions
+    "SchedulerError",
+    "TaskValidationError",
+    "TaskExecutionError",
+    "ParserUnavailableError",
+    "QueueError",
+    # RPC Request models
+    "ParserRegisterRequest",
+    "ParserStatusRequest",
+    "ParserListRequest",
+    "ParserHealthRequest",
+    "SessionStartRequest",
+    "SessionEndRequest",
+    "CommandExecuteRequest",
+    "CommandCreateRequest",
+    "CommandStatusRequest",
+    "ProxyAllocateRequest",
+    "ProxyReleaseRequest",
+    "ProxyCheckRequest",
+    # RPC Response models
+    "ParserRegisterResponse",
+    "ParserStatusResponse",
+    "ParserListResponse",
+    "ParserHealthResponse",
+    "SessionStartResponse",
+    "SessionEndResponse",
+    "CommandExecuteResponse",
+    "CommandCreateResponse",
+    "CommandStatusResponse",
+    "ProxyAllocateResponse",
+    "ProxyReleaseResponse",
+    "ProxyCheckResponse",
+]

unrealon_bridge/models/base.py

@@ -0,0 +1,28 @@
+"""
+Base models for parser bridge operations.
+
+Provides common base classes and shared types for all parser models.
+"""
+
+from pydantic import BaseModel, Field, ConfigDict
+from typing import Optional
+
+
+class BaseParserModel(BaseModel):
+    """Base class for all parser models with strict validation."""
+
+    model_config = ConfigDict(extra="forbid", validate_assignment=True, str_strip_whitespace=True, validate_default=True)
+
+
+class BaseRPCRequest(BaseParserModel):
+    """Base class for all RPC requests."""
+
+    pass
+
+
+class BaseRPCResponse(BaseParserModel):
+    """Base class for all RPC responses."""
+
+    success: bool = Field(description="Whether the operation was successful")
+    message: Optional[str] = Field(default=None, description="Human-readable message")
+    error: Optional[str] = Field(default=None, description="Error message if success=False")

unrealon_bridge/models/command.py

@@ -0,0 +1,41 @@
+"""
+Command-related models.
+
+Contains models for command execution, tracking, and results.
+"""
+
+from typing import Optional, Dict, Literal, Any
+from datetime import datetime
+from pydantic import Field
+from typing_extensions import Annotated
+
+from .base import BaseParserModel
+
+
+class ParserCommand(BaseParserModel):
+    """Parser command definition and tracking."""
+
+    command_id: Annotated[str, Field(min_length=1, description="Unique command identifier")]
+    command_type: Annotated[str, Field(min_length=1, description="Type of command (scrape, parse, etc.)")]
+    parser_id: Annotated[str, Field(min_length=1, description="Parser ID that will execute the command")]
+    parameters: Dict[str, str] = Field(default_factory=dict, description="Command parameters")
+    status: Literal["pending", "running", "completed", "failed", "cancelled"] = "pending"
+    priority: Annotated[int, Field(ge=1, le=10)] = 5
+    timeout: Annotated[int, Field(gt=0, le=3600, description="Command timeout in seconds")] = 300
+    created_at: datetime = Field(default_factory=datetime.now)
+    started_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+    retry_count: Annotated[int, Field(ge=0)] = 0
+    max_retries: Annotated[int, Field(ge=0)] = 3
+
+
+class CommandResult(BaseParserModel):
+    """Result of command execution."""
+
+    command_id: Annotated[str, Field(min_length=1, description="Command ID")]
+    success: bool = Field(description="Whether command executed successfully")
+    result_data: Dict[str, Any] = Field(default_factory=dict, description="Command result data")
+    error_message: Optional[str] = None
+    execution_time: Annotated[float, Field(ge=0)] = 0.0
+    output_size: Annotated[int, Field(ge=0)] = 0
+    created_at: datetime = Field(default_factory=datetime.now)

unrealon_bridge/models/events.py

@@ -0,0 +1,40 @@
+"""
+Event and webhook-related models.
+
+Contains models for parser events, logging, and webhook configurations.
+"""
+
+from typing import Optional, Dict, List, Literal
+from datetime import datetime
+from pydantic import Field
+from typing_extensions import Annotated
+
+from .base import BaseParserModel
+
+
+class ParserEvent(BaseParserModel):
+    """Parser event for logging and monitoring."""
+
+    event_id: Annotated[str, Field(min_length=1, description="Unique event identifier")]
+    parser_id: Annotated[str, Field(min_length=1, description="Parser ID")]
+    event_type: Annotated[str, Field(min_length=1, description="Type of event")]
+    level: Literal["debug", "info", "warning", "error", "critical"] = "info"
+    message: Annotated[str, Field(min_length=1, description="Event message")]
+    data: Dict[str, str] = Field(default_factory=dict, description="Additional event data (string values only)")
+    command_id: Optional[str] = None
+    session_id: Optional[str] = None
+    timestamp: datetime = Field(default_factory=datetime.now)
+    source: Optional[str] = None
+
+
+class WebhookConfig(BaseParserModel):
+    """Webhook configuration for parser events."""
+
+    webhook_id: Annotated[str, Field(min_length=1, description="Unique webhook identifier")]
+    parser_id: Annotated[str, Field(min_length=1, description="Parser ID")]
+    url: Annotated[str, Field(min_length=1, description="Webhook URL")]
+    events: List[str] = Field(default_factory=list, description="Events to trigger webhook")
+    headers: Dict[str, str] = Field(default_factory=dict, description="Custom headers")
+    secret: Optional[str] = None
+    active: bool = True
+    created_at: datetime = Field(default_factory=datetime.now)

unrealon_bridge/models/html_parser.py

@@ -0,0 +1,79 @@
+"""
+HTML Parser models for LLM-based HTML parsing.
+
+Models for sending HTML content to Django backend for AI-powered parsing.
+"""
+
+from datetime import datetime
+from typing import Dict, List, Optional, Any
+from pydantic import BaseModel, Field
+
+from .base import BaseParserModel
+
+
+class HTMLParseRequest(BaseParserModel):
+    """Request for HTML parsing via AI/LLM."""
+    
+    html_content: str = Field(..., description="Raw HTML content to parse")
+    parser_id: str = Field(..., description="ID of the parser making the request")
+    url: Optional[str] = Field(None, description="Source URL of the HTML (for context)")
+    parse_type: str = Field("general", description="Type of parsing (product, listing, article, etc.)")
+    instructions: Optional[str] = Field(None, description="Additional parsing instructions")
+    timeout: int = Field(60, description="Timeout in seconds (default 60s for LLM processing)")
+    metadata: Dict[str, str] = Field(default_factory=dict, description="Additional metadata")
+
+
+class HTMLParseResult(BaseParserModel):
+    """Result of HTML parsing."""
+    
+    success: bool = Field(..., description="Whether parsing was successful")
+    parsed_data: Optional[Dict[str, Any]] = Field(None, description="Parsed JSON data")
+    markdown: Optional[str] = Field(None, description="Markdown instructions from Django parser")
+    error_message: Optional[str] = Field(None, description="Error message if parsing failed")
+
+
+class HTMLParseResponse(BaseParserModel):
+    """Response for HTML parsing request."""
+    
+    success: bool = Field(..., description="Whether the request was successful")
+    message: Optional[str] = Field(None, description="Response message")
+    error: Optional[str] = Field(None, description="Error message if failed")
+    result: Optional[HTMLParseResult] = Field(None, description="Parsing result")
+    request_id: Optional[str] = Field(None, description="Request ID for tracking")
+    processed_at: datetime = Field(default_factory=datetime.now, description="When the request was processed")
+
+
+
+
+
+
+
+
+# Request/Response models for RPC
+class HTMLParseRPCRequest(BaseParserModel):
+    """RPC request for HTML parsing."""
+    
+    html_content: str
+    parser_id: str
+    url: Optional[str] = None
+    parse_type: str = "general"
+    instructions: Optional[str] = None
+    timeout: int = 60
+    metadata: Dict[str, str] = Field(default_factory=dict)
+
+
+class HTMLParseRPCResponse(BaseParserModel):
+    """RPC response for HTML parsing."""
+    
+    success: bool
+    message: Optional[str] = None
+    error: Optional[str] = None
+    result: Optional[HTMLParseResult] = None
+    request_id: Optional[str] = None
+    processed_at: datetime = Field(default_factory=datetime.now)
+
+
+
+
+
+

unrealon_bridge/models/logging.py

@@ -0,0 +1,55 @@
+"""
+Parser logging models.
+
+Models for sending parser logs to Django via WebSocket/Redis.
+"""
+
+from datetime import datetime
+from typing import Dict, Optional
+from enum import Enum
+from pydantic import BaseModel, Field
+
+from .base import BaseParserModel
+
+
+class ParserLogLevel(str, Enum):
+    """Parser log levels."""
+    
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+class ParserLogEntry(BaseParserModel):
+    """Parser log entry to send to Django."""
+    
+    parser_id: str = Field(..., description="ID of the parser")
+    level: str = Field(..., description="Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)")
+    message: str = Field(..., description="Log message")
+    timestamp: datetime = Field(default_factory=datetime.now, description="When the log was created")
+    
+    # Optional context
+    session_id: Optional[str] = Field(None, description="Parser session ID")
+    command_id: Optional[str] = Field(None, description="Command ID if related to command")
+    url: Optional[str] = Field(None, description="URL being processed")
+    operation: Optional[str] = Field(None, description="Operation being performed")
+    
+    # Additional data
+    data: Dict[str, str] = Field(default_factory=dict, description="Additional log data")
+    error_details: Optional[str] = Field(None, description="Error details if error log")
+
+
+class ParserLogRequest(BaseParserModel):
+    """RPC request for sending parser log."""
+    
+    log_entry: ParserLogEntry
+
+
+class ParserLogResponse(BaseParserModel):
+    """RPC response for parser log."""
+    
+    success: bool = Field(..., description="Whether log was sent successfully")
+    message: Optional[str] = Field(None, description="Response message")
+    error: Optional[str] = Field(None, description="Error message if failed")