unrealon 1.1.1__py3-none-any.whl → 1.1.4__py3-none-any.whl

unrealon_bridge/configs/bridge_config.py

@@ -1,212 +0,0 @@
-"""
-Bridge Server Configuration
-
-Pydantic2-based configuration management for the bridge server.
-Includes default test API keys for debugging.
-"""
-
-import os
-import yaml
-from pathlib import Path
-from typing import Optional, Dict, List, Any
-from pydantic import BaseModel, Field, ConfigDict, field_validator
-
-
-class RedisConfig(BaseModel):
-    """Redis connection configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True)
-
-    url: str = Field(default="redis://localhost:6379/0", description="Redis connection URL")
-    max_connections: int = Field(default=10, description="Maximum Redis connections")
-    connection_timeout: int = Field(default=5, description="Redis connection timeout in seconds")
-
-
-class WebSocketConfig(BaseModel):
-    """WebSocket server configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True)
-
-    host: str = Field(default="localhost", description="WebSocket server host")
-    port: int = Field(default=8002, description="WebSocket server port")
-    max_connections: int = Field(default=100, description="Maximum WebSocket connections")
-    ping_interval: int = Field(default=20, description="WebSocket ping interval in seconds")
-    ping_timeout: int = Field(default=10, description="WebSocket ping timeout in seconds")
-
-
-class SecurityConfig(BaseModel):
-    """Security and authentication configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True)
-
-    # Default test API keys for debugging
-    test_api_keys: List[str] = Field(default=["amazon_parser_key_123", "test_parser_key_456", "debug_key_789", "development_key_000"], description="Test API keys for debugging (DO NOT USE IN PRODUCTION)")
-
-    # Production API keys (loaded from environment or config)
-    api_keys: List[str] = Field(default_factory=list, description="Production API keys")
-
-    require_api_key: bool = Field(default=True, description="Whether API key is required for parser registration")
-
-    allow_test_keys: bool = Field(default=True, description="Allow test API keys (disable in production)")
-
-
-class RPCConfig(BaseModel):
-    """RPC configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True)
-
-    channel: str = Field(default="amazon_parser_rpc", description="RPC channel name")
-    timeout: int = Field(default=30, description="RPC call timeout in seconds")
-
-
-class PubSubConfig(BaseModel):
-    """PubSub configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True)
-
-    prefix: str = Field(default="amazon_parser", description="PubSub channel prefix")
-
-
-class LoggingConfig(BaseModel):
-    """Logging configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True)
-
-    level: str = Field(default="INFO", description="Logging level")
-    format: str = Field(default="%(asctime)s - %(name)s - %(levelname)s - %(message)s", description="Log message format")
-    enable_debug: bool = Field(default=False, description="Enable debug logging")
-
-
-class BridgeConfig(BaseModel):
-    """Main bridge server configuration."""
-
-    model_config = ConfigDict(str_strip_whitespace=True, validate_assignment=True, extra="forbid")
-
-    # Sub-configurations
-    redis: RedisConfig = Field(default_factory=RedisConfig)
-    websocket: WebSocketConfig = Field(default_factory=WebSocketConfig)
-    security: SecurityConfig = Field(default_factory=SecurityConfig)
-    rpc: RPCConfig = Field(default_factory=RPCConfig)
-    pubsub: PubSubConfig = Field(default_factory=PubSubConfig)
-    logging: LoggingConfig = Field(default_factory=LoggingConfig)
-
-    # Environment
-    environment: str = Field(default="development", description="Environment (development, testing, production)")
-
-    def get_all_api_keys(self) -> List[str]:
-        """Get all valid API keys (test + production)."""
-        all_keys = []
-
-        # Add production keys
-        all_keys.extend(self.security.api_keys)
-
-        # Add test keys if allowed
-        if self.security.allow_test_keys:
-            all_keys.extend(self.security.test_api_keys)
-
-        return all_keys
-
-    def is_valid_api_key(self, api_key: str) -> bool:
-        """Check if API key is valid."""
-        if not self.security.require_api_key:
-            return True
-
-        return api_key in self.get_all_api_keys()
-
-    def is_production(self) -> bool:
-        """Check if running in production."""
-        return self.environment.lower() == "production"
-
-    def is_development(self) -> bool:
-        """Check if running in development."""
-        return self.environment.lower() in ("development", "dev")
-
-    @field_validator("environment")
-    @classmethod
-    def validate_environment(cls, v):
-        """Validate environment value."""
-        valid_envs = ["development", "dev", "testing", "test", "production", "prod"]
-        if v.lower() not in valid_envs:
-            raise ValueError(f"Environment must be one of: {valid_envs}")
-        return v.lower()
-
-
-# Global config cache
-_config_cache: Optional[BridgeConfig] = None
-
-
-def get_config_path() -> Path:
-    """Get configuration file path."""
-    # Check environment variable first
-    config_path = os.getenv("BRIDGE_CONFIG_PATH")
-    if config_path:
-        return Path(config_path)
-
-    # Default path
-    return Path(__file__).parent / "bridge_config.yaml"
-
-
-def load_bridge_config(config_path: Optional[Path] = None) -> BridgeConfig:
-    """
-    Load bridge configuration from YAML file.
-
-    Args:
-        config_path: Optional path to config file
-
-    Returns:
-        BridgeConfig instance
-    """
-    global _config_cache
-
-    if _config_cache is not None:
-        return _config_cache
-
-    if config_path is None:
-        config_path = get_config_path()
-
-    if config_path.exists():
-        with open(config_path, "r", encoding="utf-8") as f:
-            config_data = yaml.safe_load(f)
-            _config_cache = BridgeConfig(**config_data)
-    else:
-        # Create default config
-        _config_cache = BridgeConfig()
-        save_bridge_config(_config_cache, config_path)
-
-    return _config_cache
-
-
-def save_bridge_config(config: BridgeConfig, config_path: Optional[Path] = None) -> Path:
-    """
-    Save bridge configuration to YAML file.
-
-    Args:
-        config: BridgeConfig instance
-        config_path: Optional path to save config
-
-    Returns:
-        Path where config was saved
-    """
-    if config_path is None:
-        config_path = get_config_path()
-
-    config_path.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(config_path, "w", encoding="utf-8") as f:
-        yaml.dump(config.model_dump(), f, default_flow_style=False, indent=2, allow_unicode=True)
-
-    return config_path
-
-
-def reload_config() -> BridgeConfig:
-    """Reload configuration from file, clearing cache."""
-    global _config_cache
-    _config_cache = None
-    return load_bridge_config()
-
-
-def create_sample_config() -> BridgeConfig:
-    """Create a sample configuration for testing."""
-    return BridgeConfig(
-        environment="development", websocket=WebSocketConfig(host="localhost", port=8002), security=SecurityConfig(test_api_keys=["amazon_parser_key_123", "test_parser_key_456", "debug_key_789"], allow_test_keys=True, require_api_key=True), logging=LoggingConfig(level="DEBUG", enable_debug=True)
-    )

unrealon_bridge/configs/bridge_config.yaml

@@ -1,39 +0,0 @@
-environment: development
-
-redis:
-  url: redis://localhost:6379/0
-  max_connections: 10
-  connection_timeout: 5
-
-websocket:
-  host: localhost
-  port: 8002
-  max_connections: 100
-  ping_interval: 20
-  ping_timeout: 10
-
-security:
-  # Test API keys for debugging (DO NOT USE IN PRODUCTION)
-  test_api_keys:
-    - amazon_parser_key_123
-    - test_parser_key_456
-    - debug_key_789
-    - development_key_000
-  
-  # Production API keys (add your keys here)
-  api_keys: []
-  
-  require_api_key: true
-  allow_test_keys: true
-
-rpc:
-  channel: amazon_parser_rpc
-  timeout: 30
-
-pubsub:
-  prefix: amazon_parser
-
-logging:
-  level: INFO
-  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-  enable_debug: false

unrealon_bridge/models/__init__.py

@@ -1,138 +0,0 @@
-"""
-Bridge Parsers Models - Decomposed and strictly typed.
-
-All models follow CRITICAL_REQUIREMENTS.md:
-- NO Dict[str, Any] usage
-- Strict Pydantic v2 typing
-- NO raw JSON handling
-- Complete type annotations
-"""
-
-# Base models
-from .base import BaseParserModel, BaseRPCRequest, BaseRPCResponse
-
-# Core domain models
-from .parser import ParserInfo, ParserStats, ParserHealth, ParserSystemStats
-from .command import ParserCommand, CommandResult
-from .proxy import ProxyRequest, ProxyInfo
-from .session import ParserSession
-from .events import ParserEvent, WebhookConfig
-from .html_parser import (
-    HTMLParseRequest, HTMLParseResult, HTMLParseResponse,
-    HTMLParseRPCRequest, HTMLParseRPCResponse
-)
-from .logging import (
-    ParserLogEntry, ParserLogRequest, ParserLogResponse
-)
-from .scheduler import (
-    TaskStatus, TaskPriority, ScheduleType, TaskExecutionMode,
-    CronExpression, TaskParameters, TaskRetryConfig, ScheduledTask,
-    TaskQueue, TaskExecutionResult, ParserStatus,
-    SchedulerError, TaskValidationError, TaskExecutionError,
-    ParserUnavailableError, QueueError
-)
-
-# RPC Request models (strict typing)
-from .requests import (
-    ParserRegisterRequest,
-    ParserStatusRequest,
-    ParserListRequest,
-    ParserHealthRequest,
-    SessionStartRequest,
-    SessionEndRequest,
-    CommandExecuteRequest,
-    CommandCreateRequest,
-    CommandStatusRequest,
-    ProxyAllocateRequest,
-    ProxyReleaseRequest,
-    ProxyCheckRequest,
-)
-
-# RPC Response models (strict typing)
-from .responses import (
-    ParserRegisterResponse,
-    ParserStatusResponse,
-    ParserListResponse,
-    ParserHealthResponse,
-    SessionStartResponse,
-    SessionEndResponse,
-    CommandExecuteResponse,
-    CommandCreateResponse,
-    CommandStatusResponse,
-    ProxyAllocateResponse,
-    ProxyReleaseResponse,
-    ProxyCheckResponse,
-)
-
-__all__ = [
-    # Base models
-    "BaseParserModel",
-    "BaseRPCRequest",
-    "BaseRPCResponse",
-    # Core domain models
-    "ParserInfo",
-    "ParserStats",
-    "ParserHealth",
-    "ParserSystemStats",
-    "ParserCommand",
-    "CommandResult",
-    "ProxyRequest",
-    "ProxyInfo",
-    "ParserSession",
-    "ParserEvent",
-    "WebhookConfig",
-    # HTML Parser models
-    "HTMLParseRequest",
-    "HTMLParseResult", 
-    "HTMLParseResponse",
-    "HTMLParseRPCRequest",
-    "HTMLParseRPCResponse",
-    # Parser Logging models
-    "ParserLogEntry",
-    "ParserLogRequest", 
-    "ParserLogResponse",
-    # Scheduler models
-    "TaskStatus",
-    "TaskPriority", 
-    "ScheduleType",
-    "TaskExecutionMode",
-    "CronExpression",
-    "TaskParameters",
-    "TaskRetryConfig",
-    "ScheduledTask",
-    "TaskQueue", 
-    "TaskExecutionResult",
-    "ParserStatus",
-    # Scheduler exceptions
-    "SchedulerError",
-    "TaskValidationError",
-    "TaskExecutionError",
-    "ParserUnavailableError",
-    "QueueError",
-    # RPC Request models
-    "ParserRegisterRequest",
-    "ParserStatusRequest",
-    "ParserListRequest",
-    "ParserHealthRequest",
-    "SessionStartRequest",
-    "SessionEndRequest",
-    "CommandExecuteRequest",
-    "CommandCreateRequest",
-    "CommandStatusRequest",
-    "ProxyAllocateRequest",
-    "ProxyReleaseRequest",
-    "ProxyCheckRequest",
-    # RPC Response models
-    "ParserRegisterResponse",
-    "ParserStatusResponse",
-    "ParserListResponse",
-    "ParserHealthResponse",
-    "SessionStartResponse",
-    "SessionEndResponse",
-    "CommandExecuteResponse",
-    "CommandCreateResponse",
-    "CommandStatusResponse",
-    "ProxyAllocateResponse",
-    "ProxyReleaseResponse",
-    "ProxyCheckResponse",
-]

unrealon_bridge/models/base.py

@@ -1,28 +0,0 @@
-"""
-Base models for parser bridge operations.
-
-Provides common base classes and shared types for all parser models.
-"""
-
-from pydantic import BaseModel, Field, ConfigDict
-from typing import Optional
-
-
-class BaseParserModel(BaseModel):
-    """Base class for all parser models with strict validation."""
-
-    model_config = ConfigDict(extra="forbid", validate_assignment=True, str_strip_whitespace=True, validate_default=True)
-
-
-class BaseRPCRequest(BaseParserModel):
-    """Base class for all RPC requests."""
-
-    pass
-
-
-class BaseRPCResponse(BaseParserModel):
-    """Base class for all RPC responses."""
-
-    success: bool = Field(description="Whether the operation was successful")
-    message: Optional[str] = Field(default=None, description="Human-readable message")
-    error: Optional[str] = Field(default=None, description="Error message if success=False")

unrealon_bridge/models/command.py

@@ -1,41 +0,0 @@
-"""
-Command-related models.
-
-Contains models for command execution, tracking, and results.
-"""
-
-from typing import Optional, Dict, Literal, Any
-from datetime import datetime
-from pydantic import Field
-from typing_extensions import Annotated
-
-from .base import BaseParserModel
-
-
-class ParserCommand(BaseParserModel):
-    """Parser command definition and tracking."""
-
-    command_id: Annotated[str, Field(min_length=1, description="Unique command identifier")]
-    command_type: Annotated[str, Field(min_length=1, description="Type of command (scrape, parse, etc.)")]
-    parser_id: Annotated[str, Field(min_length=1, description="Parser ID that will execute the command")]
-    parameters: Dict[str, str] = Field(default_factory=dict, description="Command parameters")
-    status: Literal["pending", "running", "completed", "failed", "cancelled"] = "pending"
-    priority: Annotated[int, Field(ge=1, le=10)] = 5
-    timeout: Annotated[int, Field(gt=0, le=3600, description="Command timeout in seconds")] = 300
-    created_at: datetime = Field(default_factory=datetime.now)
-    started_at: Optional[datetime] = None
-    completed_at: Optional[datetime] = None
-    retry_count: Annotated[int, Field(ge=0)] = 0
-    max_retries: Annotated[int, Field(ge=0)] = 3
-
-
-class CommandResult(BaseParserModel):
-    """Result of command execution."""
-
-    command_id: Annotated[str, Field(min_length=1, description="Command ID")]
-    success: bool = Field(description="Whether command executed successfully")
-    result_data: Dict[str, Any] = Field(default_factory=dict, description="Command result data")
-    error_message: Optional[str] = None
-    execution_time: Annotated[float, Field(ge=0)] = 0.0
-    output_size: Annotated[int, Field(ge=0)] = 0
-    created_at: datetime = Field(default_factory=datetime.now)

unrealon_bridge/models/events.py

@@ -1,40 +0,0 @@
-"""
-Event and webhook-related models.
-
-Contains models for parser events, logging, and webhook configurations.
-"""
-
-from typing import Optional, Dict, List, Literal
-from datetime import datetime
-from pydantic import Field
-from typing_extensions import Annotated
-
-from .base import BaseParserModel
-
-
-class ParserEvent(BaseParserModel):
-    """Parser event for logging and monitoring."""
-
-    event_id: Annotated[str, Field(min_length=1, description="Unique event identifier")]
-    parser_id: Annotated[str, Field(min_length=1, description="Parser ID")]
-    event_type: Annotated[str, Field(min_length=1, description="Type of event")]
-    level: Literal["debug", "info", "warning", "error", "critical"] = "info"
-    message: Annotated[str, Field(min_length=1, description="Event message")]
-    data: Dict[str, str] = Field(default_factory=dict, description="Additional event data (string values only)")
-    command_id: Optional[str] = None
-    session_id: Optional[str] = None
-    timestamp: datetime = Field(default_factory=datetime.now)
-    source: Optional[str] = None
-
-
-class WebhookConfig(BaseParserModel):
-    """Webhook configuration for parser events."""
-
-    webhook_id: Annotated[str, Field(min_length=1, description="Unique webhook identifier")]
-    parser_id: Annotated[str, Field(min_length=1, description="Parser ID")]
-    url: Annotated[str, Field(min_length=1, description="Webhook URL")]
-    events: List[str] = Field(default_factory=list, description="Events to trigger webhook")
-    headers: Dict[str, str] = Field(default_factory=dict, description="Custom headers")
-    secret: Optional[str] = None
-    active: bool = True
-    created_at: datetime = Field(default_factory=datetime.now)

unrealon_bridge/models/html_parser.py

@@ -1,79 +0,0 @@
-"""
-HTML Parser models for LLM-based HTML parsing.
-
-Models for sending HTML content to Django backend for AI-powered parsing.
-"""
-
-from datetime import datetime
-from typing import Dict, List, Optional, Any
-from pydantic import BaseModel, Field
-
-from .base import BaseParserModel
-
-
-class HTMLParseRequest(BaseParserModel):
-    """Request for HTML parsing via AI/LLM."""
-    
-    html_content: str = Field(..., description="Raw HTML content to parse")
-    parser_id: str = Field(..., description="ID of the parser making the request")
-    url: Optional[str] = Field(None, description="Source URL of the HTML (for context)")
-    parse_type: str = Field("general", description="Type of parsing (product, listing, article, etc.)")
-    instructions: Optional[str] = Field(None, description="Additional parsing instructions")
-    timeout: int = Field(60, description="Timeout in seconds (default 60s for LLM processing)")
-    metadata: Dict[str, str] = Field(default_factory=dict, description="Additional metadata")
-
-
-class HTMLParseResult(BaseParserModel):
-    """Result of HTML parsing."""
-    
-    success: bool = Field(..., description="Whether parsing was successful")
-    parsed_data: Optional[Dict[str, Any]] = Field(None, description="Parsed JSON data")
-    markdown: Optional[str] = Field(None, description="Markdown instructions from Django parser")
-    error_message: Optional[str] = Field(None, description="Error message if parsing failed")
-
-
-class HTMLParseResponse(BaseParserModel):
-    """Response for HTML parsing request."""
-    
-    success: bool = Field(..., description="Whether the request was successful")
-    message: Optional[str] = Field(None, description="Response message")
-    error: Optional[str] = Field(None, description="Error message if failed")
-    result: Optional[HTMLParseResult] = Field(None, description="Parsing result")
-    request_id: Optional[str] = Field(None, description="Request ID for tracking")
-    processed_at: datetime = Field(default_factory=datetime.now, description="When the request was processed")
-
-
-
-
-
-
-
-
-# Request/Response models for RPC
-class HTMLParseRPCRequest(BaseParserModel):
-    """RPC request for HTML parsing."""
-    
-    html_content: str
-    parser_id: str
-    url: Optional[str] = None
-    parse_type: str = "general"
-    instructions: Optional[str] = None
-    timeout: int = 60
-    metadata: Dict[str, str] = Field(default_factory=dict)
-
-
-class HTMLParseRPCResponse(BaseParserModel):
-    """RPC response for HTML parsing."""
-    
-    success: bool
-    message: Optional[str] = None
-    error: Optional[str] = None
-    result: Optional[HTMLParseResult] = None
-    request_id: Optional[str] = None
-    processed_at: datetime = Field(default_factory=datetime.now)
-
-
-
-
-
-

unrealon_bridge/models/logging.py

@@ -1,55 +0,0 @@
-"""
-Parser logging models.
-
-Models for sending parser logs to Django via WebSocket/Redis.
-"""
-
-from datetime import datetime
-from typing import Dict, Optional
-from enum import Enum
-from pydantic import BaseModel, Field
-
-from .base import BaseParserModel
-
-
-class ParserLogLevel(str, Enum):
-    """Parser log levels."""
-    
-    DEBUG = "DEBUG"
-    INFO = "INFO"
-    WARNING = "WARNING"
-    ERROR = "ERROR"
-    CRITICAL = "CRITICAL"
-
-
-class ParserLogEntry(BaseParserModel):
-    """Parser log entry to send to Django."""
-    
-    parser_id: str = Field(..., description="ID of the parser")
-    level: str = Field(..., description="Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)")
-    message: str = Field(..., description="Log message")
-    timestamp: datetime = Field(default_factory=datetime.now, description="When the log was created")
-    
-    # Optional context
-    session_id: Optional[str] = Field(None, description="Parser session ID")
-    command_id: Optional[str] = Field(None, description="Command ID if related to command")
-    url: Optional[str] = Field(None, description="URL being processed")
-    operation: Optional[str] = Field(None, description="Operation being performed")
-    
-    # Additional data
-    data: Dict[str, str] = Field(default_factory=dict, description="Additional log data")
-    error_details: Optional[str] = Field(None, description="Error details if error log")
-
-
-class ParserLogRequest(BaseParserModel):
-    """RPC request for sending parser log."""
-    
-    log_entry: ParserLogEntry
-
-
-class ParserLogResponse(BaseParserModel):
-    """RPC response for parser log."""
-    
-    success: bool = Field(..., description="Whether log was sent successfully")
-    message: Optional[str] = Field(None, description="Response message")
-    error: Optional[str] = Field(None, description="Error message if failed")