unrealon 1.1.6__py3-none-any.whl → 2.0.4__py3-none-any.whl

unrealon_core/enums/types.py

@@ -0,0 +1,240 @@
+"""
+Type enums for UnrealOn system.
+
+These enums define types for messages, proxies, tasks, and other
+system components. They ensure type safety and consistent
+categorization throughout the system.
+
+Phase 1: Foundation type enums
+"""
+
+from enum import Enum
+
+
+class MessageType(str, Enum):
+    """
+    WebSocket message types for driver communication.
+    
+    Defines all possible message types that can be sent
+    between the system and drivers via WebSocket.
+    """
+    
+    # Driver lifecycle messages
+    DRIVER_REGISTER = "driver_register"           # Driver registration request
+    DRIVER_REGISTER_RESPONSE = "driver_register_response"  # Registration response
+    DRIVER_HEARTBEAT = "driver_heartbeat"         # Driver heartbeat/ping
+    DRIVER_STATUS = "driver_status"               # Driver status update
+    DRIVER_DISCONNECT = "driver_disconnect"      # Driver disconnect notification
+    
+    # Task management messages
+    TASK_ASSIGN = "task_assign"                   # Assign task to driver
+    TASK_RESULT = "task_result"                   # Task result from driver
+    TASK_ERROR = "task_error"                     # Task error from driver
+    TASK_PROGRESS = "task_progress"               # Task progress update
+    TASK_CANCEL = "task_cancel"                   # Cancel task request
+    
+    # Command messages
+    COMMAND_START = "command_start"               # Start command
+    COMMAND_STOP = "command_stop"                 # Stop command
+    COMMAND_RESTART = "command_restart"           # Restart command
+    COMMAND_PAUSE = "command_pause"               # Pause command
+    COMMAND_RESUME = "command_resume"             # Resume command
+    COMMAND_CONFIG_UPDATE = "command_config_update"  # Update configuration
+    
+    # Proxy management messages
+    PROXY_REQUEST = "proxy_request"               # Request proxy from pool
+    PROXY_RESPONSE = "proxy_response"             # Proxy response
+    PROXY_HEALTH_REPORT = "proxy_health_report"  # Report proxy health
+    PROXY_ROTATION_REQUEST = "proxy_rotation_request"  # Request proxy rotation
+    PROXY_RELEASE = "proxy_release"              # Release proxy assignment
+    
+    # Logging messages
+    LOG_MESSAGE = "log_message"                   # Log message from driver
+    LOG_BATCH = "log_batch"                       # Batch of log messages
+    
+    # System messages
+    PING = "ping"                                 # Ping message
+    PONG = "pong"                                 # Pong response
+    ERROR = "error"                               # Error message
+    ACK = "ack"                                   # Acknowledgment
+    
+    def is_driver_lifecycle(self) -> bool:
+        """Check if message is related to driver lifecycle."""
+        return self in [
+            MessageType.DRIVER_REGISTER,
+            MessageType.DRIVER_REGISTER_RESPONSE,
+            MessageType.DRIVER_HEARTBEAT,
+            MessageType.DRIVER_STATUS,
+            MessageType.DRIVER_DISCONNECT
+        ]
+    
+    def is_task_related(self) -> bool:
+        """Check if message is related to task management."""
+        return self in [
+            MessageType.TASK_ASSIGN,
+            MessageType.TASK_RESULT,
+            MessageType.TASK_ERROR,
+            MessageType.TASK_PROGRESS,
+            MessageType.TASK_CANCEL
+        ]
+    
+    def is_command(self) -> bool:
+        """Check if message is a command."""
+        return self.value.startswith("command_")
+    
+    def requires_response(self) -> bool:
+        """Check if message type requires a response."""
+        return self in [
+            MessageType.DRIVER_REGISTER,
+            MessageType.PROXY_REQUEST,
+            MessageType.PING
+        ]
+
+
+class ProxyType(str, Enum):
+    """
+    Proxy server types supported by the system.
+    
+    Defines the different types of proxy servers
+    that can be used for web requests.
+    """
+    
+    HTTP = "http"                  # HTTP proxy
+    HTTPS = "https"                # HTTPS proxy
+    SOCKS4 = "socks4"             # SOCKS4 proxy
+    SOCKS5 = "socks5"             # SOCKS5 proxy
+    
+    def supports_https(self) -> bool:
+        """Check if proxy type supports HTTPS."""
+        return self in [ProxyType.HTTPS, ProxyType.SOCKS5]
+    
+    def is_socks(self) -> bool:
+        """Check if proxy is a SOCKS proxy."""
+        return self in [ProxyType.SOCKS4, ProxyType.SOCKS5]
+    
+    def get_default_port(self) -> int:
+        """Get default port for proxy type."""
+        ports = {
+            ProxyType.HTTP: 8080,
+            ProxyType.HTTPS: 8080,
+            ProxyType.SOCKS4: 1080,
+            ProxyType.SOCKS5: 1080,
+        }
+        return ports.get(self, 8080)
+
+
+class TaskPriority(str, Enum):
+    """
+    Task priority levels for queue management.
+    
+    Defines priority levels that determine the order
+    in which tasks are processed by drivers.
+    """
+    
+    LOW = "low"                    # Low priority task
+    NORMAL = "normal"              # Normal priority task (default)
+    HIGH = "high"                  # High priority task
+    URGENT = "urgent"              # Urgent priority task
+    CRITICAL = "critical"          # Critical priority task
+    
+    def get_numeric_priority(self) -> int:
+        """Get numeric priority for sorting (higher = more urgent)."""
+        priorities = {
+            TaskPriority.LOW: 1,
+            TaskPriority.NORMAL: 5,
+            TaskPriority.HIGH: 10,
+            TaskPriority.URGENT: 20,
+            TaskPriority.CRITICAL: 50,
+        }
+        return priorities.get(self, 5)
+    
+    def is_urgent(self) -> bool:
+        """Check if task has urgent or critical priority."""
+        return self in [TaskPriority.URGENT, TaskPriority.CRITICAL]
+    
+    @classmethod
+    def from_numeric(cls, priority: int) -> 'TaskPriority':
+        """Create TaskPriority from numeric value."""
+        if priority >= 50:
+            return cls.CRITICAL
+        elif priority >= 20:
+            return cls.URGENT
+        elif priority >= 10:
+            return cls.HIGH
+        elif priority >= 5:
+            return cls.NORMAL
+        else:
+            return cls.LOW
+
+
+class DriverType(str, Enum):
+    """
+    Driver types for categorization and capability matching.
+    
+    Defines different types of drivers based on their
+    primary functionality and target websites.
+    """
+    
+    # General purpose
+    UNIVERSAL = "universal"        # Universal driver for any site
+    GENERIC = "generic"           # Generic web scraping driver
+    
+    # E-commerce specific
+    ECOMMERCE = "ecommerce"       # E-commerce sites
+    MARKETPLACE = "marketplace"    # Online marketplaces
+    PRODUCT_CATALOG = "product_catalog"  # Product catalogs
+    
+    # Content specific
+    NEWS = "news"                 # News websites
+    BLOG = "blog"                 # Blog sites
+    SOCIAL_MEDIA = "social_media" # Social media platforms
+    
+    # Data specific
+    API = "api"                   # API-based data collection
+    DATABASE = "database"         # Database extraction
+    FILE = "file"                 # File processing
+    
+    def is_web_scraping(self) -> bool:
+        """Check if driver type involves web scraping."""
+        return self in [
+            DriverType.UNIVERSAL,
+            DriverType.GENERIC,
+            DriverType.ECOMMERCE,
+            DriverType.MARKETPLACE,
+            DriverType.PRODUCT_CATALOG,
+            DriverType.NEWS,
+            DriverType.BLOG,
+            DriverType.SOCIAL_MEDIA
+        ]
+    
+    def requires_browser(self) -> bool:
+        """Check if driver type typically requires a browser."""
+        return self in [
+            DriverType.ECOMMERCE,
+            DriverType.MARKETPLACE,
+            DriverType.SOCIAL_MEDIA
+        ]
+
+
+class ConfigType(str, Enum):
+    """
+    Configuration types for different system components.
+    
+    Used to categorize and validate different types
+    of configuration objects in the system.
+    """
+    
+    SYSTEM = "system"             # System-wide configuration
+    DRIVER = "driver"             # Driver-specific configuration
+    HTTP = "http"                 # HTTP client configuration
+    PROXY = "proxy"               # Proxy management configuration
+    BROWSER = "browser"           # Browser automation configuration
+    LOGGING = "logging"           # Logging configuration
+    CACHE = "cache"               # Cache configuration
+    THREAD = "thread"             # Threading configuration
+    DATABASE = "database"         # Database configuration
+    SECURITY = "security"         # Security configuration
+    
+    def get_config_schema(self) -> str:
+        """Get the schema identifier for this config type."""
+        return f"unrealon.config.{self.value}"

unrealon_core/error_handling/__init__.py

@@ -0,0 +1,45 @@
+"""
+Error Handling Package
+
+Comprehensive error handling and retry logic for UnrealOn RPC system.
+Following critical requirements - max 500 lines, 100% Pydantic v2.
+
+Phase 2: Core Systems - Error Handling
+"""
+
+from .retry import (
+    RetryConfig, RetryStrategy, RetryResult,
+    ExponentialBackoff, LinearBackoff, FixedBackoff,
+    retry_async, retry_sync
+)
+from .circuit_breaker import (
+    CircuitBreakerConfig, CircuitBreakerState,
+    CircuitBreaker, circuit_breaker, get_circuit_breaker
+)
+from .error_context import (
+    ErrorContext, ErrorSeverity,
+    create_error_context, format_error_context
+)
+from .recovery import (
+    RecoveryStrategy, RecoveryAction,
+    AutoRecovery, recovery_handler
+)
+
+__all__ = [
+    # Retry system
+    'RetryConfig', 'RetryStrategy', 'RetryResult',
+    'ExponentialBackoff', 'LinearBackoff', 'FixedBackoff',
+    'retry_async', 'retry_sync',
+    
+    # Circuit breaker
+    'CircuitBreakerConfig', 'CircuitBreakerState',
+    'CircuitBreaker', 'circuit_breaker', 'get_circuit_breaker',
+    
+    # Error context
+    'ErrorContext', 'ErrorSeverity',
+    'create_error_context', 'format_error_context',
+    
+    # Recovery system
+    'RecoveryStrategy', 'RecoveryAction',
+    'AutoRecovery', 'recovery_handler',
+]

unrealon_core/error_handling/circuit_breaker.py

@@ -0,0 +1,292 @@
+"""
+Circuit Breaker Pattern
+
+Prevents cascading failures by temporarily disabling failing services.
+Following critical requirements - max 500 lines, functions < 20 lines.
+
+Phase 2: Core Systems - Error Handling
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Callable, Optional, Dict
+from enum import Enum
+
+from pydantic import BaseModel, Field, ConfigDict
+
+from ..exceptions.base import UnrealOnError
+from ..utils.time import utc_now
+
+logger = logging.getLogger(__name__)
+
+
+class CircuitBreakerState(str, Enum):
+    """Circuit breaker states."""
+    CLOSED = "closed"      # Normal operation
+    OPEN = "open"          # Failing, requests blocked
+    HALF_OPEN = "half_open"  # Testing if service recovered
+
+
+class CircuitBreakerError(UnrealOnError):
+    """Circuit breaker is open."""
+    pass
+
+
+class CircuitBreakerConfig(BaseModel):
+    """Configuration for circuit breaker."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    failure_threshold: int = Field(
+        default=5, 
+        ge=1, 
+        le=20, 
+        description="Failures needed to open circuit"
+    )
+    recovery_timeout: float = Field(
+        default=60.0, 
+        ge=1.0, 
+        le=300.0, 
+        description="Seconds before trying half-open"
+    )
+    success_threshold: int = Field(
+        default=3, 
+        ge=1, 
+        le=10, 
+        description="Successes needed to close circuit"
+    )
+    timeout: float = Field(
+        default=30.0, 
+        ge=1.0, 
+        le=120.0, 
+        description="Operation timeout in seconds"
+    )
+
+
+class CircuitBreakerStats(BaseModel):
+    """Circuit breaker statistics."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    total_requests: int = Field(default=0, description="Total requests processed")
+    successful_requests: int = Field(default=0, description="Successful requests")
+    failed_requests: int = Field(default=0, description="Failed requests")
+    rejected_requests: int = Field(default=0, description="Requests rejected by circuit")
+    last_failure_time: Optional[datetime] = Field(default=None, description="Last failure timestamp")
+    last_success_time: Optional[datetime] = Field(default=None, description="Last success timestamp")
+    
+    def get_failure_rate(self) -> float:
+        """Calculate failure rate percentage."""
+        if self.total_requests == 0:
+            return 0.0
+        return (self.failed_requests / self.total_requests) * 100.0
+
+
+class CircuitBreaker:
+    """
+    Circuit breaker implementation.
+    
+    Prevents cascading failures by monitoring service health
+    and temporarily blocking requests to failing services.
+    """
+    
+    def __init__(self, name: str, config: CircuitBreakerConfig):
+        """Initialize circuit breaker."""
+        self.name = name
+        self.config = config
+        self.state = CircuitBreakerState.CLOSED
+        self.stats = CircuitBreakerStats()
+        
+        # State tracking
+        self._consecutive_failures = 0
+        self._consecutive_successes = 0
+        self._last_failure_time: Optional[datetime] = None
+        self._lock = asyncio.Lock()
+        
+        self.logger = logging.getLogger(f"circuit_breaker.{name}")
+    
+    async def call(self, func: Callable[..., Any], *args, **kwargs) -> Any:
+        """
+        Execute function through circuit breaker.
+        
+        Args:
+            func: Function to execute
+            *args, **kwargs: Function arguments
+            
+        Returns:
+            Function result
+            
+        Raises:
+            CircuitBreakerError: If circuit is open
+        """
+        async with self._lock:
+            await self._update_state()
+            
+            if self.state == CircuitBreakerState.OPEN:
+                self.stats.rejected_requests += 1
+                self.logger.warning(f"Circuit breaker {self.name} is OPEN - rejecting request")
+                raise CircuitBreakerError(f"Circuit breaker {self.name} is open")
+            
+            self.stats.total_requests += 1
+        
+        # Execute function with timeout
+        try:
+            result = await asyncio.wait_for(
+                func(*args, **kwargs),
+                timeout=self.config.timeout
+            )
+            
+            await self._record_success()
+            return result
+            
+        except Exception as e:
+            await self._record_failure()
+            raise
+    
+    async def _update_state(self) -> None:
+        """Update circuit breaker state based on current conditions."""
+        now = utc_now()
+        
+        if self.state == CircuitBreakerState.OPEN:
+            # Check if we should transition to half-open
+            if (self._last_failure_time and 
+                (now - self._last_failure_time).total_seconds() >= self.config.recovery_timeout):
+                
+                self.state = CircuitBreakerState.HALF_OPEN
+                self._consecutive_successes = 0
+                self.logger.info(f"Circuit breaker {self.name} transitioning to HALF_OPEN")
+        
+        elif self.state == CircuitBreakerState.HALF_OPEN:
+            # Check if we should close the circuit
+            if self._consecutive_successes >= self.config.success_threshold:
+                self.state = CircuitBreakerState.CLOSED
+                self._consecutive_failures = 0
+                self.logger.info(f"Circuit breaker {self.name} transitioning to CLOSED")
+    
+    async def _record_success(self) -> None:
+        """Record successful operation."""
+        async with self._lock:
+            self.stats.successful_requests += 1
+            self.stats.last_success_time = utc_now()
+            
+            if self.state == CircuitBreakerState.HALF_OPEN:
+                self._consecutive_successes += 1
+                self.logger.debug(
+                    f"Circuit breaker {self.name} success "
+                    f"({self._consecutive_successes}/{self.config.success_threshold})"
+                )
+            
+            # Reset failure counter on success
+            self._consecutive_failures = 0
+    
+    async def _record_failure(self) -> None:
+        """Record failed operation."""
+        async with self._lock:
+            self.stats.failed_requests += 1
+            self.stats.last_failure_time = utc_now()
+            self._last_failure_time = self.stats.last_failure_time
+            
+            self._consecutive_failures += 1
+            self._consecutive_successes = 0
+            
+            self.logger.debug(
+                f"Circuit breaker {self.name} failure "
+                f"({self._consecutive_failures}/{self.config.failure_threshold})"
+            )
+            
+            # Check if we should open the circuit
+            if (self.state == CircuitBreakerState.CLOSED and 
+                self._consecutive_failures >= self.config.failure_threshold):
+                
+                self.state = CircuitBreakerState.OPEN
+                self.logger.warning(f"Circuit breaker {self.name} transitioning to OPEN")
+            
+            elif self.state == CircuitBreakerState.HALF_OPEN:
+                # Any failure in half-open state reopens the circuit
+                self.state = CircuitBreakerState.OPEN
+                self.logger.warning(f"Circuit breaker {self.name} reopening due to failure in HALF_OPEN")
+    
+    def get_status(self) -> Dict[str, Any]:
+        """Get circuit breaker status."""
+        return {
+            'name': self.name,
+            'state': self.state.value,
+            'consecutive_failures': self._consecutive_failures,
+            'consecutive_successes': self._consecutive_successes,
+            'config': self.config.model_dump(),
+            'stats': self.stats.model_dump()
+        }
+    
+    def reset(self) -> None:
+        """Reset circuit breaker to closed state."""
+        self.state = CircuitBreakerState.CLOSED
+        self._consecutive_failures = 0
+        self._consecutive_successes = 0
+        self._last_failure_time = None
+        self.logger.info(f"Circuit breaker {self.name} manually reset to CLOSED")
+
+
+# Global circuit breaker registry
+_circuit_breakers: Dict[str, CircuitBreaker] = {}
+
+
+def get_circuit_breaker(
+    name: str, 
+    config: Optional[CircuitBreakerConfig] = None
+) -> CircuitBreaker:
+    """Get or create circuit breaker by name."""
+    if name not in _circuit_breakers:
+        if config is None:
+            config = CircuitBreakerConfig()
+        _circuit_breakers[name] = CircuitBreaker(name, config)
+    
+    return _circuit_breakers[name]
+
+
+def circuit_breaker(
+    name: str, 
+    config: Optional[CircuitBreakerConfig] = None
+):
+    """Decorator to wrap function with circuit breaker."""
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        cb = get_circuit_breaker(name, config)
+        
+        async def async_wrapper(*args, **kwargs):
+            return await cb.call(func, *args, **kwargs)
+        
+        def sync_wrapper(*args, **kwargs):
+            # For sync functions, we need to handle differently
+            # This is a simplified version - in production you might want
+            # to use threading or a different approach
+            import asyncio
+            try:
+                loop = asyncio.get_event_loop()
+                return loop.run_until_complete(cb.call(func, *args, **kwargs))
+            except RuntimeError:
+                # No event loop running
+                return asyncio.run(cb.call(func, *args, **kwargs))
+        
+        if asyncio.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator
+
+
+def get_all_circuit_breakers() -> Dict[str, Dict[str, Any]]:
+    """Get status of all circuit breakers."""
+    return {name: cb.get_status() for name, cb in _circuit_breakers.items()}
+
+
+def reset_all_circuit_breakers() -> None:
+    """Reset all circuit breakers."""
+    for cb in _circuit_breakers.values():
+        cb.reset()