unrealon 1.1.1__py3-none-any.whl → 1.1.4__py3-none-any.whl

unrealon_bridge/models/scheduler.py

@@ -1,592 +0,0 @@
-"""
-Task Scheduler Models - UnrealOn RPC v2.0
-
-Pydantic v2 models for hybrid RPC + Redis Queue task scheduling system.
-Provides complete type safety for scheduled tasks, cron expressions, and task execution.
-
-COMPLIANCE: 100% Pydantic v2, no Dict[str, Any], strict typing everywhere.
-"""
-
-import uuid
-from enum import Enum
-from typing import Optional, List, Union
-from datetime import datetime, timedelta
-from pydantic import BaseModel, Field, ConfigDict, field_validator, model_validator
-from typing_extensions import Annotated
-
-from .base import BaseParserModel
-
-
-class TaskStatus(str, Enum):
-    """Task execution status with clear state transitions."""
-    
-    PENDING = "pending"        # Task created, waiting for execution
-    QUEUED = "queued"         # Task added to Redis Queue
-    RUNNING = "running"       # Task currently executing
-    COMPLETED = "completed"   # Task finished successfully
-    FAILED = "failed"         # Task failed with error
-    CANCELLED = "cancelled"   # Task cancelled by user
-    TIMEOUT = "timeout"       # Task exceeded timeout
-    RETRY = "retry"           # Task failed, will retry
-
-
-class TaskPriority(int, Enum):
-    """Task priority levels for queue ordering."""
-    
-    CRITICAL = 1    # System critical tasks
-    HIGH = 3        # High priority tasks
-    NORMAL = 5      # Default priority
-    LOW = 7         # Background tasks
-    BULK = 9        # Bulk processing tasks
-
-
-class ScheduleType(str, Enum):
-    """Types of task scheduling."""
-    
-    IMMEDIATE = "immediate"    # Execute immediately via RPC
-    DELAYED = "delayed"        # Execute after delay via Queue
-    RECURRING = "recurring"    # Execute on cron schedule
-    CONDITIONAL = "conditional" # Execute when condition met
-
-
-class TaskExecutionMode(str, Enum):
-    """Task execution mode selection."""
-    
-    RPC_ONLY = "rpc_only"         # Force RPC execution
-    QUEUE_ONLY = "queue_only"     # Force Queue execution
-    HYBRID_AUTO = "hybrid_auto"   # Auto-select based on parser status
-    HYBRID_PREFER_RPC = "hybrid_prefer_rpc"     # Prefer RPC, fallback to Queue
-    HYBRID_PREFER_QUEUE = "hybrid_prefer_queue" # Prefer Queue, fallback to RPC
-
-
-class CronExpression(BaseModel):
-    """Cron expression with validation and parsing."""
-    
-    model_config = ConfigDict(
-        validate_assignment=True,
-        extra="forbid",
-        str_strip_whitespace=True
-    )
-    
-    expression: Annotated[str, Field(
-        description="Cron expression (minute hour day month weekday)",
-        examples=["0 2 * * *", "*/15 * * * *", "0 9 * * 1-5"]
-    )]
-    
-    timezone: Annotated[str, Field(
-        default="UTC",
-        description="Timezone for cron execution",
-        examples=["UTC", "America/New_York", "Asia/Seoul"]
-    )]
-    
-    @field_validator('expression')
-    @classmethod
-    def validate_cron_expression(cls, v: str) -> str:
-        """Validate cron expression format."""
-        if not v or not v.strip():
-            raise ValueError("Cron expression cannot be empty")
-        
-        parts = v.strip().split()
-        if len(parts) != 5:
-            raise ValueError("Cron expression must have exactly 5 parts: minute hour day month weekday")
-        
-        # Basic validation for each part
-        minute, hour, day, month, weekday = parts
-        
-        # Validate ranges (basic check)
-        for part, name, max_val in [
-            (minute, "minute", 59),
-            (hour, "hour", 23), 
-            (day, "day", 31),
-            (month, "month", 12),
-            (weekday, "weekday", 7)
-        ]:
-            if part != "*" and not any(c in part for c in ["/", "-", ","]):
-                try:
-                    val = int(part)
-                    if val < 0 or val > max_val:
-                        raise ValueError(f"Invalid {name} value: {val}")
-                except ValueError as e:
-                    if "invalid literal" not in str(e):
-                        raise
-        
-        return v.strip()
-
-
-class TaskParameters(BaseModel):
-    """Strongly typed task parameters."""
-    
-    model_config = ConfigDict(
-        validate_assignment=True,
-        extra="forbid"
-    )
-    
-    command_type: Annotated[str, Field(
-        min_length=1,
-        max_length=100,
-        description="Type of command to execute",
-        examples=["scrape", "parse", "daily_update", "cleanup"]
-    )]
-    
-    parameters: Annotated[dict[str, str], Field(
-        default_factory=dict,
-        description="Command parameters (string values only for Redis compatibility)"
-    )]
-    
-    timeout: Annotated[int, Field(
-        default=300,
-        ge=1,
-        le=86400,  # 24 hours max
-        description="Task timeout in seconds"
-    )]
-    
-    @field_validator('parameters')
-    @classmethod
-    def validate_parameters(cls, v: dict[str, str]) -> dict[str, str]:
-        """Ensure all parameter values are strings."""
-        if not isinstance(v, dict):
-            raise ValueError("Parameters must be a dictionary")
-        
-        for key, value in v.items():
-            if not isinstance(key, str):
-                raise ValueError(f"Parameter key must be string, got {type(key)}")
-            if not isinstance(value, str):
-                raise ValueError(f"Parameter value must be string, got {type(value)} for key '{key}'")
-        
-        return v
-
-
-class TaskRetryConfig(BaseModel):
-    """Task retry configuration."""
-    
-    model_config = ConfigDict(
-        validate_assignment=True,
-        extra="forbid"
-    )
-    
-    max_retries: Annotated[int, Field(
-        default=3,
-        ge=0,
-        le=10,
-        description="Maximum number of retry attempts"
-    )]
-    
-    retry_delay: Annotated[int, Field(
-        default=60,
-        ge=1,
-        le=3600,
-        description="Initial retry delay in seconds"
-    )]
-    
-    exponential_backoff: Annotated[bool, Field(
-        default=True,
-        description="Use exponential backoff for retry delays"
-    )]
-    
-    max_retry_delay: Annotated[int, Field(
-        default=3600,
-        ge=60,
-        le=86400,
-        description="Maximum retry delay in seconds"
-    )]
-
-
-class ScheduledTask(BaseParserModel):
-    """Complete scheduled task definition with full type safety."""
-    
-    # Task identification
-    task_id: Annotated[str, Field(
-        default_factory=lambda: str(uuid.uuid4()),
-        description="Unique task identifier"
-    )]
-    
-    task_name: Annotated[str, Field(
-        min_length=1,
-        max_length=200,
-        description="Human-readable task name"
-    )]
-    
-    # Parser targeting
-    parser_type: Annotated[str, Field(
-        min_length=1,
-        max_length=100,
-        description="Target parser type",
-        examples=["encar_parser", "autotrader_parser"]
-    )]
-    
-    parser_id: Annotated[Optional[str], Field(
-        default=None,
-        description="Specific parser ID (optional, for targeting specific instance)"
-    )]
-    
-    # Task execution
-    task_parameters: TaskParameters
-    execution_mode: TaskExecutionMode = TaskExecutionMode.HYBRID_AUTO
-    priority: TaskPriority = TaskPriority.NORMAL
-    
-    # Scheduling
-    schedule_type: ScheduleType
-    scheduled_at: Annotated[Optional[datetime], Field(
-        default=None,
-        description="When to execute (for delayed tasks)"
-    )]
-    
-    cron_schedule: Annotated[Optional[CronExpression], Field(
-        default=None,
-        description="Cron schedule (for recurring tasks)"
-    )]
-    
-    # Status and tracking
-    status: TaskStatus = TaskStatus.PENDING
-    created_at: Annotated[datetime, Field(
-        default_factory=datetime.utcnow,
-        description="Task creation timestamp"
-    )]
-    
-    updated_at: Annotated[datetime, Field(
-        default_factory=datetime.utcnow,
-        description="Last update timestamp"
-    )]
-    
-    # Execution tracking
-    started_at: Annotated[Optional[datetime], Field(
-        default=None,
-        description="Task execution start time"
-    )]
-    
-    completed_at: Annotated[Optional[datetime], Field(
-        default=None,
-        description="Task completion time"
-    )]
-    
-    # Retry configuration
-    retry_config: TaskRetryConfig = Field(default_factory=TaskRetryConfig)
-    retry_count: Annotated[int, Field(
-        default=0,
-        ge=0,
-        description="Current retry attempt count"
-    )]
-    
-    # Results and errors
-    last_error: Annotated[Optional[str], Field(
-        default=None,
-        description="Last error message if failed"
-    )]
-    
-    execution_log: Annotated[List[str], Field(
-        default_factory=list,
-        description="Execution log entries"
-    )]
-    
-    # Metadata
-    tags: Annotated[List[str], Field(
-        default_factory=list,
-        description="Task tags for filtering and organization"
-    )]
-    
-    metadata: Annotated[dict[str, str], Field(
-        default_factory=dict,
-        description="Additional task metadata (string values only)"
-    )]
-    
-    @model_validator(mode='after')
-    def validate_schedule_consistency(self) -> 'ScheduledTask':
-        """Validate scheduling configuration consistency."""
-        if self.schedule_type == ScheduleType.DELAYED:
-            if not self.scheduled_at:
-                raise ValueError("Delayed tasks must have scheduled_at timestamp")
-        
-        elif self.schedule_type == ScheduleType.RECURRING:
-            if not self.cron_schedule:
-                raise ValueError("Recurring tasks must have cron_schedule")
-        
-        elif self.schedule_type == ScheduleType.IMMEDIATE:
-            if self.scheduled_at or self.cron_schedule:
-                raise ValueError("Immediate tasks cannot have scheduling configuration")
-        
-        return self
-    
-    @field_validator('metadata')
-    @classmethod
-    def validate_metadata(cls, v: dict[str, str]) -> dict[str, str]:
-        """Ensure metadata values are strings."""
-        if not isinstance(v, dict):
-            raise ValueError("Metadata must be a dictionary")
-        
-        for key, value in v.items():
-            if not isinstance(key, str):
-                raise ValueError(f"Metadata key must be string, got {type(key)}")
-            if not isinstance(value, str):
-                raise ValueError(f"Metadata value must be string, got {type(value)} for key '{key}'")
-        
-        return v
-    
-    def add_log_entry(self, message: str) -> None:
-        """Add entry to execution log."""
-        timestamp = datetime.utcnow().isoformat()
-        log_entry = f"[{timestamp}] {message}"
-        self.execution_log.append(log_entry)
-        self.updated_at = datetime.utcnow()
-    
-    def mark_started(self) -> None:
-        """Mark task as started."""
-        self.status = TaskStatus.RUNNING
-        self.started_at = datetime.utcnow()
-        self.updated_at = datetime.utcnow()
-        self.add_log_entry("Task execution started")
-    
-    def mark_completed(self, result_message: Optional[str] = None) -> None:
-        """Mark task as completed."""
-        self.status = TaskStatus.COMPLETED
-        self.completed_at = datetime.utcnow()
-        self.updated_at = datetime.utcnow()
-        
-        message = result_message or "Task completed successfully"
-        self.add_log_entry(message)
-    
-    def mark_failed(self, error_message: str) -> None:
-        """Mark task as failed."""
-        self.status = TaskStatus.FAILED
-        self.last_error = error_message
-        self.updated_at = datetime.utcnow()
-        self.add_log_entry(f"Task failed: {error_message}")
-    
-    def increment_retry(self) -> None:
-        """Increment retry count and update status."""
-        self.retry_count += 1
-        self.status = TaskStatus.RETRY
-        self.updated_at = datetime.utcnow()
-        self.add_log_entry(f"Retry attempt {self.retry_count}/{self.retry_config.max_retries}")
-    
-    def can_retry(self) -> bool:
-        """Check if task can be retried."""
-        return (
-            self.status in [TaskStatus.FAILED, TaskStatus.TIMEOUT] and
-            self.retry_count < self.retry_config.max_retries
-        )
-    
-    def get_next_retry_delay(self) -> int:
-        """Calculate next retry delay in seconds."""
-        if not self.can_retry():
-            return 0
-        
-        base_delay = self.retry_config.retry_delay
-        
-        if self.retry_config.exponential_backoff:
-            delay = base_delay * (2 ** self.retry_count)
-            return min(delay, self.retry_config.max_retry_delay)
-        
-        return base_delay
-
-
-class TaskQueue(BaseModel):
-    """Redis Queue configuration for task scheduling."""
-    
-    model_config = ConfigDict(
-        validate_assignment=True,
-        extra="forbid"
-    )
-    
-    queue_name: Annotated[str, Field(
-        min_length=1,
-        max_length=100,
-        description="Redis queue name"
-    )]
-    
-    redis_url: Annotated[str, Field(
-        description="Redis connection URL",
-        examples=["redis://localhost:6379/1"]
-    )]
-    
-    max_workers: Annotated[int, Field(
-        default=4,
-        ge=1,
-        le=20,
-        description="Maximum concurrent workers"
-    )]
-    
-    worker_timeout: Annotated[int, Field(
-        default=3600,
-        ge=60,
-        le=86400,
-        description="Worker timeout in seconds"
-    )]
-    
-    visibility_timeout: Annotated[int, Field(
-        default=300,
-        ge=30,
-        le=3600,
-        description="Task visibility timeout in seconds"
-    )]
-    
-    dead_letter_queue: Annotated[Optional[str], Field(
-        default=None,
-        description="Dead letter queue name for failed tasks"
-    )]
-
-
-class TaskExecutionResult(BaseParserModel):
-    """Result of task execution with complete type safety."""
-    
-    task_id: Annotated[str, Field(description="Task identifier")]
-    
-    success: Annotated[bool, Field(description="Execution success status")]
-    
-    execution_time: Annotated[float, Field(
-        ge=0.0,
-        description="Execution time in seconds"
-    )]
-    
-    result_data: Annotated[dict[str, str], Field(
-        default_factory=dict,
-        description="Task result data (string values only)"
-    )]
-    
-    error_message: Annotated[Optional[str], Field(
-        default=None,
-        description="Error message if failed"
-    )]
-    
-    retry_count: Annotated[int, Field(
-        default=0,
-        ge=0,
-        description="Number of retries performed"
-    )]
-    
-    executed_at: Annotated[datetime, Field(
-        default_factory=datetime.utcnow,
-        description="Execution timestamp"
-    )]
-    
-    executed_by: Annotated[Optional[str], Field(
-        default=None,
-        description="Parser ID that executed the task"
-    )]
-    
-    @field_validator('result_data')
-    @classmethod
-    def validate_result_data(cls, v: dict[str, str]) -> dict[str, str]:
-        """Ensure result data values are strings."""
-        if not isinstance(v, dict):
-            raise ValueError("Result data must be a dictionary")
-        
-        for key, value in v.items():
-            if not isinstance(key, str):
-                raise ValueError(f"Result key must be string, got {type(key)}")
-            if not isinstance(value, str):
-                raise ValueError(f"Result value must be string, got {type(value)} for key '{key}'")
-        
-        return v
-
-
-class ParserStatus(BaseModel):
-    """Parser online/offline status tracking."""
-    
-    model_config = ConfigDict(
-        validate_assignment=True,
-        extra="forbid"
-    )
-    
-    parser_id: Annotated[str, Field(description="Parser identifier")]
-    
-    parser_type: Annotated[str, Field(description="Parser type")]
-    
-    status: Annotated[str, Field(
-        description="Parser status",
-        pattern="^(online|offline|connecting|disconnecting)$"
-    )]
-    
-    last_seen: Annotated[datetime, Field(
-        default_factory=datetime.utcnow,
-        description="Last heartbeat timestamp"
-    )]
-    
-    capabilities: Annotated[List[str], Field(
-        default_factory=list,
-        description="Parser capabilities"
-    )]
-    
-    current_tasks: Annotated[int, Field(
-        default=0,
-        ge=0,
-        description="Number of currently executing tasks"
-    )]
-    
-    max_concurrent_tasks: Annotated[int, Field(
-        default=5,
-        ge=1,
-        le=50,
-        description="Maximum concurrent tasks this parser can handle"
-    )]
-    
-    def is_online(self, timeout_seconds: int = 300) -> bool:
-        """Check if parser is considered online."""
-        if self.status != "online":
-            return False
-        
-        time_since_last_seen = datetime.utcnow() - self.last_seen
-        return time_since_last_seen.total_seconds() <= timeout_seconds
-    
-    def can_accept_task(self) -> bool:
-        """Check if parser can accept new tasks."""
-        return (
-            self.is_online() and 
-            self.current_tasks < self.max_concurrent_tasks
-        )
-
-
-# Custom exceptions for scheduler
-class SchedulerError(Exception):
-    """Base scheduler error."""
-    
-    def __init__(self, message: str, task_id: Optional[str] = None, details: Optional[dict[str, str]] = None):
-        self.message = message
-        self.task_id = task_id
-        self.details = details or {}
-        super().__init__(message)
-
-
-class TaskValidationError(SchedulerError):
-    """Task validation errors."""
-    pass
-
-
-class TaskExecutionError(SchedulerError):
-    """Task execution errors."""
-    pass
-
-
-class ParserUnavailableError(SchedulerError):
-    """Parser not available for task execution."""
-    pass
-
-
-class QueueError(SchedulerError):
-    """Redis Queue operation errors."""
-    pass
-
-
-# Export all models and exceptions
-__all__ = [
-    # Enums
-    "TaskStatus",
-    "TaskPriority", 
-    "ScheduleType",
-    "TaskExecutionMode",
-    
-    # Models
-    "CronExpression",
-    "TaskParameters",
-    "TaskRetryConfig", 
-    "ScheduledTask",
-    "TaskQueue",
-    "TaskExecutionResult",
-    "ParserStatus",
-    
-    # Exceptions
-    "SchedulerError",
-    "TaskValidationError",
-    "TaskExecutionError",
-    "ParserUnavailableError",
-    "QueueError"
-]

unrealon_bridge/models/session.py

@@ -1,28 +0,0 @@
-"""
-Session-related models.
-
-Contains models for parser session management and tracking.
-"""
-
-from typing import Optional, Dict, Literal
-from datetime import datetime
-from pydantic import Field
-from typing_extensions import Annotated
-
-from .base import BaseParserModel
-
-
-class ParserSession(BaseParserModel):
-    """Parser session information and tracking."""
-
-    session_id: Annotated[str, Field(min_length=1, description="Unique session identifier")]
-    parser_id: Annotated[str, Field(min_length=1, description="Parser ID")]
-    session_type: Annotated[str, Field(min_length=1, description="Type of session (scraping, parsing, etc.)")]
-    status: Literal["active", "paused", "completed", "failed", "cancelled"] = "active"
-    metadata: Dict[str, str] = Field(default_factory=dict, description="Session metadata")
-    started_at: datetime = Field(default_factory=datetime.now)
-    ended_at: Optional[datetime] = None
-    duration_seconds: Optional[Annotated[int, Field(ge=0)]] = None
-    commands_executed: Annotated[int, Field(ge=0)] = 0
-    data_processed: Annotated[int, Field(ge=0)] = 0
-    errors_count: Annotated[int, Field(ge=0)] = 0

unrealon_bridge/server/__init__.py

@@ -1,91 +0,0 @@
-"""
-Parser Bridge Server - Modular implementation with composition.
-
-Clean architecture with separated handlers and no inheritance hell.
-"""
-
-from typing import Callable
-from unrealon_rpc.logging import get_logger
-
-from .base import ParserBridgeServerBase
-from .handlers import ParserHandlers, SessionHandlers, CommandHandlers, ProxyHandlers, HTMLParserHandlers, LoggingHandlers, SchedulerHandlers
-
-from ..models import ParserSystemStats
-
-logger = get_logger(__name__)
-
-
-class ParserBridgeServer(
-    ParserBridgeServerBase,
-    ParserHandlers,
-    SessionHandlers,
-    CommandHandlers,
-    ProxyHandlers,
-    HTMLParserHandlers,
-    LoggingHandlers,
-    SchedulerHandlers
-):
-    """
-    Complete Parser Bridge Server with all functionality.
-    
-    Combines base server with all handlers to provide full server capabilities:
-    - Parser registration and management
-    - Session lifecycle management
-    - Command execution and tracking
-    - Proxy allocation and management
-    - HTML parsing via AI/LLM integration
-    - Parser logging to Django backend
-    """
-
-    def __init__(self, redis_url: str = "redis://localhost:6379/0", rpc_channel: str = "parser_rpc", pubsub_prefix: str = "parser", **kwargs):
-        """
-        Initialize complete parser bridge server.
-
-        Args:
-            redis_url: Redis connection URL
-            rpc_channel: RPC channel name
-            pubsub_prefix: PubSub channel prefix
-            **kwargs: Additional arguments for WebSocketBridge
-        """
-        super().__init__(redis_url, rpc_channel, pubsub_prefix, **kwargs)
-        self._register_rpc_methods()
-
-    def _register_rpc_methods(self) -> None:
-        """Register all RPC methods with their handlers."""
-        # Parser management
-        self.parser_rpc.register_method("parser.register", self.handle_parser_register)
-        self.parser_rpc.register_method("parser.get_status", self.handle_parser_get_status)
-        self.parser_rpc.register_method("parser.list", self.handle_parser_list)
-        self.parser_rpc.register_method("parser.get_health", self.handle_parser_get_health)
-
-        # Session management
-        self.parser_rpc.register_method("parser.start_session", self.handle_session_start)
-        self.parser_rpc.register_method("parser.end_session", self.handle_session_end)
-
-        # Command management
-        self.parser_rpc.register_method("parser.execute_command", self.handle_command_execute)
-        self.parser_rpc.register_method("command.create", self.handle_command_create)
-        self.parser_rpc.register_method("command.get_status", self.handle_command_get_status)
-
-        # Proxy management
-        self.parser_rpc.register_method("proxy.allocate", self.handle_proxy_allocate)
-        self.parser_rpc.register_method("proxy.release", self.handle_proxy_release)
-        self.parser_rpc.register_method("proxy.check", self.handle_proxy_check)
-
-        # HTML Parser management
-        self.parser_rpc.register_method("html_parser.parse", self.handle_html_parse)
-
-        # Parser Logging
-        self.parser_rpc.register_method("parser.log", self.handle_parser_log)
-        
-        # Scheduler management
-        self.parser_rpc.register_method("scheduler.create_task", self.handle_scheduler_create_task)
-        self.parser_rpc.register_method("scheduler.list_tasks", self.handle_scheduler_list_tasks)
-        self.parser_rpc.register_method("scheduler.get_task", self.handle_scheduler_get_task)
-        self.parser_rpc.register_method("scheduler.cancel_task", self.handle_scheduler_cancel_task)
-        self.parser_rpc.register_method("scheduler.update_parser_status", self.handle_scheduler_update_parser_status)
-        self.parser_rpc.register_method("scheduler.get_parser_status", self.handle_scheduler_get_parser_status)
-        self.parser_rpc.register_method("scheduler.get_stats", self.handle_scheduler_get_stats)
-
-
-__all__ = ["ParserBridgeServer"]