unrealon 1.1.5__py3-none-any.whl → 2.0.4__py3-none-any.whl

unrealon_core/models/arq_context.py

@@ -0,0 +1,252 @@
+"""
+ARQ Context Models
+
+Strictly typed models for ARQ worker context and job parameters.
+Following critical requirements - no raw Dict[str, Any].
+
+Phase 2: Core Systems - ARQ Context
+"""
+
+from datetime import datetime
+from typing import Optional
+from pydantic import BaseModel, Field, ConfigDict
+
+from .base import UnrealOnBaseModel
+
+
+class ARQWorkerStats(UnrealOnBaseModel):
+    """ARQ worker statistics."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    processed_jobs: int = Field(default=0, description="Number of processed jobs")
+    failed_jobs: int = Field(default=0, description="Number of failed jobs")
+    start_time: datetime = Field(description="Worker start time")
+    websocket_messages_sent: int = Field(default=0, description="WebSocket messages sent")
+    websocket_messages_failed: int = Field(default=0, description="Failed WebSocket messages")
+
+
+class ARQWorkerContext(UnrealOnBaseModel):
+    """ARQ worker context with strict typing."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    stats: ARQWorkerStats = Field(description="Worker statistics")
+    websocket_bridge: Optional[object] = Field(default=None, description="WebSocket bridge instance")
+
+
+class TaskAssignmentParams(UnrealOnBaseModel):
+    """Parameters for task assignment job."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    task_type: str = Field(description="Type of task")
+    task_data: "TaskDataModel" = Field(description="Task data")
+    driver_id: Optional[str] = Field(default=None, description="Target driver ID")
+    priority: str = Field(default="normal", description="Task priority")
+    timeout: float = Field(default=300.0, description="Task timeout in seconds")
+    retry_count: int = Field(default=3, description="Number of retries")
+
+
+class TaskDataModel(UnrealOnBaseModel):
+    """Strictly typed task data."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    task_id: str = Field(description="Unique task identifier")
+    url: str = Field(description="Target URL")
+    parser_type: Optional[str] = Field(default=None, description="Parser type")
+    options: Optional["TaskOptionsModel"] = Field(default=None, description="Task options")
+
+
+class TaskOptionsModel(UnrealOnBaseModel):
+    """Task options model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    timeout_seconds: Optional[float] = Field(default=None, description="Request timeout")
+    user_agent: Optional[str] = Field(default=None, description="User agent string")
+    headers: Optional["HeadersModel"] = Field(default=None, description="HTTP headers")
+    proxy_enabled: bool = Field(default=True, description="Whether to use proxy")
+
+
+class HeadersModel(UnrealOnBaseModel):
+    """HTTP headers model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    accept: Optional[str] = Field(default=None, description="Accept header")
+    accept_language: Optional[str] = Field(default=None, description="Accept-Language header")
+    referer: Optional[str] = Field(default=None, description="Referer header")
+    authorization: Optional[str] = Field(default=None, description="Authorization header")
+
+
+class TaskResultParams(UnrealOnBaseModel):
+    """Parameters for task result processing."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(description="Driver ID")
+    connection_id: str = Field(description="WebSocket connection ID")
+    result_data: "TaskResultDataModel" = Field(description="Task result data")
+
+
+class TaskResultDataModel(UnrealOnBaseModel):
+    """Task result data model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    task_id: str = Field(description="Task ID")
+    status: str = Field(description="Task status")
+    result_data: Optional["ParsedDataModel"] = Field(default=None, description="Parsed data")
+    error_message: Optional[str] = Field(default=None, description="Error message if failed")
+    execution_time_seconds: float = Field(description="Execution time")
+
+
+class ParsedDataModel(UnrealOnBaseModel):
+    """Parsed data model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    title: Optional[str] = Field(default=None, description="Page title")
+    content: Optional[str] = Field(default=None, description="Page content")
+    links: Optional[list[str]] = Field(default=None, description="Extracted links")
+    images: Optional[list[str]] = Field(default=None, description="Extracted images")
+    metadata: Optional["MetadataModel"] = Field(default=None, description="Page metadata")
+
+
+class MetadataModel(UnrealOnBaseModel):
+    """Page metadata model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    description: Optional[str] = Field(default=None, description="Page description")
+    keywords: Optional[str] = Field(default=None, description="Page keywords")
+    author: Optional[str] = Field(default=None, description="Page author")
+    language: Optional[str] = Field(default=None, description="Page language")
+
+
+class LogEntryParams(UnrealOnBaseModel):
+    """Parameters for log entry processing."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(description="Driver ID")
+    connection_id: str = Field(description="WebSocket connection ID")
+    log_data: "LogDataModel" = Field(description="Log data")
+
+
+class LogDataModel(UnrealOnBaseModel):
+    """Log data model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    level: str = Field(description="Log level")
+    message: str = Field(description="Log message")
+    logger_name: str = Field(description="Logger name")
+    module: str = Field(description="Module name")
+    timestamp: datetime = Field(description="Log timestamp")
+
+
+class DriverRegistrationParams(UnrealOnBaseModel):
+    """Parameters for driver registration."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    connection_id: str = Field(description="WebSocket connection ID")
+    driver_data: "DriverDataModel" = Field(description="Driver data")
+
+
+class DriverDataModel(UnrealOnBaseModel):
+    """Driver data model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(description="Driver ID")
+    driver_type: str = Field(description="Driver type")
+    version: str = Field(description="Driver version")
+    capabilities: list[str] = Field(description="Driver capabilities")
+
+
+class HeartbeatParams(UnrealOnBaseModel):
+    """Parameters for heartbeat processing."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(description="Driver ID")
+    connection_id: str = Field(description="WebSocket connection ID")
+    heartbeat_data: "HeartbeatDataModel" = Field(description="Heartbeat data")
+
+
+class HeartbeatDataModel(UnrealOnBaseModel):
+    """Heartbeat data model."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(description="Driver ID")
+    driver_status: str = Field(description="Driver status")
+    cpu_usage: float = Field(description="CPU usage percentage")
+    memory_usage: float = Field(description="Memory usage in MB")
+    active_tasks: int = Field(description="Number of active tasks")
+    completed_tasks: int = Field(description="Number of completed tasks")
+    failed_tasks: int = Field(description="Number of failed tasks")
+
+
+# Update forward references
+TaskAssignmentParams.model_rebuild()
+TaskDataModel.model_rebuild()
+TaskOptionsModel.model_rebuild()
+TaskResultParams.model_rebuild()
+TaskResultDataModel.model_rebuild()
+ParsedDataModel.model_rebuild()
+LogEntryParams.model_rebuild()
+DriverRegistrationParams.model_rebuild()
+HeartbeatParams.model_rebuild()

unrealon_core/models/arq_responses.py

@@ -0,0 +1,125 @@
+"""
+ARQ Response Models
+
+Strictly typed response models for ARQ job functions.
+Following critical requirements - no raw Dict[str, Any].
+
+Phase 2: Core Systems - ARQ Responses
+"""
+
+from datetime import datetime
+from typing import Optional
+from pydantic import Field, ConfigDict
+
+from .base import UnrealOnBaseModel
+from ..utils.time import utc_now
+
+
+class JobResponseBase(UnrealOnBaseModel):
+    """Base response model for ARQ jobs."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    success: bool = Field(description="Whether operation succeeded")
+    message: str = Field(description="Response message")
+    timestamp: datetime = Field(default_factory=utc_now, description="Response timestamp")
+    error_code: Optional[str] = Field(default=None, description="Error code if failed")
+
+
+class TaskAssignmentResponse(JobResponseBase):
+    """Response for task assignment job."""
+    
+    task_id: str = Field(description="Assigned task ID")
+    driver_id: str = Field(description="Target driver ID")
+    task_type: str = Field(description="Task type")
+    priority: str = Field(description="Task priority")
+
+
+class TaskResultResponse(JobResponseBase):
+    """Response for task result processing job."""
+    
+    task_id: str = Field(description="Task ID")
+    driver_id: str = Field(description="Driver ID")
+    status: str = Field(description="Task status")
+    processing_time: Optional[float] = Field(default=None, description="Processing time in seconds")
+
+
+class LogEntryResponse(JobResponseBase):
+    """Response for log entry processing job."""
+    
+    driver_id: str = Field(description="Driver ID")
+    level: str = Field(description="Log level")
+    entries_processed: int = Field(default=1, description="Number of log entries processed")
+
+
+class DriverRegistrationResponse(JobResponseBase):
+    """Response for driver registration job."""
+    
+    driver_id: str = Field(description="Driver ID")
+    driver_type: str = Field(description="Driver type")
+    capabilities: list[str] = Field(description="Driver capabilities")
+    connection_id: str = Field(description="WebSocket connection ID")
+
+
+class HeartbeatResponse(JobResponseBase):
+    """Response for heartbeat processing job."""
+    
+    driver_id: str = Field(description="Driver ID")
+    status: str = Field(description="Driver status")
+    is_healthy: bool = Field(description="Whether driver is healthy")
+    success_rate: Optional[float] = Field(default=None, description="Driver success rate")
+
+
+class ConfigurationUpdateResponse(JobResponseBase):
+    """Response for configuration update job."""
+    
+    driver_id: str = Field(description="Driver ID")
+    configuration_applied: bool = Field(description="Whether configuration was applied")
+    restart_required: bool = Field(default=False, description="Whether restart is required")
+
+
+class PingResponse(JobResponseBase):
+    """Response for ping job."""
+    
+    pong_timestamp: datetime = Field(description="Pong timestamp")
+    latency_ms: Optional[float] = Field(default=None, description="Latency in milliseconds")
+
+
+class DriverStatusResponse(JobResponseBase):
+    """Response for driver status query."""
+    
+    driver_id: str = Field(description="Driver ID")
+    status: str = Field(description="Driver status")
+    last_seen: Optional[datetime] = Field(default=None, description="Last seen timestamp")
+    active_tasks: int = Field(default=0, description="Number of active tasks")
+
+
+class DriverListResponse(JobResponseBase):
+    """Response for driver list query."""
+    
+    drivers: list["DriverInfo"] = Field(description="List of available drivers")
+    total_count: int = Field(description="Total number of drivers")
+    filter_applied: Optional[str] = Field(default=None, description="Filter that was applied")
+
+
+class DriverInfo(UnrealOnBaseModel):
+    """Driver information for list response."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(description="Driver ID")
+    driver_type: str = Field(description="Driver type")
+    status: str = Field(description="Driver status")
+    capabilities: list[str] = Field(description="Driver capabilities")
+    last_seen: Optional[datetime] = Field(default=None, description="Last seen timestamp")
+    active_tasks: int = Field(default=0, description="Number of active tasks")
+
+
+# Update forward references
+DriverListResponse.model_rebuild()

unrealon_core/models/base.py

@@ -0,0 +1,291 @@
+"""
+Base Pydantic models for UnrealOn system.
+
+These models provide common functionality and validation patterns
+used throughout the system. Built with Pydantic v2 for maximum
+performance and type safety.
+
+Phase 1: Bedrock Foundation - These models are the foundation
+for all other models in the system.
+"""
+
+from datetime import datetime
+from typing import Optional, Dict, Any, Union
+from pydantic import BaseModel, Field, ConfigDict, field_validator
+import uuid
+
+from ..utils.time import utc_now
+
+
+class UnrealOnBaseModel(BaseModel):
+    """
+    Base model for all UnrealOn models.
+    
+    Provides:
+    - Strict validation (no extra fields)
+    - JSON serialization/deserialization
+    - Type safety with Pydantic v2
+    - Common utility methods
+    """
+    
+    model_config = ConfigDict(
+        # Pydantic v2 configuration for maximum strictness
+        str_strip_whitespace=True,      # Auto-strip whitespace
+        validate_assignment=True,       # Validate on assignment
+        use_enum_values=True,          # Use enum values in serialization
+        extra='forbid',                # Strict - no extra fields allowed
+        frozen=False,                  # Allow mutation by default
+        arbitrary_types_allowed=True,  # Allow custom types
+        validate_default=True,         # Validate default values
+    )
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert model to dictionary for JSON serialization."""
+        return self.model_dump(mode='json', exclude_none=True)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]):
+        """Create instance from dictionary."""
+        return cls.model_validate(data)
+    
+    def to_json(self) -> str:
+        """Convert model to JSON string."""
+        return self.model_dump_json(exclude_none=True)
+    
+    @classmethod
+    def from_json(cls, json_str: str):
+        """Create instance from JSON string."""
+        return cls.model_validate_json(json_str)
+    
+    def update_from_dict(self, data: Dict[str, Any]) -> 'UnrealOnBaseModel':
+        """Update model fields from dictionary."""
+        for key, value in data.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        return self
+    
+    def copy_with_updates(self, **updates) -> 'UnrealOnBaseModel':
+        """Create a copy with updated fields."""
+        return self.model_copy(update=updates)
+
+
+class TimestampedModel(UnrealOnBaseModel):
+    """
+    Base model with automatic timestamp fields.
+    
+    Provides:
+    - created_at: Auto-set on creation
+    - updated_at: Manual update via touch()
+    - age_seconds: Calculated property
+    """
+    
+    created_at: datetime = Field(
+        default_factory=utc_now,
+        description="Timestamp when the model was created"
+    )
+    updated_at: Optional[datetime] = Field(
+        default=None,
+        description="Timestamp when the model was last updated"
+    )
+    
+    def touch(self) -> 'TimestampedModel':
+        """Update the updated_at timestamp."""
+        self.updated_at = utc_now()
+        return self
+    
+    @property
+    def age_seconds(self) -> float:
+        """Get age of the model in seconds."""
+        return (utc_now() - self.created_at).total_seconds()
+    
+    @property
+    def last_modified_seconds(self) -> float:
+        """Get seconds since last modification."""
+        reference_time = self.updated_at or self.created_at
+        return (utc_now() - reference_time).total_seconds()
+
+
+class IdentifiedModel(UnrealOnBaseModel):
+    """
+    Base model with unique ID field.
+    
+    Provides:
+    - id: UUID4 string by default
+    - String representation
+    - Equality comparison by ID
+    """
+    
+    id: str = Field(
+        default_factory=lambda: str(uuid.uuid4()),
+        description="Unique identifier for the model",
+        min_length=1
+    )
+    
+    @field_validator('id')
+    @classmethod
+    def validate_id(cls, v: str) -> str:
+        """Validate ID is not empty."""
+        if not v or not v.strip():
+            raise ValueError("ID cannot be empty")
+        return v.strip()
+    
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}(id={self.id})"
+    
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(id='{self.id}')"
+    
+    def __eq__(self, other) -> bool:
+        """Compare models by ID."""
+        if not isinstance(other, IdentifiedModel):
+            return False
+        return self.id == other.id
+    
+    def __hash__(self) -> int:
+        """Hash by ID for use in sets/dicts."""
+        return hash(self.id)
+
+
+class StatusModel(UnrealOnBaseModel):
+    """
+    Base model with status tracking.
+    
+    Provides:
+    - status: Current status string
+    - status_message: Optional status description
+    - status_updated_at: Timestamp of last status change
+    - Status update methods
+    """
+    
+    status: str = Field(
+        description="Current status of the model",
+        min_length=1
+    )
+    status_message: Optional[str] = Field(
+        default=None,
+        description="Optional message describing the status"
+    )
+    status_updated_at: datetime = Field(
+        default_factory=utc_now,
+        description="Timestamp when status was last updated"
+    )
+    
+    @field_validator('status')
+    @classmethod
+    def validate_status(cls, v: str) -> str:
+        """Validate status is not empty."""
+        if not v or not v.strip():
+            raise ValueError("Status cannot be empty")
+        return v.strip()
+    
+    def update_status(self, status: str, message: Optional[str] = None) -> 'StatusModel':
+        """Update status with timestamp and optional message."""
+        self.status = status
+        self.status_message = message
+        self.status_updated_at = utc_now()
+        return self
+    
+    @property
+    def status_age_seconds(self) -> float:
+        """Get age of current status in seconds."""
+        return (utc_now() - self.status_updated_at).total_seconds()
+    
+    def is_status(self, *statuses: str) -> bool:
+        """Check if current status matches any of the provided statuses."""
+        return self.status in statuses
+
+
+class MetadataModel(UnrealOnBaseModel):
+    """
+    Base model with metadata field.
+    
+    Provides:
+    - metadata: Dictionary for arbitrary key-value data
+    - Helper methods for metadata manipulation
+    """
+    
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Arbitrary metadata key-value pairs"
+    )
+    
+    def set_metadata(self, key: str, value: Any) -> 'MetadataModel':
+        """Set metadata value."""
+        self.metadata[key] = value
+        return self
+    
+    def get_metadata(self, key: str, default: Any = None) -> Any:
+        """Get metadata value with optional default."""
+        return self.metadata.get(key, default)
+    
+    def has_metadata(self, key: str) -> bool:
+        """Check if metadata key exists."""
+        return key in self.metadata
+    
+    def remove_metadata(self, key: str) -> 'MetadataModel':
+        """Remove metadata key if it exists."""
+        self.metadata.pop(key, None)
+        return self
+    
+    def clear_metadata(self) -> 'MetadataModel':
+        """Clear all metadata."""
+        self.metadata.clear()
+        return self
+    
+    def update_metadata(self, **updates) -> 'MetadataModel':
+        """Update metadata with multiple key-value pairs."""
+        self.metadata.update(updates)
+        return self
+
+
+# Combined base models for common use cases
+class FullBaseModel(IdentifiedModel, TimestampedModel, StatusModel, MetadataModel):
+    """
+    Complete base model with ID, timestamps, status, and metadata.
+    
+    Use this for complex entities that need full tracking:
+    - Drivers
+    - Tasks
+    - Complex configurations
+    """
+    pass
+
+
+class SimpleBaseModel(IdentifiedModel, TimestampedModel):
+    """
+    Simple base model with ID and timestamps only.
+    
+    Use this for simple entities:
+    - Messages
+    - Simple configurations
+    - Temporary objects
+    """
+    pass
+
+
+class ConfigBaseModel(UnrealOnBaseModel):
+    """
+    Base model for configuration objects.
+    
+    Provides validation for configuration models
+    without unnecessary fields like timestamps.
+    """
+    
+    def validate_config(self) -> bool:
+        """Override in subclasses to add custom validation."""
+        return True
+    
+    def merge_with(self, other: 'ConfigBaseModel') -> 'ConfigBaseModel':
+        """Merge this config with another, other takes precedence."""
+        if not isinstance(other, self.__class__):
+            raise ValueError(f"Cannot merge {self.__class__} with {other.__class__}")
+        
+        # Get all fields from both models
+        self_dict = self.model_dump()
+        other_dict = other.model_dump()
+        
+        # Merge dictionaries (other takes precedence)
+        merged = {**self_dict, **other_dict}
+        
+        # Create new instance
+        return self.__class__.model_validate(merged)