unrealon 2.0.4__tar.gz → 2.0.6__tar.gz

{unrealon-2.0.4/unrealon.egg-info → unrealon-2.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.4
+Version: 2.0.6
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT

{unrealon-2.0.4 → unrealon-2.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unrealon"
-version = "2.0.4"
+version = "2.0.6"
 description = "Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities"
 authors = [
     {name = "UnrealOn Team", email = "team@unrealon.com"}

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-browser/src/unrealon_browser/cli/browser_cli.py

@@ -16,7 +16,7 @@ from rich.table import Table
 from unrealon_browser.core.browser_manager import BrowserManager
 from unrealon_browser.dto.models.config import BrowserConfig, BrowserType, BrowserMode
 
-from unrealon_core.config import get_url_config
+from unrealon_core.config.urls import get_url_config
 
 console = Console()
 

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-browser/src/unrealon_browser/cli/interactive_mode.py

@@ -11,7 +11,7 @@ from rich.panel import Panel
 from rich.table import Table
 from typing import Dict, Any
 
-from unrealon_core.config import get_url_config
+from unrealon_core.config.urls import get_url_config
 
 console = Console()
 

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-browser/src/unrealon_browser/dto/models/config.py

@@ -8,7 +8,7 @@ from typing import Optional
 from pydantic import BaseModel, Field, ConfigDict
 from .enums import BrowserType, BrowserMode
 
-from unrealon_core.config import get_url_config
+from unrealon_core.config.urls import get_url_config
 
 
 def _get_default_stealth_url() -> str:

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-core/src/unrealon_core/__init__.py

@@ -42,7 +42,10 @@ from .models.communication import (
     create_error_message, create_ack_message,
     # Data models for strict typing
     TaskAssignmentData, TaskResultData,
-    DriverRegistrationData, HeartbeatData, LogEntryData, LogContext
+    DriverRegistrationData, HeartbeatData, LogEntryData, LogContext,
+    # Broadcast models
+    DriverBroadcastData, DriverRegisterBroadcast,
+    DriverHeartbeatBroadcast, DriverDisconnectBroadcast
 )
 from .enums.status import DriverStatus, TaskStatus, ProxyStatus, LogLevel
 from .enums.types import MessageType, ProxyType, TaskPriority
@@ -76,9 +79,19 @@ from .monitoring import (
 )
 
 # Configuration system
-from .config import (
-    EnvironmentConfig, URLConfig,
-    get_environment_config, get_url_config
+from .config.environment import (
+    EnvironmentConfig, Environment,
+    get_environment_config, set_environment_config
+)
+from .config.urls import (
+    URLConfig,
+    get_url_config
+)
+from .models.websocket.config import (
+    DriverConfiguration,
+    LoggingConfiguration,
+    TaskConfiguration,
+    ProxyConfiguration
 )
 
 __all__ = [
@@ -117,6 +130,12 @@ __all__ = [
     "LogEntryData",
     "LogContext",
     
+    # Broadcast models
+    "DriverBroadcastData",
+    "DriverRegisterBroadcast",
+    "DriverHeartbeatBroadcast", 
+    "DriverDisconnectBroadcast",
+    
     # Status enums
     "DriverStatus",
     "TaskStatus",
@@ -155,6 +174,8 @@ __all__ = [
     "MonitoringDashboard", "get_system_overview",
     
     # Configuration system
-    "EnvironmentConfig", "URLConfig",
-    "get_environment_config", "get_url_config",
+    "EnvironmentConfig", "Environment", "URLConfig",
+    "get_environment_config", "set_environment_config", "get_url_config",
+    "DriverConfiguration", "LoggingConfiguration", 
+    "TaskConfiguration", "ProxyConfiguration",
 ]

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-core/src/unrealon_core/config/environment.py

@@ -18,7 +18,7 @@ class Environment(str, Enum):
 
 
 class EnvironmentConfig(BaseModel):
-    """Environment configuration settings."""
+    """Environment configuration settings with all system URLs and settings."""
     
     model_config = ConfigDict(
         # Add any specific config here if needed
@@ -39,6 +39,17 @@ class EnvironmentConfig(BaseModel):
         description="Logging level"
     )
     
+    # System URLs - environment-aware
+    redis_url: str = Field(
+        default_factory=lambda: os.getenv("REDIS_URL", "redis://localhost:6379/0"),
+        description="Redis connection URL"
+    )
+    
+    max_workers: int = Field(
+        default_factory=lambda: int(os.getenv("MAX_WORKERS", "10")),
+        description="Maximum number of workers"
+    )
+    
     @classmethod
     def from_env(cls) -> "EnvironmentConfig":
         """Create config from environment variables."""
@@ -76,6 +87,48 @@ class EnvironmentConfig(BaseModel):
     def is_testing(self) -> bool:
         """Check if running in testing mode."""
         return self.environment == Environment.TESTING
+    
+    @property
+    def websocket_url(self) -> str:
+        """Get WebSocket URL based on environment."""
+        # Check explicit env var first
+        if ws_url := os.getenv("WEBSOCKET_URL"):
+            return ws_url
+        
+        if self.is_production:
+            return "wss://ws.unrealon.com/ws"
+        elif self.is_development:
+            return "ws://localhost:8001/ws"  # RPC server port
+        else:  # testing
+            return "ws://localhost:8001/ws"
+    
+    @property
+    def api_url(self) -> str:
+        """Get API URL based on environment."""
+        # Check explicit env var first
+        if api_url := os.getenv("API_URL"):
+            return api_url
+        
+        if self.is_production:
+            return "https://api-m.unrealon.com"
+        elif self.is_development:
+            return "http://localhost:8002"  # Backend server port
+        else:  # testing
+            return "http://localhost:8002"
+    
+    @property
+    def django_api_url(self) -> str:
+        """Get Django API URL based on environment."""
+        # Check explicit env var first
+        if api_url := os.getenv("DJANGO_API_URL"):
+            return api_url
+        
+        if self.is_production:
+            return "https://api.unrealon.com"
+        elif self.is_development:
+            return "http://localhost:8000"  # Django server port
+        else:  # testing
+            return "http://localhost:8000"
 
 
 # Global config instance

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-core/src/unrealon_core/config/urls.py

@@ -18,6 +18,7 @@ class URLConfig(BaseModel):
     
     # Scanner/Detection URLs
     scanner_url: str = Field(
+        default="https://cloud.unrealon.com/scanner",
         description="URL for browser detection and stealth testing"
     )
     
@@ -39,19 +40,19 @@ class URLConfig(BaseModel):
         
         if environment == Environment.PRODUCTION:
             return cls(
-                scanner_url="https://cloud.unrealon.com/scanner",
+                # scanner_url="https://cloud.unrealon.com/scanner",
                 cloud_base_url="https://cloud.unrealon.com",
                 api_base_url="https://api.unrealon.com"
             )
-        elif environment == Environment.TESTING:
-            return cls(
-                scanner_url="https://staging.unrealon.com/scanner", 
-                cloud_base_url="https://staging.unrealon.com",
-                api_base_url="https://api-staging.unrealon.com"
-            )
+        # elif environment == Environment.TESTING:
+        #     return cls(
+        #         scanner_url="https://staging.unrealon.com/scanner", 
+        #         cloud_base_url="https://staging.unrealon.com",
+        #         api_base_url="https://api-staging.unrealon.com"
+        #     )
         else:  # Development
             return cls(
-                scanner_url="http://localhost:3000/scanner",
+                # scanner_url="http://localhost:3000/scanner",
                 cloud_base_url="http://localhost:3000", 
                 api_base_url="http://localhost:8000"
             )

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-core/src/unrealon_core/enums/types.py

@@ -53,6 +53,7 @@ class MessageType(str, Enum):
     LOG_BATCH = "log_batch"                       # Batch of log messages
     
     # System messages
+    WELCOME = "welcome"                           # Welcome message from server
     PING = "ping"                                 # Ping message
     PONG = "pong"                                 # Pong response
     ERROR = "error"                               # Error message

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-core/src/unrealon_core/models/__init__.py

@@ -46,16 +46,6 @@ from .logging import (
     LogMetrics
 )
 
-from .config import (
-    SystemConfig,
-    HttpConfig,
-    ProxyConfig,
-    BrowserConfig, 
-    LoggingConfig,
-    CacheConfig,
-    ThreadConfig
-)
-
 __all__ = [
     # Base models
     "UnrealOnBaseModel",
@@ -86,13 +76,4 @@ __all__ = [
     "LogEntry",
     "LogQuery",
     "LogMetrics",
-    
-    # Configuration models
-    "SystemConfig",
-    "HttpConfig",
-    "ProxyConfig",
-    "BrowserConfig", 
-    "LoggingConfig",
-    "CacheConfig",
-    "ThreadConfig",
 ]

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-core/src/unrealon_core/models/websocket/__init__.py

@@ -38,6 +38,12 @@ from .proxy import (
 )
 
 from .utils import create_error_message, create_ack_message
+from .broadcast import (
+    DriverBroadcastData,
+    DriverRegisterBroadcast,
+    DriverHeartbeatBroadcast,
+    DriverDisconnectBroadcast
+)
 
 __all__ = [
     # Base
@@ -87,5 +93,11 @@ __all__ = [
     
     # Utilities
     'create_error_message',
-    'create_ack_message'
+    'create_ack_message',
+    
+    # Broadcast messages
+    'DriverBroadcastData',
+    'DriverRegisterBroadcast',
+    'DriverHeartbeatBroadcast',
+    'DriverDisconnectBroadcast'
 ]

unrealon-2.0.6/unrealon-core/src/unrealon_core/models/websocket/broadcast.py

@@ -0,0 +1,137 @@
+"""
+Broadcast WebSocket Models.
+
+Strictly typed models for broadcasting driver events to monitoring clients.
+These are Server → Client messages for real-time monitoring.
+
+Phase 2: Core Systems - WebSocket Bridge
+"""
+
+from typing import List, Optional
+from pydantic import Field, ConfigDict
+
+from ..base import UnrealOnBaseModel
+from .base import WebSocketMessage, MessageType
+
+
+class DriverBroadcastData(UnrealOnBaseModel):
+    """Driver broadcast data payload - strictly typed."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    driver_id: str = Field(
+        description="Unique driver identifier",
+        min_length=1
+    )
+    
+    driver_type: str = Field(
+        description="Type of driver (e.g., 'universal', 'ecommerce')",
+        min_length=1
+    )
+    
+    status: str = Field(
+        default="active",
+        pattern=r"^(active|idle|busy|offline|error)$",
+        description="Current driver status"
+    )
+    
+    capabilities: List[str] = Field(
+        default_factory=list,
+        description="List of supported task types"
+    )
+    
+    active_tasks: int = Field(
+        default=0,
+        ge=0,
+        description="Number of currently active tasks"
+    )
+    
+    completed_tasks: int = Field(
+        default=0,
+        ge=0,
+        description="Total completed tasks since startup"
+    )
+    
+    failed_tasks: int = Field(
+        default=0,
+        ge=0,
+        description="Total failed tasks since startup"
+    )
+    
+    uptime_seconds: float = Field(
+        default=0.0,
+        ge=0.0,
+        description="Driver uptime in seconds"
+    )
+    
+    last_seen: Optional[str] = Field(
+        default=None,
+        description="ISO timestamp of last heartbeat"
+    )
+    
+    connected_at: Optional[str] = Field(
+        default=None,
+        description="ISO timestamp when driver connected"
+    )
+    
+    disconnected_at: Optional[str] = Field(
+        default=None,
+        description="ISO timestamp when driver disconnected"
+    )
+
+
+class DriverRegisterBroadcast(WebSocketMessage):
+    """Driver registration broadcast message (Server → Monitoring Clients)."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    type: MessageType = Field(
+        default=MessageType.DRIVER_REGISTER,
+        frozen=True
+    )
+    
+    data: DriverBroadcastData = Field(
+        description="Driver registration broadcast data"
+    )
+
+
+class DriverHeartbeatBroadcast(WebSocketMessage):
+    """Driver heartbeat broadcast message (Server → Monitoring Clients)."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    type: MessageType = Field(
+        default=MessageType.DRIVER_HEARTBEAT,
+        frozen=True
+    )
+    
+    data: DriverBroadcastData = Field(
+        description="Driver heartbeat broadcast data"
+    )
+
+
+class DriverDisconnectBroadcast(WebSocketMessage):
+    """Driver disconnect broadcast message (Server → Monitoring Clients)."""
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid"
+    )
+    
+    type: MessageType = Field(
+        default=MessageType.DRIVER_DISCONNECT,
+        frozen=True
+    )
+    
+    data: DriverBroadcastData = Field(
+        description="Driver disconnect broadcast data"
+    )

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-driver/src/unrealon_driver/driver/communication/session.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel, Field
 
 from unrealon_core.models.websocket import (
     TaskAssignmentData,
+    TaskAssignmentMessage,
     TaskResultData,
     WebSocketMessage
 )
@@ -69,21 +70,26 @@ class DriverSession:
         
         try:
             # Connect WebSocket
+            logger.info(f"🔌 Connecting WebSocket for driver: {self.driver_id}")
             self.status = SessionStatus.CONNECTING
             if not await self.websocket_client.connect():
+                logger.error(f"❌ WebSocket connection failed for driver: {self.driver_id}")
                 self.status = SessionStatus.ERROR
                 return False
             
+            logger.info(f"✅ WebSocket connected for driver: {self.driver_id}")
             self.status = SessionStatus.CONNECTED
             self.stats.connected_at = utc_now()
             
             # Register driver
+            logger.info(f"📝 Registering driver: {self.driver_id} with capabilities: {capabilities}")
             if await self.register(capabilities or []):
                 self.status = SessionStatus.REGISTERED
                 self.stats.registered_at = utc_now()
-                logger.info(f"Session started for driver: {self.driver_id}")
+                logger.info(f"✅ Session started for driver: {self.driver_id}")
                 return True
             else:
+                logger.error(f"❌ Driver registration failed for: {self.driver_id}")
                 self.status = SessionStatus.ERROR
                 return False
                 
@@ -139,8 +145,11 @@ class DriverSession:
     async def _handle_task_assignment(self, message: WebSocketMessage):
         """Handle task assignment."""
         try:
-            # Validate task data
-            task_data = TaskAssignmentData.model_validate(message.data)
+            # Parse as TaskAssignmentMessage
+            
+            # Convert to proper task assignment message
+            task_message = TaskAssignmentMessage.model_validate(message.model_dump())
+            task_data = task_message.data
             
             # Find handler
             handler = self.task_handlers.get(task_data.task_type)

{unrealon-2.0.4 → unrealon-2.0.6}/unrealon-driver/src/unrealon_driver/driver/communication/websocket_client.py

@@ -9,7 +9,15 @@ from typing import Optional, Callable, List
 from collections import deque
 
 import websockets
-from websockets.asyncio.client import ClientConnection
+try:
+    from websockets.asyncio.client import ClientConnection
+except ImportError:
+    # Fallback for older websockets versions
+    try:
+        from websockets.client import WebSocketClientProtocol as ClientConnection
+    except ImportError:
+        # Ultimate fallback
+        ClientConnection = None
 
 from unrealon_core.models.websocket import (
     DriverRegistrationMessage,
@@ -32,9 +40,11 @@ class WebSocketClient:
     - Clean error handling
     """
     
-    def __init__(self, websocket_url: str, driver_id: str):
+    def __init__(self, websocket_url: str, driver_id: str, custom_logger=None):
         self.websocket_url = websocket_url
         self.driver_id = driver_id
+        # Use custom logger if provided, otherwise use default
+        self._logger = custom_logger if custom_logger else logger
         self.websocket: Optional[ClientConnection] = None
         self.connected = False
         self.running = False
@@ -86,20 +96,20 @@ class WebSocketClient:
             self.websocket = None
         
         self.connected = False
-        logger.info("WebSocket client stopped")
+        self._logger.info("WebSocket client stopped")
     
     async def _establish_connection(self) -> bool:
         """Establish WebSocket connection."""
         try:
-            logger.info(f"Connecting to WebSocket: {self.websocket_url}")
+            self._logger.info(f"Connecting to WebSocket: {self.websocket_url}")
             self.websocket = await websockets.connect(self.websocket_url)
             self.connected = True
             self.reconnect_attempts = 0
             self.reconnect_delay = 1.0
-            logger.info("WebSocket connected successfully")
+            self._logger.info("WebSocket connected successfully")
             return True
         except Exception as e:
-            logger.error(f"WebSocket connection failed: {e}")
+            self._logger.error(f"WebSocket connection failed: {e}")
             self.connected = False
             return False
     
@@ -107,10 +117,10 @@ class WebSocketClient:
         """Monitor connection and handle reconnection."""
         while self.running:
             if not self.connected and self.running:
-                logger.info(f"Attempting reconnection (attempt {self.reconnect_attempts + 1})")
+                self._logger.info(f"Attempting reconnection (attempt {self.reconnect_attempts + 1})")
                 
                 if await self._establish_connection():
-                    logger.info("Reconnection successful")
+                    self._logger.info("Reconnection successful")
                 else:
                     self.reconnect_attempts += 1
                     # Exponential backoff
@@ -128,13 +138,16 @@ class WebSocketClient:
             if self.connected and self.websocket and self.message_queue:
                 try:
                     message = self.message_queue.popleft()
+                    self._logger.info(f"📤 Sending WebSocket message: {message[:200]}...")  # Log first 200 chars
                     await self.websocket.send(message)
+                    self._logger.info(f"✅ Message sent successfully")
                 except (websockets.exceptions.ConnectionClosed, ConnectionResetError):
                     self.connected = False
                     # Put message back in queue
                     self.message_queue.appendleft(message)
+                    self._logger.warning("🔌 Connection lost, message queued for retry")
                 except Exception as e:
-                    logger.error(f"Error sending message: {e}")
+                    self._logger.error(f"❌ Error sending message: {e}")
             else:
                 await asyncio.sleep(0.1)
     
@@ -155,9 +168,9 @@ class WebSocketClient:
                         
                 except (websockets.exceptions.ConnectionClosed, ConnectionResetError):
                     self.connected = False
-                    logger.warning("WebSocket connection lost")
+                    self._logger.warning("WebSocket connection lost")
                 except Exception as e:
-                    logger.error(f"Error receiving message: {e}")
+                    self._logger.error(f"Error receiving message: {e}")
             else:
                 await asyncio.sleep(0.1)
     
@@ -189,9 +202,9 @@ class WebSocketClient:
             # Queue for sending
             self.send(registration_message)
             
-            logger.info(f"Driver registration queued: {self.driver_id}")
+            self._logger.info(f"Driver registration queued: {self.driver_id}")
             return True
             
         except Exception as e:
-            logger.error(f"Driver registration failed: {e}")
+            self._logger.error(f"Driver registration failed: {e}")
             return False